Replace usage of RSA BSAFE lockbox file with custom implementation.

The proprietary RSA BSAFE library was used for one thing only: for storing
the "lockbox" file containing the credentials for DDBoost. Replace with
a little hand-written key-value configuration file format, with password
obfuscation based on XOR, rand() and base64 encoding. This allows us to
finally remove the dependency to the BSAFE library.

The configuration file is now in a human-readable and -editable format
(except for the password), which is a bonus.

gpAux/Makefile

@@ -1152,13 +1152,7 @@ endif
 
 libcopy=(cd $(1) && $(TAR) cf - $(2)) | (cd $(3) && $(TAR) xvf -)$(check_pipe_for_errors)
 
-.PHONY: copy-rsa-libs copy-nbu-libs
-
-copy-rsa-libs:
-	# Copy rsa lockbox libs - supported only on rhel (required for Data
-ifeq "$(BLD_ARCH)" "rhel5_x86_64"
-	cp -fpr $(BLD_THIRDPARTY_LIB_DIR)/rsa_csp $(INSTLOC)/lib/;
-endif
+.PHONY: copy-nbu-libs
 
 copy-nbu-libs:
 	#Copy NetBackup libs
@@ -1171,7 +1165,7 @@ ifeq "$(BLD_ARCH)" "rhel5_x86_64"
 	cp -fpr $(BLD_THIRDPARTY_LIB_DIR)/../Netbackup/nbu75 $(INSTLOC)/lib/;
 endif
 
-copylibs : thirdparty-dist copy-rsa-libs copy-nbu-libs
+copylibs : thirdparty-dist copy-nbu-libs
 	if [ `uname -s` = 'SunOS' ]; then cp `ldd $(INSTLOC)/bin/psql | grep gcc | head -1 | awk '{print $$3}'` $(INSTLOC)/lib; fi
 	if [ `uname -s` = 'Darwin' ] ; then \
 	    for lib in `otool -L $(INSTLOC)/bin/psql | egrep 'libssl|libcrypto|libcom_err|libkrb5|curl|readline|numa' | awk '{print $$1}'`; do \

src/bin/pg_dump/cdb/Makefile

@@ -25,7 +25,6 @@ LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses -lcurl -lssl
 
 ifeq ($(enable_ddboost), yes)
 DDBOOSTLIB += -lDDBoost
-override CFLAGS += -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb
 endif
 
 ifeq ($(enable_netbackup), yes)
@@ -51,23 +50,23 @@ KEYWRDOBJS = ../keywords.o ../kwlookup.o
 
 all: submake-libpq submake-libpgport cdb_dump cdb_dump_agent cdb_restore cdb_restore_agent gpddboost libgpbsa.so libgpbsa75.so libgpbsa71.so cdb_bsa_dump_agent cdb_bsa_restore_agent cdb_bsa_query_agent cdb_bsa_delete_agent
 
-cdb_dump: cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a $(EXTRA_OBJS) 
-	$(CC) $(CFLAGS) cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
+cdb_dump: cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o cdb_lockbox.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a $(EXTRA_OBJS)
+	$(CC) $(CFLAGS) cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o cdb_lockbox.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
 
-cdb_dump_agent: cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_archiver.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
+cdb_dump_agent: cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_archiver.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
 
-#cdb_dumpall_agent: cdb_dumpall_agent.o ../common.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a 
+#cdb_dumpall_agent: cdb_dumpall_agent.o ../common.o cdb_backup_archiver.o cdb_dump_util.o cdb_lockbox.o cdb_seginst.o cdb_table.o cdb_backup_status.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a
 #	$(CC) $(CFLAGS) cdb_dumpall_agent.o cdb_backup_archiver.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(LIBS) -o $@
 
-cdb_restore: cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a 
-	$(CC) $(CFLAGS) cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(KEYWRDOBJS) $(PGDUMP_DIR)/dumputils.o $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
+cdb_restore: cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(KEYWRDOBJS) $(PGDUMP_DIR)/dumputils.o $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
 
-cdb_restore_agent: cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o $(OBJS) $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_restore_agent.o  cdb_backup_archiver.o  cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
+cdb_restore_agent: cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_lockbox.o $(OBJS) $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_restore_agent.o  cdb_backup_archiver.o  cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS)  $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
 
-gpddboost: cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a 
-	$(CC) $(CFLAGS) cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
+gpddboost: cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@
 
 libgpbsa75.so: cdb_bsa_util.c $(libpq_builddir)/libpq.a
 	$(CC) $(CFLAGS_SL) $(CPPFLAGS) cdb_bsa_util.c $(libpq) $(LDFLAGS) -shared $(DDBOOSTLIB) $(NETBACKUPLIB75) $(LIBS) -o $@
@@ -78,17 +77,17 @@ libgpbsa71.so: cdb_bsa_util.o $(libpq_builddir)/libpq.a
 libgpbsa.so: libgpbsa75.so
 	cp libgpbsa75.so libgpbsa.so
 
-cdb_bsa_dump_agent: cdb_bsa_dump_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_bsa_dump_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
+cdb_bsa_dump_agent: cdb_bsa_dump_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_bsa_dump_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
 
-cdb_bsa_restore_agent: cdb_bsa_restore_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_bsa_restore_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
+cdb_bsa_restore_agent: cdb_bsa_restore_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_bsa_restore_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
 
-cdb_bsa_query_agent: cdb_bsa_query_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_bsa_query_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
+cdb_bsa_query_agent: cdb_bsa_query_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_bsa_query_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
 
-cdb_bsa_delete_agent: cdb_bsa_delete_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a
-	$(CC) $(CFLAGS) cdb_bsa_delete_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
+cdb_bsa_delete_agent: cdb_bsa_delete_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a
+	$(CC) $(CFLAGS) cdb_bsa_delete_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@
 
 .PHONY: submake-backend
 submake-backend:

src/bin/pg_dump/cdb/cdb_dump_util.c

@@ -22,6 +22,7 @@
 #include "pqexpbuffer.h"
 #include "../dumputils.h"
 #include "cdb_dump_util.h"
+#include "cdb_lockbox.h"
 
 #define DDP_CL_DDP 1
 #define DEFAULT_STORAGE_UNIT "GPDB"
@@ -1212,8 +1213,6 @@ formPostDataFilterCommandLine(char** retVal, const char* post_data_filter_script
 }
 
 #ifdef USE_DDBOOST
-#include <dlfcn.h>
-#include "clb_base.h"
 
 #define	NO_ERR	0
 
@@ -1224,315 +1223,178 @@ struct ddboost_logs
 	unsigned int logsSize ;
 } ddboost_logs_info;
 
-/* define function pointers in order to use the LB functions with dlsym */
-void (*clb_getErrorMessage)(int errorCode, char **errorMessage) = NULL;
-int (*clb_create)(const char* lockboxId, const char* passphrase, int overwrite, clbContext lbCntx, clbHandle *clbH) = NULL;
-int (*clb_open)(const char* lockboxId, const char* passphrase, clbContext lbCntx, clbHandle *clbH) = NULL;
-int (*clb_close)(clbHandle clbH) = NULL;
-void (*clb_free)(void *buffer) = NULL;
-int (*clb_setLockboxMode)(clbHandle clbH, clb_mode_e mode) = NULL;
-int (*clb_createItemAsText)(clbHandle clbH, const char* itemName, const char* secret) = NULL;
-int (*clb_retrieveItemAsText)(clbHandle clbH, const char* itemName, char** secret) = NULL;
-
-
-static int setItem(clbHandle* LB, char *key, char *value);
-static int getItem(clbHandle* LB, char *key, char **value);
-static int setLBEnv(void);
-static int createLB(clbHandle* LB,char* name);
-static int openLB(clbHandle* LB,char* name);
 static int validateDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, bool remote);
-int getDDBoostCredential(char** hostname, char** user, char** password, char** log_level ,char** log_size, char **default_backup_directory, char **ddboost_storage_unit, bool remote);
 
 /*
- * Set the environment variable LD_LIBRARY_PATH in order to dynamically load LB's libraries.
+ * Write the hostname, user, password, log_level and log_size to the LB
  * Returns 0 in case of success, and -1 otherwise.
  */
-static int
-setLBEnv(void)
+int
+setDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, char *ddboost_storage_unit, bool remote)
 {
-	void *hdl = NULL;
-	char *gphome = getenv("GPHOME");
-	char *ldpath = getenv("LD_LIBRARY_PATH");
-	char LBpath[PATH_MAX];
-	char *newldpath = NULL;
-	char libpath[PATH_MAX];
-	char *libdirname = "lib/rsa_csp";
-	char *libname = "libCSP-lb.so";
-	int  newldpath_len = 0;
-
-	if (NULL == gphome)
-	{
-		mpp_err_msg("ERROR", "ddboost", "GPHOME undefined, can't set ddboost credentials\n");
-		return -1;
-	}
+#define MAX_ITEMS 7
+	lockbox_content content;
+	lockbox_item items[MAX_ITEMS];
+	int			nitems;
+	char		filepath[MAXPGPATH];
+	char	   *filename;
+	char	   *home;
+	char	   *obfuscated_pw;
 
-	if (NULL == ldpath)
-	{
-		mpp_err_msg("ERROR", "ddboost", "LD_LIBRARY_PATH undefined, can't set ddboost credentials\n");
-		return -1;
-	}
+	/*
+	 * TODO: validate default backup directory name if needed
+	 * TODO: validate storage unit
+	 */
+	if (validateDDBoostCredential(hostname, user, password,
+								  log_level, log_size,
+								  default_backup_directory, remote))
+		return -1;	/* validateDDBoostCredential() reported an error to user already */
 
-	snprintf(LBpath, strlen(gphome) + strlen(libdirname) + 2, "%s/%s", gphome, libdirname);
+	obfuscated_pw = lb_obfuscate(password);
+	if (!obfuscated_pw)
+		return -1;	/* lb_obfuscate() reported an error to user already */
 
-	newldpath_len = strlen(ldpath) + strlen(LBpath) + 2;
-	newldpath = malloc(newldpath_len);
+	nitems = 0;
 
-	if (NULL == newldpath)
-	{
-		mpp_err_msg("ERROR", "ddboost", "Memory allocation failed during DDBoost credentials initialization\n");
-		return -1;
-	}
+	items[nitems].key = "hostname";
+	items[nitems].value = hostname;
+	nitems++;
 
-	snprintf(newldpath, newldpath_len, "%s:%s", LBpath, ldpath);
-	setenv("LD_LIBRARY_PATH", newldpath, 1);
-	free(newldpath);
-	newldpath = NULL;
+	items[nitems].key = "user";
+	items[nitems].value = user;
+	nitems++;
 
-	snprintf(libpath, strlen(LBpath) + strlen(libname) + 2, "%s/%s", LBpath, libname);
+	items[nitems].key = "password";
+	items[nitems].value = obfuscated_pw;
+	nitems++;
 
-	if (NULL == (hdl = dlopen(libpath, RTLD_NOW | RTLD_LOCAL)))
+	if (!remote)
 	{
-		mpp_err_msg("ERROR", "ddboost", "libCSP-lb.so was not found. Can't set ddboost credentials\n");
-		return -1;
-	}
+		items[nitems].key = "default_backup_directory";
+		items[nitems].value = default_backup_directory;
+		nitems++;
 
-	if (NULL == (clb_getErrorMessage = dlsym(hdl, "clb_getErrorMessage")) ||
-			NULL == (clb_create = dlsym(hdl, "clb_create")) ||
-			NULL == (clb_open = dlsym(hdl, "clb_open")) ||
-			NULL == (clb_close = dlsym(hdl, "clb_close")) ||
-			NULL == (clb_free = dlsym(hdl, "clb_free")) ||
-			NULL == (clb_setLockboxMode = dlsym(hdl, "clb_setLockboxMode")) ||
-			NULL == (clb_createItemAsText = dlsym(hdl, "clb_createItemAsText")) ||
-			NULL == (clb_retrieveItemAsText = dlsym(hdl, "clb_retrieveItemAsText")))
-	{
-		mpp_err_msg("ERROR", "ddboost", "Failed to load dynamic libraries. Can't set ddboost credentials\n");
-		return -1;
+		items[nitems].key = "ddboost_storage_unit";
+		items[nitems].value = ddboost_storage_unit ? ddboost_storage_unit : DEFAULT_STORAGE_UNIT;
+		nitems++;
 	}
 
-	mpp_err_msg("DEBUG", "ddboost", "Libraries were loaded successfully\n");
-	return 0;
-}
+	items[nitems].key = "log_level";
+	items[nitems].value = log_level ? log_level : "WARNING";
+	nitems++;
 
-static int
-setItem(clbHandle* LB, char *key, char *value)
-{
-	int iError = clb_createItemAsText(*LB, key, value);
-	if (iError != NO_ERR)
-	{
-		char* eMsg= NULL;;
-		clb_getErrorMessage(iError,&eMsg);
-		mpp_err_msg("ERROR", "ddboost", eMsg);
-		clb_free(eMsg);
-		clb_close(*LB);
-		return iError;
-	};
+	items[nitems].key = "log_size";
+	items[nitems].value = log_size ? log_size : "60";
+	nitems++;
 
-	return 0;
-}
+	assert(nitems <= MAX_ITEMS);
+	content.items = items;
+	content.nitems = nitems;
 
-static int
-setItemWithDefault(clbHandle *LB, char *key, char *value, char *defaultValue)
-{
-	return setItem(LB, key, value ?: defaultValue);
-}
-
-static int
-getItem(clbHandle* LB, char *key, char **value)
-{
-	int iError = clb_retrieveItemAsText(*LB, key, value);
-	if (iError != NO_ERR)
-	{
-		char* eMsg= NULL;;
-		clb_getErrorMessage(iError,&eMsg);
-		mpp_err_msg("ERROR", "ddboost", eMsg);
-		clb_free(eMsg);
-		clb_close(*LB);
-		return iError;
-	};
-
-	return 0;
-}
-
-static int
-createLB(clbHandle* LB,char* name)
-{
-	int iError, i;
-	char filepath[PATH_MAX];
-	char *home = getenv("HOME");
-	char* eMsg = NULL;
-	char clb_pass[35] = "1!qQ";
-	int _base64_len = sizeof(_base64)/sizeof(unsigned char) - 2;
-
-	if (NULL == home)
+	/* Store the credentials file to home directory */
+	home = getenv("HOME");
+	if (home == NULL)
 	{
 		mpp_err_msg("ERROR", "ddboost", "HOME undefined, can't set ddboost credentials\n");
 		return -1;
 	}
 
-	memset(filepath, 0, PATH_MAX);
-	snprintf(filepath, strlen(home) + strlen(name) + 2, "%s/%s", home, name);
-
-	if (setLBEnv() < 0)
+	if (remote)
+		filename = "DDBOOST_MFR_CONFIG";
+	else
+		filename = "DDBOOST_CONFIG";
+	if (snprintf(filepath, MAXPGPATH, "%s/%s", home, filename) >= MAXPGPATH)
 	{
+		mpp_err_msg("ERROR", "ddboost", "path \"%s/%s\" is too long\n", home, filename);
 		return -1;
 	}
 
-	/* generate random password to create the lockbox */
-	srand ((unsigned) time(NULL));
-
-	/* choose valid characters from _base64 for the lockbox password */
-	for (i=4; i < 34; i++)
-	{
-		clb_pass[i] = _base64[rand() % _base64_len];
-	}
-	clb_pass[34] = '\0';
-
-	/*
-	 * for creating the lockbox we should call to clb_create.
-	 * this function needs a password with at least 8 characters, with several constraints.
-	 * the password is set to optional few lines later, but we must initialize it during the LB creation.
-	 * of course we don't want to use fixed password, so we're using a random password
-	 */
-	mpp_err_msg("INFO", "ddboost", "creating LB on %s\n", filepath);
-	if ((iError = clb_create(filepath, clb_pass, 1 /*overwrite flag*/, 0 /*reserved*/,LB))!=NO_ERR)
-	{
-		clb_getErrorMessage(iError,&eMsg);
-		mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg);
-		clb_free(eMsg);
-		return iError;
-	};
-
-	if ((iError = clb_setLockboxMode(*LB, CLB_PASSPHRASE_OPTIONAL))!=NO_ERR)
-	{
-		clb_getErrorMessage(iError,&eMsg);
-		mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg);
-		clb_free(eMsg);
-		clb_close(*LB);
-
-		return iError;
-	};
+	if (lb_store(filepath, &content))
+		return -1;	/* lb_store() reported an error to user already */
 
 	return 0;
 }
 
-static int
-openLB(clbHandle* LB,char* name)
+
+int
+getDDBoostCredential(char **hostname, char **user, char **password,
+					 char **log_level, char **log_size,
+					 char **default_backup_directory,
+					 char **ddboost_storage_unit,
+					 bool remote)
 {
-	int iError;
-	char filepath[PATH_MAX];
-	char *home = getenv("HOME");
-	char* eMsg = NULL;
+	char		filepath[MAXPGPATH];
+	char	   *home;
+	char	   *filename;
+	char	   *obfuscated_pw;
+	lockbox_content *content;
+
 
-	if (NULL == home)
+	/* Load the credentials file from home directory */
+	home = getenv("HOME");
+	if (home == NULL)
 	{
 		mpp_err_msg("ERROR", "ddboost", "HOME undefined, can't set ddboost credentials\n");
 		return -1;
 	}
 
-	memset(filepath, 0, PATH_MAX);
-	snprintf(filepath, strlen(home) + strlen(name) + 2, "%s/%s", home, name);
-
-	if (setLBEnv() < 0)
+	if (remote)
+		filename = "DDBOOST_MFR_CONFIG";
+	else
+		filename = "DDBOOST_CONFIG";
+	if (snprintf(filepath, MAXPGPATH, "%s/%s", home, filename) >= MAXPGPATH)
 	{
+		mpp_err_msg("ERROR", "ddboost", "path \"%s/%s\" is too long\n", home, filename);
 		return -1;
 	}
 
 	mpp_err_msg("INFO", "ddboost", "opening LB on %s\n", filepath);
-	if ((iError = clb_open(filepath, NULL, 0 /*reserved*/,LB))!=NO_ERR)
-	{
-		clb_getErrorMessage(iError,&eMsg);
-		mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg);
-		clb_free(eMsg);
-		return iError;
-	};
-	return 0;
-}
 
-/*
- * Write the hostname, user, password, log_level and log_size to the LB
- * Returns 0 in case of success, and -1 otherwise.
- */
-int
-setDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, char *ddboost_storage_unit, bool remote)
-{
-	/* TODO: validate default backup directory name if needed 
-	   TODO: validate storage unit
-	*/
-	if (validateDDBoostCredential(hostname, user, password, log_level , log_size, default_backup_directory, remote))
-		return -1;
-
-	clbHandle LB;
-	if (remote)
-	{
-		if (createLB(&LB, "DDBOOST_MFR_CONFIG"))
-			return -1;
-	}
-	else
-	{
-		if (createLB(&LB, "DDBOOST_CONFIG"))
-			return -1;
-	}
-	if (setItem(&LB , "hostname",hostname))
-		return -1;
-	if (setItem(&LB , "user",user))
-		return -1;
-	if (setItem(&LB , "password",password))
-		return -1;
+	content = lb_load(filepath);
+	if (!content)
+		return -1;	/* lb_load() reported an error already */
 
-	int ret_code = 0;
-	if (!remote)
-	{
-		ret_code |= setItem(&LB , "default_backup_directory",default_backup_directory);
-		ret_code |= setItemWithDefault(&LB, "ddboost_storage_unit", ddboost_storage_unit, DEFAULT_STORAGE_UNIT);
-	}
+	/* Extract the fields we expect the file to contain. */
 
-	ret_code |= setItemWithDefault(&LB, "log_level", log_level, "WARNING");
-	ret_code |= setItemWithDefault(&LB, "log_size", log_size, "50");
+	*hostname = lb_get_item_or_error(content, "hostname", filepath);
+	if (*hostname == NULL)
+		return -1;	/* lb_get_item_or_error() reported an error already */
 
-	clb_close(LB);
+	*user = lb_get_item_or_error(content, "user", filepath);
+	if (*user == NULL)
+		return -1;	/* lb_get_item_or_error() reported an error already */
 
-	return ret_code;
-}
+	obfuscated_pw = lb_get_item_or_error(content, "password", filepath);
+	if (obfuscated_pw == NULL)
+		return -1;	/* lb_get_item_or_error() reported an error already */
 
-int
-getDDBoostCredential(char** hostname, char** user, char** password, char **log_level ,char** log_size, char **default_backup_directory, char **ddboost_storage_unit, bool remote)
-{
-	clbHandle LB;
+	*password = lb_deobfuscate(obfuscated_pw);
+	if (*password == NULL)
+		return -1;	/* lb_deobfuscate() reported an error already */
 
-	if (remote)
-	{
-		if (openLB(&LB,"DDBOOST_MFR_CONFIG"))
-			return -1;
-	}
-	else
-	{
-		if (openLB(&LB,"DDBOOST_CONFIG"))
-			return -1;
-	}
-	if (getItem(&LB , "hostname",hostname))
-		return -1;
-	if (getItem(&LB , "user",user))
-		return -1;
-	if (getItem(&LB , "password",password))
-		return -1;
 	if (!remote)
 	{
-		if (getItem(&LB , "default_backup_directory", default_backup_directory))
-			return -1;
+		*default_backup_directory = lb_get_item_or_error(content, "default_backup_directory", filepath);
+		if (*default_backup_directory == NULL)
+			return -1;	/* lb_get_item_or_error() reported an error already */
 
-		if (getItem(&LB , "ddboost_storage_unit", ddboost_storage_unit))
-			return -1;
+		*ddboost_storage_unit = lb_get_item_or_error(content, "ddboost_storage_unit", filepath);
+		if (*ddboost_storage_unit == NULL)
+			return -1;	/* lb_get_item_or_error() reported an error already */
 	}
-	if (getItem(&LB , "log_level",log_level))
-		return -1;
-	if (getItem(&LB , "log_size",log_size))
-		return -1;
 
-	clb_close(LB);
+	*log_level = lb_get_item_or_error(content, "log_level", filepath);
+	if (*log_level == NULL)
+		return -1;	/* lb_get_item_or_error() reported an error already */
+
+	*log_size = lb_get_item_or_error(content, "log_size", filepath);
+	if (*log_size == NULL)
+		return -1;	/* lb_get_item_or_error() reported an error already */
+
 	return 0;
 }
 
-int
+static int
 validateDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char * default_backup_directory, bool remote)
 {
 	if (!user)

src/bin/pg_dump/cdb/cdb_dump_util.h

@@ -23,6 +23,10 @@
 
 #define DDBOOST_CONFIG_FILE ".ddconfig"
 #define DDBOOST_USERNAME_MAXLENGTH 30
+/*
+ * NOTE: If you increase this, you must also increase OBFUSCATE_PAYLOAD_LENGTH
+ * in cdb_lockbox.c!
+ */
 #define DDBOOST_PASSWORD_MAXLENGTH 40
 #define DDBOOST_LOG_NUM_OF_FILES 10
 #ifndef DDBOOST_POOL_SIZE

src/bin/pg_dump/cdb/cdb_lockbox.c

+/*-------------------------------------------------------------------------
+ *
+ * cdb_lockbox.c
+ *	  Utilities for reading and writing the gpddboost credentials file.
+ *
+ * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ * Portions Copyright (c) 2016, Pivotal Software Inc
+ *
+ * IDENTIFICATION
+ *	  src/bin/pg_dump/cdb/cdb_lockbox.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "cdb_dump_util.h"
+#include "cdb_lockbox.h"
+
+/* This should be at least as large as DDBOOST_PASSWORD_MAXLENGTH! */
+#define OBFUSCATE_PAYLOAD_LENGTH	40
+
+#define OBFUSCATE_MAGIC 1
+
+/*
+ * Format for obfuscated strings. lb_obfuscate returns this in base64 encoded
+ * form.
+ */
+typedef struct
+{
+	/* constant to identify the format. */
+	char		magic;
+
+	/* seed used for the obfuscation */
+	unsigned char seed[sizeof(uint32)];
+
+	unsigned char payload[OBFUSCATE_PAYLOAD_LENGTH];
+}	obfuscate_buf;
+
+/*
+ * A random number generator, used by lb_obfuscate(). The algorithm is taken from
+ * the POSIX spec for rand().
+ */
+static unsigned char
+lb_obfuscate_rand(uint32 *seed) /* RAND_MAX assumed to be 32767. */
+{
+	*seed = (*seed) * 1103515245 + 12345;
+	return ((unsigned char) ((*seed) / 65536) % 32768);
+}
+
+/*
+ * Obfuscate a string.
+ *
+ * The obfuscation is totally reversible, but makes it unreadable to a casual
+ * observer. This is intended to prevent an administrator from accidentally
+ * seeing a password she does not want to see, not as a way to keep the bad
+ * guys out.
+ */
+char *
+lb_obfuscate(const char *input)
+{
+	int			input_len;
+	int			i;
+	uint32		seed;
+	uint32		nseed;
+	obfuscate_buf ob_buf;
+
+	input_len = strlen(input);
+	if (input_len > OBFUSCATE_PAYLOAD_LENGTH)
+	{
+		mpp_err_msg("ERROR", "ddboost", "password is too long, the maximum is %d characters\n",
+					OBFUSCATE_PAYLOAD_LENGTH);
+		return NULL;
+	}
+
+	ob_buf.magic = OBFUSCATE_MAGIC;
+
+	seed = (uint32) time(NULL);
+	nseed = htonl(seed);
+	memcpy(ob_buf.seed, &nseed, sizeof(uint32));
+
+	for (i = 0; i < input_len; i++)
+		ob_buf.payload[i] = ((unsigned char) input[i]) ^ lb_obfuscate_rand(&seed);
+	/* pad with zeros */
+	for (; i < OBFUSCATE_PAYLOAD_LENGTH; i++)
+		ob_buf.payload[i] = 0 ^ lb_obfuscate_rand(&seed);
+
+	return DataToBase64((char *) &ob_buf, sizeof(obfuscate_buf));
+}
+
+/*
+ * Reverse of lb_obfuscate().
+ */
+char *
+lb_deobfuscate(const char *input)
+{
+	unsigned int len;
+	obfuscate_buf ob_buf;
+	char	   *buf;
+	uint32		nseed;
+	uint32		seed;
+	int			i;
+	char		retbuf[OBFUSCATE_PAYLOAD_LENGTH + 1];
+
+	buf = Base64ToData(input, &len);
+	if (!buf)
+		return NULL;
+
+	if (len != sizeof(obfuscate_buf))
+		return NULL;
+
+	memcpy(&ob_buf, buf, sizeof(obfuscate_buf));
+	free(buf);
+
+	if (ob_buf.magic != OBFUSCATE_MAGIC)
+		return NULL;
+
+	memcpy(&nseed, ob_buf.seed, sizeof(uint32));
+	seed = ntohl(nseed);
+
+	for (i = 0; i < OBFUSCATE_PAYLOAD_LENGTH; i++)
+		retbuf[i] = (char) (ob_buf.payload[i] ^ lb_obfuscate_rand(&seed));
+	retbuf[i] = '\0';
+
+	return strdup(retbuf);
+}
+
+/*
+ * Write configuration to given file.
+ *
+ * Returns 0 on success. On error, returns -1 and reports an error message with
+ * mpp_err_msg().
+ */
+int
+lb_store(const char *filepath, const lockbox_content * content)
+{
+	FILE	   *fp;
+	int			i;
+
+	fp = fopen(filepath, "wb");
+	if (fp == NULL)
+	{
+		mpp_err_msg("ERROR", "ddboost", "could not open credentials file \"%s\" for writing: %s\n",
+					filepath, strerror(errno));
+		return -1;
+	}
+
+	/* Write a file header, as a comment. */
+	fprintf(fp, "# gpddboost config file\n");
+
+	/* Write all the key-value pairs. */
+	for (i = 0; i < content->nitems; i++)
+	{
+		lockbox_item *item = &content->items[i];
+		char	   *p;
+
+		fprintf(fp, "%s='", item->key);
+		for (p = item->value; *p != '\0'; p++)
+		{
+			if (*p == '\'')
+				fputc('\'', fp);
+			fputc(*p, fp);
+		}
+		fprintf(fp, "'\n");
+	}
+
+	if (ferror(fp))
+	{
+		mpp_err_msg("ERROR", "ddboost", "error writing credentials file \"%s\"\n",
+					filepath);
+		fclose(fp);
+		return -1;
+	}
+
+	if (fclose(fp) != 0)
+	{
+		mpp_err_msg("ERROR", "ddboost", "error writing credentials file \"%s\"\n",
+					strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * Reads a configuration from given file in user's home directory.
+ *
+ * Returns a lockbox_content object containing all the key-value pairs
+ * from the file. On error, returns NULL, and reports an error message with
+ * mpp_err_msg().
+ */
+lockbox_content *
+lb_load(const char *filepath)
+{
+	FILE	   *fp;
+	lockbox_content *content;
+	char		linebuf[1024];
+	char		keybuf[1024];
+	char		valuebuf[1024];
+	char	   *lp;
+	char	   *kp;
+	char	   *vp;
+
+	content = malloc(sizeof(lockbox_content));
+	if (content == NULL)
+	{
+		mpp_err_msg("ERROR", "ddboost", "out of memory\n");
+		return NULL;
+	}
+	content->items = NULL;
+	content->nitems = 0;
+
+	fp = fopen(filepath, "rb");
+	if (fp == NULL)
+	{
+		mpp_err_msg("ERROR", "ddboost", "could not open credentials file \"%s\" for reading: %s\n",
+					filepath, strerror(errno));
+		return NULL;
+	}
+
+	/* Parse the file, line by line */
+	while (fgets(linebuf, sizeof(linebuf) - 1, fp) != NULL)
+	{
+		lockbox_item *item;
+
+		lp = linebuf;
+		while (*lp == ' ' || *lp == '\t')
+			lp++;				/* ignore leading space */
+
+		if (*lp == '\0' || *lp == '#')
+			continue;			/* ignore empty lines and comments */
+
+		kp = keybuf;
+		for (;;)
+		{
+			if (*lp == '\0')
+			{
+				mpp_err_msg("ERROR", "ddboost", "missing '=' in credentials file\n");
+				goto fail;
+			}
+			else if (*lp == '=')
+				break;
+			*(kp++) = *(lp++);
+		}
+		*kp = '\0';
+
+		/* skip = */
+		lp++;
+		if (*lp != '\'')
+		{
+			mpp_err_msg("ERROR", "ddboost", "missing ' in credentials file\n");
+			goto fail;
+		}
+		lp++;
+
+		/* parse value, e.g. 'foo''bar' */
+		vp = valuebuf;
+		for (;;)
+		{
+			if (*lp == '\0' || *lp == '\n')
+			{
+				mpp_err_msg("ERROR", "ddboost", "unexpected end-of-line in credentials file\n");
+				goto fail;
+			}
+			else if (lp[0] == '\'' && lp[1] == '\'')
+			{
+				/* escaped quote character */
+				*(vp++) = '\'';
+				lp += 2;
+			}
+			else if (lp[0] == '\'')
+			{
+				lp++;
+				break;			/* end of value string */
+			}
+			else
+				*(vp++) = *(lp++);
+		}
+		*vp = '\0';
+
+		while (*lp == ' ' || *lp == '\t' || *lp == '\n')
+			lp++;				/* ignore trailing space */
+
+		if (*lp != '\0')
+		{
+			mpp_err_msg("ERROR", "ddboost", "extra characters at end of line in credentials file: %s\n");
+			goto fail;
+		}
+
+		content->items = realloc(content->items, (content->nitems + 1) * sizeof(lockbox_item));
+		if (content->items == NULL)
+		{
+			mpp_err_msg("ERROR", "ddboost", "out of memory\n");
+			return NULL;
+		}
+
+		item = &content->items[content->nitems++];
+
+		item->key = strdup(keybuf);
+		item->value = strdup(valuebuf);
+		if (item->key == NULL || item->value == NULL)
+		{
+			mpp_err_msg("ERROR", "ddboost", "out of memory\n");
+			return NULL;
+		}
+	}
+
+	if (ferror(fp))
+	{
+		mpp_err_msg("ERROR", "ddboost", "error reading credentials file \"%s\"\n",
+					filepath);
+		fclose(fp);
+		return NULL;
+	}
+
+	if (fclose(fp) != 0)
+	{
+		mpp_err_msg("ERROR", "ddboost", "error closing credentials file \"%s\"\n",
+					strerror(errno));
+		return NULL;
+	}
+
+	return content;
+
+fail:
+	fclose(fp);
+	return NULL;
+}
+
+/* Find item with given key from a lockbox_content object. */
+char *
+lb_get_item(lockbox_content *content, const char *key)
+{
+	int			i;
+
+	for (i = 0; i < content->nitems; i++)
+	{
+		if (strcmp(content->items[i].key, key) == 0)
+			return content->items[i].value;
+	}
+
+	return NULL;				/* not found */
+}
+
+/* Same as lb_get_item(), but reports an error if no item if found */
+char *
+lb_get_item_or_error(lockbox_content *content, const char *key,
+					 const char *filepath)
+{
+	char	   *value;
+
+	value = lb_get_item(content, key);
+	if (!value)
+		mpp_err_msg("ERROR", "ddboost", "'%s' not found in credentials file \"%s\"\n", key, filepath);
+
+	return value;
+}

src/bin/pg_dump/cdb/cdb_lockbox.h

+/*-------------------------------------------------------------------------
+ *
+ * cdb_lockbox.h
+ *	  Definitions for cdb_lockbox.c
+ *
+ * Portions Copyright (c) 2016, Pivotal Software Inc
+ *
+ * src/bin/pg_dump/cdb/cdb_lockbox.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef CDB_LOCKBOX_H
+#define CDB_LOCKBOX_H
+
+typedef struct
+{
+	char	   *key;
+	char	   *value;
+} lockbox_item;
+
+/*
+ * A list of key-value pairs, used to hold the contents of lockbox file
+ * in memory.
+ */
+typedef struct
+{
+	int			nitems;
+	lockbox_item *items;
+} lockbox_content;
+
+extern int lb_store(const char *filepath, const lockbox_content *content);
+extern lockbox_content *lb_load(const char *filepath);
+extern char *lb_get_item(lockbox_content *content, const char *key);
+extern char *lb_get_item_or_error(lockbox_content *content, const char *key,
+					 const char *filepath);
+
+extern char *lb_obfuscate(const char *input);
+extern char *lb_deobfuscate(const char *input);
+
+#endif   /* CDB_LOCKBOX_H */

src/bin/pg_dump/cdb/test/Makefile

@@ -2,9 +2,9 @@ subdir=src/bin/pg_dump/cdb
 top_builddir=../../../../..
 include $(top_builddir)/src/Makefile.global
 
-TARGETS=cdb_dump_util cdb_bsa_util
+TARGETS=cdb_dump_util cdb_bsa_util cdb_lockbox
 
-override CPPFLAGS+= -I$(top_srcdir)/src/interfaces/libpq -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb
+override CPPFLAGS+= -I$(top_srcdir)/src/interfaces/libpq -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include
 
 include $(top_builddir)/src/Makefile.mock
 
@@ -15,7 +15,6 @@ LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses -lcurl -lssl
 
 ifeq ($(enable_ddboost), yes)
 DDBOOSTLIB += -lDDBoost
-override CFLAGS += -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb
 endif
 
 ifeq ($(enable_netbackup), yes)
@@ -27,9 +26,13 @@ endif
 
 
 # The command line here should resemble the one used to build cdb_dump_agent
-cdb_dump_util.t: cdb_dump_util_test.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS)
+cdb_dump_util.t: cdb_dump_util_test.o ../../dumputils.o ../cdb_lockbox.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS)
 	$(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) -o $@
 
 # The command line here should resemble the one used to build libgpbsa
-cdb_bsa_util.t: cdb_bsa_util_test.o ../cdb_dump_util.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS)
+cdb_bsa_util.t: cdb_bsa_util_test.o ../cdb_dump_util.o ../cdb_lockbox.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS)
+	$(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) $(NETBACKUPLIB75) -o $@
+
+# The command line here should resemble the one used to build libgpbsa
+cdb_lockbox.t: cdb_lockbox_test.o ../cdb_dump_util.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS)
 	$(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) $(NETBACKUPLIB75) -o $@

src/bin/pg_dump/cdb/test/cdb_lockbox_test.c

+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include "cmockery.h"
+
+#include "c.h"
+#include "../cdb_lockbox.c"
+
+void
+test__obfuscate(void **state)
+{
+	char		*str;
+	char		*back;
+
+	str = lb_obfuscate("foobar");
+	back = lb_deobfuscate(str);
+	assert_string_equal("foobar", back);
+}
+
+int
+main(int argc, char* argv[])
+{
+	cmockery_parse_arguments(argc, argv);
+
+	const UnitTest tests[] = {
+		unit_test(test__obfuscate),
+	};
+	return run_tests(tests);
+}