diff --git a/gpAux/Makefile b/gpAux/Makefile index d4552d9390edb85f336aa26c56624319ef331b97..e102f255cb4a6aebc996107c5686bc7c48b587da 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -1152,13 +1152,7 @@ endif libcopy=(cd $(1) && $(TAR) cf - $(2)) | (cd $(3) && $(TAR) xvf -)$(check_pipe_for_errors) -.PHONY: copy-rsa-libs copy-nbu-libs - -copy-rsa-libs: - # Copy rsa lockbox libs - supported only on rhel (required for Data -ifeq "$(BLD_ARCH)" "rhel5_x86_64" - cp -fpr $(BLD_THIRDPARTY_LIB_DIR)/rsa_csp $(INSTLOC)/lib/; -endif +.PHONY: copy-nbu-libs copy-nbu-libs: #Copy NetBackup libs @@ -1171,7 +1165,7 @@ ifeq "$(BLD_ARCH)" "rhel5_x86_64" cp -fpr $(BLD_THIRDPARTY_LIB_DIR)/../Netbackup/nbu75 $(INSTLOC)/lib/; endif -copylibs : thirdparty-dist copy-rsa-libs copy-nbu-libs +copylibs : thirdparty-dist copy-nbu-libs if [ `uname -s` = 'SunOS' ]; then cp `ldd $(INSTLOC)/bin/psql | grep gcc | head -1 | awk '{print $$3}'` $(INSTLOC)/lib; fi if [ `uname -s` = 'Darwin' ] ; then \ for lib in `otool -L $(INSTLOC)/bin/psql | egrep 'libssl|libcrypto|libcom_err|libkrb5|curl|readline|numa' | awk '{print $$1}'`; do \ diff --git a/src/bin/pg_dump/cdb/Makefile b/src/bin/pg_dump/cdb/Makefile index 54c09247797af425362b310c2f9cc32ec97db08e..2732352dd5316a5a1fb276f9318af4eb28e77767 100644 --- a/src/bin/pg_dump/cdb/Makefile +++ b/src/bin/pg_dump/cdb/Makefile @@ -25,7 +25,6 @@ LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses -lcurl -lssl ifeq ($(enable_ddboost), yes) DDBOOSTLIB += -lDDBoost -override CFLAGS += -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb endif ifeq ($(enable_netbackup), yes) @@ -51,23 +50,23 @@ KEYWRDOBJS = ../keywords.o ../kwlookup.o all: submake-libpq submake-libpgport cdb_dump cdb_dump_agent cdb_restore cdb_restore_agent gpddboost libgpbsa.so libgpbsa75.so libgpbsa71.so cdb_bsa_dump_agent cdb_bsa_restore_agent cdb_bsa_query_agent cdb_bsa_delete_agent -cdb_dump: cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a $(EXTRA_OBJS) - $(CC) $(CFLAGS) cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ +cdb_dump: cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o cdb_lockbox.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a $(EXTRA_OBJS) + $(CC) $(CFLAGS) cdb_dump.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_backup_archiver.o cdb_dump_util.o cdb_dump_include.o cdb_lockbox.o $(PGDUMP_DIR)/common.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ -cdb_dump_agent: cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_archiver.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ +cdb_dump_agent: cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_dump_agent.o $(PGDUMP_DIR)/common.o $(PGDUMP_DIR)/pg_dump_sort.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_archiver.o cdb_backup_status.o cdb_backup_state.o cdb_dump_include.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ -#cdb_dumpall_agent: cdb_dumpall_agent.o ../common.o cdb_backup_archiver.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_backup_status.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a +#cdb_dumpall_agent: cdb_dumpall_agent.o ../common.o cdb_backup_archiver.o cdb_dump_util.o cdb_lockbox.o cdb_seginst.o cdb_table.o cdb_backup_status.o $(OBJS) $(KEYWRDOBJS) $(libpq_builddir)/libpq.a # $(CC) $(CFLAGS) cdb_dumpall_agent.o cdb_backup_archiver.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(LIBS) -o $@ -cdb_restore: cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(KEYWRDOBJS) $(PGDUMP_DIR)/dumputils.o $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ +cdb_restore: cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_restore.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(KEYWRDOBJS) $(PGDUMP_DIR)/dumputils.o $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ -cdb_restore_agent: cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o $(OBJS) $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ +cdb_restore_agent: cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_lockbox.o $(OBJS) $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_restore_agent.o cdb_backup_archiver.o cdb_backup_status.o cdb_dump_util.o cdb_seginst.o cdb_table.o cdb_lockbox.o $(OBJS) $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ -gpddboost: cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ +gpddboost: cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_ddboost_util.o cdb_backup_status.o cdb_seginst.o cdb_backup_state.o cdb_table.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(LIBS) -o $@ libgpbsa75.so: cdb_bsa_util.c $(libpq_builddir)/libpq.a $(CC) $(CFLAGS_SL) $(CPPFLAGS) cdb_bsa_util.c $(libpq) $(LDFLAGS) -shared $(DDBOOSTLIB) $(NETBACKUPLIB75) $(LIBS) -o $@ @@ -78,17 +77,17 @@ libgpbsa71.so: cdb_bsa_util.o $(libpq_builddir)/libpq.a libgpbsa.so: libgpbsa75.so cp libgpbsa75.so libgpbsa.so -cdb_bsa_dump_agent: cdb_bsa_dump_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_bsa_dump_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ +cdb_bsa_dump_agent: cdb_bsa_dump_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_bsa_dump_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ -cdb_bsa_restore_agent: cdb_bsa_restore_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_bsa_restore_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ +cdb_bsa_restore_agent: cdb_bsa_restore_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_bsa_restore_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ -cdb_bsa_query_agent: cdb_bsa_query_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_bsa_query_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ +cdb_bsa_query_agent: cdb_bsa_query_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_bsa_query_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ -cdb_bsa_delete_agent: cdb_bsa_delete_agent.o cdb_dump_util.o $(libpq_builddir)/libpq.a - $(CC) $(CFLAGS) cdb_bsa_delete_agent.o cdb_dump_util.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ +cdb_bsa_delete_agent: cdb_bsa_delete_agent.o cdb_dump_util.o cdb_lockbox.o $(libpq_builddir)/libpq.a + $(CC) $(CFLAGS) cdb_bsa_delete_agent.o cdb_dump_util.o cdb_lockbox.o $(PGDUMP_DIR)/dumputils.o $(KEYWRDOBJS) $(libpq) $(LDFLAGS) $(DDBOOSTLIB) $(GPBSALIB) $(LIBS) -o $@ .PHONY: submake-backend submake-backend: diff --git a/src/bin/pg_dump/cdb/cdb_dump_util.c b/src/bin/pg_dump/cdb/cdb_dump_util.c index 8892dec77dafd533d6eb9a2f4321c6dd6c81b991..42fad65acc3f6b134a72103d68a958e5202f52c1 100644 --- a/src/bin/pg_dump/cdb/cdb_dump_util.c +++ b/src/bin/pg_dump/cdb/cdb_dump_util.c @@ -22,6 +22,7 @@ #include "pqexpbuffer.h" #include "../dumputils.h" #include "cdb_dump_util.h" +#include "cdb_lockbox.h" #define DDP_CL_DDP 1 #define DEFAULT_STORAGE_UNIT "GPDB" @@ -1212,8 +1213,6 @@ formPostDataFilterCommandLine(char** retVal, const char* post_data_filter_script } #ifdef USE_DDBOOST -#include -#include "clb_base.h" #define NO_ERR 0 @@ -1224,315 +1223,178 @@ struct ddboost_logs unsigned int logsSize ; } ddboost_logs_info; -/* define function pointers in order to use the LB functions with dlsym */ -void (*clb_getErrorMessage)(int errorCode, char **errorMessage) = NULL; -int (*clb_create)(const char* lockboxId, const char* passphrase, int overwrite, clbContext lbCntx, clbHandle *clbH) = NULL; -int (*clb_open)(const char* lockboxId, const char* passphrase, clbContext lbCntx, clbHandle *clbH) = NULL; -int (*clb_close)(clbHandle clbH) = NULL; -void (*clb_free)(void *buffer) = NULL; -int (*clb_setLockboxMode)(clbHandle clbH, clb_mode_e mode) = NULL; -int (*clb_createItemAsText)(clbHandle clbH, const char* itemName, const char* secret) = NULL; -int (*clb_retrieveItemAsText)(clbHandle clbH, const char* itemName, char** secret) = NULL; - - -static int setItem(clbHandle* LB, char *key, char *value); -static int getItem(clbHandle* LB, char *key, char **value); -static int setLBEnv(void); -static int createLB(clbHandle* LB,char* name); -static int openLB(clbHandle* LB,char* name); static int validateDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, bool remote); -int getDDBoostCredential(char** hostname, char** user, char** password, char** log_level ,char** log_size, char **default_backup_directory, char **ddboost_storage_unit, bool remote); /* - * Set the environment variable LD_LIBRARY_PATH in order to dynamically load LB's libraries. + * Write the hostname, user, password, log_level and log_size to the LB * Returns 0 in case of success, and -1 otherwise. */ -static int -setLBEnv(void) +int +setDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, char *ddboost_storage_unit, bool remote) { - void *hdl = NULL; - char *gphome = getenv("GPHOME"); - char *ldpath = getenv("LD_LIBRARY_PATH"); - char LBpath[PATH_MAX]; - char *newldpath = NULL; - char libpath[PATH_MAX]; - char *libdirname = "lib/rsa_csp"; - char *libname = "libCSP-lb.so"; - int newldpath_len = 0; - - if (NULL == gphome) - { - mpp_err_msg("ERROR", "ddboost", "GPHOME undefined, can't set ddboost credentials\n"); - return -1; - } +#define MAX_ITEMS 7 + lockbox_content content; + lockbox_item items[MAX_ITEMS]; + int nitems; + char filepath[MAXPGPATH]; + char *filename; + char *home; + char *obfuscated_pw; - if (NULL == ldpath) - { - mpp_err_msg("ERROR", "ddboost", "LD_LIBRARY_PATH undefined, can't set ddboost credentials\n"); - return -1; - } + /* + * TODO: validate default backup directory name if needed + * TODO: validate storage unit + */ + if (validateDDBoostCredential(hostname, user, password, + log_level, log_size, + default_backup_directory, remote)) + return -1; /* validateDDBoostCredential() reported an error to user already */ - snprintf(LBpath, strlen(gphome) + strlen(libdirname) + 2, "%s/%s", gphome, libdirname); + obfuscated_pw = lb_obfuscate(password); + if (!obfuscated_pw) + return -1; /* lb_obfuscate() reported an error to user already */ - newldpath_len = strlen(ldpath) + strlen(LBpath) + 2; - newldpath = malloc(newldpath_len); + nitems = 0; - if (NULL == newldpath) - { - mpp_err_msg("ERROR", "ddboost", "Memory allocation failed during DDBoost credentials initialization\n"); - return -1; - } + items[nitems].key = "hostname"; + items[nitems].value = hostname; + nitems++; - snprintf(newldpath, newldpath_len, "%s:%s", LBpath, ldpath); - setenv("LD_LIBRARY_PATH", newldpath, 1); - free(newldpath); - newldpath = NULL; + items[nitems].key = "user"; + items[nitems].value = user; + nitems++; - snprintf(libpath, strlen(LBpath) + strlen(libname) + 2, "%s/%s", LBpath, libname); + items[nitems].key = "password"; + items[nitems].value = obfuscated_pw; + nitems++; - if (NULL == (hdl = dlopen(libpath, RTLD_NOW | RTLD_LOCAL))) + if (!remote) { - mpp_err_msg("ERROR", "ddboost", "libCSP-lb.so was not found. Can't set ddboost credentials\n"); - return -1; - } + items[nitems].key = "default_backup_directory"; + items[nitems].value = default_backup_directory; + nitems++; - if (NULL == (clb_getErrorMessage = dlsym(hdl, "clb_getErrorMessage")) || - NULL == (clb_create = dlsym(hdl, "clb_create")) || - NULL == (clb_open = dlsym(hdl, "clb_open")) || - NULL == (clb_close = dlsym(hdl, "clb_close")) || - NULL == (clb_free = dlsym(hdl, "clb_free")) || - NULL == (clb_setLockboxMode = dlsym(hdl, "clb_setLockboxMode")) || - NULL == (clb_createItemAsText = dlsym(hdl, "clb_createItemAsText")) || - NULL == (clb_retrieveItemAsText = dlsym(hdl, "clb_retrieveItemAsText"))) - { - mpp_err_msg("ERROR", "ddboost", "Failed to load dynamic libraries. Can't set ddboost credentials\n"); - return -1; + items[nitems].key = "ddboost_storage_unit"; + items[nitems].value = ddboost_storage_unit ? ddboost_storage_unit : DEFAULT_STORAGE_UNIT; + nitems++; } - mpp_err_msg("DEBUG", "ddboost", "Libraries were loaded successfully\n"); - return 0; -} + items[nitems].key = "log_level"; + items[nitems].value = log_level ? log_level : "WARNING"; + nitems++; -static int -setItem(clbHandle* LB, char *key, char *value) -{ - int iError = clb_createItemAsText(*LB, key, value); - if (iError != NO_ERR) - { - char* eMsg= NULL;; - clb_getErrorMessage(iError,&eMsg); - mpp_err_msg("ERROR", "ddboost", eMsg); - clb_free(eMsg); - clb_close(*LB); - return iError; - }; + items[nitems].key = "log_size"; + items[nitems].value = log_size ? log_size : "60"; + nitems++; - return 0; -} + assert(nitems <= MAX_ITEMS); + content.items = items; + content.nitems = nitems; -static int -setItemWithDefault(clbHandle *LB, char *key, char *value, char *defaultValue) -{ - return setItem(LB, key, value ?: defaultValue); -} - -static int -getItem(clbHandle* LB, char *key, char **value) -{ - int iError = clb_retrieveItemAsText(*LB, key, value); - if (iError != NO_ERR) - { - char* eMsg= NULL;; - clb_getErrorMessage(iError,&eMsg); - mpp_err_msg("ERROR", "ddboost", eMsg); - clb_free(eMsg); - clb_close(*LB); - return iError; - }; - - return 0; -} - -static int -createLB(clbHandle* LB,char* name) -{ - int iError, i; - char filepath[PATH_MAX]; - char *home = getenv("HOME"); - char* eMsg = NULL; - char clb_pass[35] = "1!qQ"; - int _base64_len = sizeof(_base64)/sizeof(unsigned char) - 2; - - if (NULL == home) + /* Store the credentials file to home directory */ + home = getenv("HOME"); + if (home == NULL) { mpp_err_msg("ERROR", "ddboost", "HOME undefined, can't set ddboost credentials\n"); return -1; } - memset(filepath, 0, PATH_MAX); - snprintf(filepath, strlen(home) + strlen(name) + 2, "%s/%s", home, name); - - if (setLBEnv() < 0) + if (remote) + filename = "DDBOOST_MFR_CONFIG"; + else + filename = "DDBOOST_CONFIG"; + if (snprintf(filepath, MAXPGPATH, "%s/%s", home, filename) >= MAXPGPATH) { + mpp_err_msg("ERROR", "ddboost", "path \"%s/%s\" is too long\n", home, filename); return -1; } - /* generate random password to create the lockbox */ - srand ((unsigned) time(NULL)); - - /* choose valid characters from _base64 for the lockbox password */ - for (i=4; i < 34; i++) - { - clb_pass[i] = _base64[rand() % _base64_len]; - } - clb_pass[34] = '\0'; - - /* - * for creating the lockbox we should call to clb_create. - * this function needs a password with at least 8 characters, with several constraints. - * the password is set to optional few lines later, but we must initialize it during the LB creation. - * of course we don't want to use fixed password, so we're using a random password - */ - mpp_err_msg("INFO", "ddboost", "creating LB on %s\n", filepath); - if ((iError = clb_create(filepath, clb_pass, 1 /*overwrite flag*/, 0 /*reserved*/,LB))!=NO_ERR) - { - clb_getErrorMessage(iError,&eMsg); - mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg); - clb_free(eMsg); - return iError; - }; - - if ((iError = clb_setLockboxMode(*LB, CLB_PASSPHRASE_OPTIONAL))!=NO_ERR) - { - clb_getErrorMessage(iError,&eMsg); - mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg); - clb_free(eMsg); - clb_close(*LB); - - return iError; - }; + if (lb_store(filepath, &content)) + return -1; /* lb_store() reported an error to user already */ return 0; } -static int -openLB(clbHandle* LB,char* name) + +int +getDDBoostCredential(char **hostname, char **user, char **password, + char **log_level, char **log_size, + char **default_backup_directory, + char **ddboost_storage_unit, + bool remote) { - int iError; - char filepath[PATH_MAX]; - char *home = getenv("HOME"); - char* eMsg = NULL; + char filepath[MAXPGPATH]; + char *home; + char *filename; + char *obfuscated_pw; + lockbox_content *content; + - if (NULL == home) + /* Load the credentials file from home directory */ + home = getenv("HOME"); + if (home == NULL) { mpp_err_msg("ERROR", "ddboost", "HOME undefined, can't set ddboost credentials\n"); return -1; } - memset(filepath, 0, PATH_MAX); - snprintf(filepath, strlen(home) + strlen(name) + 2, "%s/%s", home, name); - - if (setLBEnv() < 0) + if (remote) + filename = "DDBOOST_MFR_CONFIG"; + else + filename = "DDBOOST_CONFIG"; + if (snprintf(filepath, MAXPGPATH, "%s/%s", home, filename) >= MAXPGPATH) { + mpp_err_msg("ERROR", "ddboost", "path \"%s/%s\" is too long\n", home, filename); return -1; } mpp_err_msg("INFO", "ddboost", "opening LB on %s\n", filepath); - if ((iError = clb_open(filepath, NULL, 0 /*reserved*/,LB))!=NO_ERR) - { - clb_getErrorMessage(iError,&eMsg); - mpp_err_msg("ERROR", "ddboost", "%s\n", eMsg); - clb_free(eMsg); - return iError; - }; - return 0; -} -/* - * Write the hostname, user, password, log_level and log_size to the LB - * Returns 0 in case of success, and -1 otherwise. - */ -int -setDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char *default_backup_directory, char *ddboost_storage_unit, bool remote) -{ - /* TODO: validate default backup directory name if needed - TODO: validate storage unit - */ - if (validateDDBoostCredential(hostname, user, password, log_level , log_size, default_backup_directory, remote)) - return -1; - - clbHandle LB; - if (remote) - { - if (createLB(&LB, "DDBOOST_MFR_CONFIG")) - return -1; - } - else - { - if (createLB(&LB, "DDBOOST_CONFIG")) - return -1; - } - if (setItem(&LB , "hostname",hostname)) - return -1; - if (setItem(&LB , "user",user)) - return -1; - if (setItem(&LB , "password",password)) - return -1; + content = lb_load(filepath); + if (!content) + return -1; /* lb_load() reported an error already */ - int ret_code = 0; - if (!remote) - { - ret_code |= setItem(&LB , "default_backup_directory",default_backup_directory); - ret_code |= setItemWithDefault(&LB, "ddboost_storage_unit", ddboost_storage_unit, DEFAULT_STORAGE_UNIT); - } + /* Extract the fields we expect the file to contain. */ - ret_code |= setItemWithDefault(&LB, "log_level", log_level, "WARNING"); - ret_code |= setItemWithDefault(&LB, "log_size", log_size, "50"); + *hostname = lb_get_item_or_error(content, "hostname", filepath); + if (*hostname == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ - clb_close(LB); + *user = lb_get_item_or_error(content, "user", filepath); + if (*user == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ - return ret_code; -} + obfuscated_pw = lb_get_item_or_error(content, "password", filepath); + if (obfuscated_pw == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ -int -getDDBoostCredential(char** hostname, char** user, char** password, char **log_level ,char** log_size, char **default_backup_directory, char **ddboost_storage_unit, bool remote) -{ - clbHandle LB; + *password = lb_deobfuscate(obfuscated_pw); + if (*password == NULL) + return -1; /* lb_deobfuscate() reported an error already */ - if (remote) - { - if (openLB(&LB,"DDBOOST_MFR_CONFIG")) - return -1; - } - else - { - if (openLB(&LB,"DDBOOST_CONFIG")) - return -1; - } - if (getItem(&LB , "hostname",hostname)) - return -1; - if (getItem(&LB , "user",user)) - return -1; - if (getItem(&LB , "password",password)) - return -1; if (!remote) { - if (getItem(&LB , "default_backup_directory", default_backup_directory)) - return -1; + *default_backup_directory = lb_get_item_or_error(content, "default_backup_directory", filepath); + if (*default_backup_directory == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ - if (getItem(&LB , "ddboost_storage_unit", ddboost_storage_unit)) - return -1; + *ddboost_storage_unit = lb_get_item_or_error(content, "ddboost_storage_unit", filepath); + if (*ddboost_storage_unit == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ } - if (getItem(&LB , "log_level",log_level)) - return -1; - if (getItem(&LB , "log_size",log_size)) - return -1; - clb_close(LB); + *log_level = lb_get_item_or_error(content, "log_level", filepath); + if (*log_level == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ + + *log_size = lb_get_item_or_error(content, "log_size", filepath); + if (*log_size == NULL) + return -1; /* lb_get_item_or_error() reported an error already */ + return 0; } -int +static int validateDDBoostCredential(char *hostname, char *user, char *password, char* log_level ,char* log_size, char * default_backup_directory, bool remote) { if (!user) diff --git a/src/bin/pg_dump/cdb/cdb_dump_util.h b/src/bin/pg_dump/cdb/cdb_dump_util.h index 260e9259c17d1da4340747a85fcabfc038d48269..45a970cfa8d622d12a938195da2be30debacca5a 100644 --- a/src/bin/pg_dump/cdb/cdb_dump_util.h +++ b/src/bin/pg_dump/cdb/cdb_dump_util.h @@ -23,6 +23,10 @@ #define DDBOOST_CONFIG_FILE ".ddconfig" #define DDBOOST_USERNAME_MAXLENGTH 30 +/* + * NOTE: If you increase this, you must also increase OBFUSCATE_PAYLOAD_LENGTH + * in cdb_lockbox.c! + */ #define DDBOOST_PASSWORD_MAXLENGTH 40 #define DDBOOST_LOG_NUM_OF_FILES 10 #ifndef DDBOOST_POOL_SIZE diff --git a/src/bin/pg_dump/cdb/cdb_lockbox.c b/src/bin/pg_dump/cdb/cdb_lockbox.c new file mode 100644 index 0000000000000000000000000000000000000000..5c1178bb64e3e71b5b82a5cdfbcfc38221c35a69 --- /dev/null +++ b/src/bin/pg_dump/cdb/cdb_lockbox.c @@ -0,0 +1,357 @@ +/*------------------------------------------------------------------------- + * + * cdb_lockbox.c + * Utilities for reading and writing the gpddboost credentials file. + * + * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2016, Pivotal Software Inc + * + * IDENTIFICATION + * src/bin/pg_dump/cdb/cdb_lockbox.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include "cdb_dump_util.h" +#include "cdb_lockbox.h" + +/* This should be at least as large as DDBOOST_PASSWORD_MAXLENGTH! */ +#define OBFUSCATE_PAYLOAD_LENGTH 40 + +#define OBFUSCATE_MAGIC 1 + +/* + * Format for obfuscated strings. lb_obfuscate returns this in base64 encoded + * form. + */ +typedef struct +{ + /* constant to identify the format. */ + char magic; + + /* seed used for the obfuscation */ + unsigned char seed[sizeof(uint32)]; + + unsigned char payload[OBFUSCATE_PAYLOAD_LENGTH]; +} obfuscate_buf; + +/* + * A random number generator, used by lb_obfuscate(). The algorithm is taken from + * the POSIX spec for rand(). + */ +static unsigned char +lb_obfuscate_rand(uint32 *seed) /* RAND_MAX assumed to be 32767. */ +{ + *seed = (*seed) * 1103515245 + 12345; + return ((unsigned char) ((*seed) / 65536) % 32768); +} + +/* + * Obfuscate a string. + * + * The obfuscation is totally reversible, but makes it unreadable to a casual + * observer. This is intended to prevent an administrator from accidentally + * seeing a password she does not want to see, not as a way to keep the bad + * guys out. + */ +char * +lb_obfuscate(const char *input) +{ + int input_len; + int i; + uint32 seed; + uint32 nseed; + obfuscate_buf ob_buf; + + input_len = strlen(input); + if (input_len > OBFUSCATE_PAYLOAD_LENGTH) + { + mpp_err_msg("ERROR", "ddboost", "password is too long, the maximum is %d characters\n", + OBFUSCATE_PAYLOAD_LENGTH); + return NULL; + } + + ob_buf.magic = OBFUSCATE_MAGIC; + + seed = (uint32) time(NULL); + nseed = htonl(seed); + memcpy(ob_buf.seed, &nseed, sizeof(uint32)); + + for (i = 0; i < input_len; i++) + ob_buf.payload[i] = ((unsigned char) input[i]) ^ lb_obfuscate_rand(&seed); + /* pad with zeros */ + for (; i < OBFUSCATE_PAYLOAD_LENGTH; i++) + ob_buf.payload[i] = 0 ^ lb_obfuscate_rand(&seed); + + return DataToBase64((char *) &ob_buf, sizeof(obfuscate_buf)); +} + +/* + * Reverse of lb_obfuscate(). + */ +char * +lb_deobfuscate(const char *input) +{ + unsigned int len; + obfuscate_buf ob_buf; + char *buf; + uint32 nseed; + uint32 seed; + int i; + char retbuf[OBFUSCATE_PAYLOAD_LENGTH + 1]; + + buf = Base64ToData(input, &len); + if (!buf) + return NULL; + + if (len != sizeof(obfuscate_buf)) + return NULL; + + memcpy(&ob_buf, buf, sizeof(obfuscate_buf)); + free(buf); + + if (ob_buf.magic != OBFUSCATE_MAGIC) + return NULL; + + memcpy(&nseed, ob_buf.seed, sizeof(uint32)); + seed = ntohl(nseed); + + for (i = 0; i < OBFUSCATE_PAYLOAD_LENGTH; i++) + retbuf[i] = (char) (ob_buf.payload[i] ^ lb_obfuscate_rand(&seed)); + retbuf[i] = '\0'; + + return strdup(retbuf); +} + +/* + * Write configuration to given file. + * + * Returns 0 on success. On error, returns -1 and reports an error message with + * mpp_err_msg(). + */ +int +lb_store(const char *filepath, const lockbox_content * content) +{ + FILE *fp; + int i; + + fp = fopen(filepath, "wb"); + if (fp == NULL) + { + mpp_err_msg("ERROR", "ddboost", "could not open credentials file \"%s\" for writing: %s\n", + filepath, strerror(errno)); + return -1; + } + + /* Write a file header, as a comment. */ + fprintf(fp, "# gpddboost config file\n"); + + /* Write all the key-value pairs. */ + for (i = 0; i < content->nitems; i++) + { + lockbox_item *item = &content->items[i]; + char *p; + + fprintf(fp, "%s='", item->key); + for (p = item->value; *p != '\0'; p++) + { + if (*p == '\'') + fputc('\'', fp); + fputc(*p, fp); + } + fprintf(fp, "'\n"); + } + + if (ferror(fp)) + { + mpp_err_msg("ERROR", "ddboost", "error writing credentials file \"%s\"\n", + filepath); + fclose(fp); + return -1; + } + + if (fclose(fp) != 0) + { + mpp_err_msg("ERROR", "ddboost", "error writing credentials file \"%s\"\n", + strerror(errno)); + return -1; + } + + return 0; +} + +/* + * Reads a configuration from given file in user's home directory. + * + * Returns a lockbox_content object containing all the key-value pairs + * from the file. On error, returns NULL, and reports an error message with + * mpp_err_msg(). + */ +lockbox_content * +lb_load(const char *filepath) +{ + FILE *fp; + lockbox_content *content; + char linebuf[1024]; + char keybuf[1024]; + char valuebuf[1024]; + char *lp; + char *kp; + char *vp; + + content = malloc(sizeof(lockbox_content)); + if (content == NULL) + { + mpp_err_msg("ERROR", "ddboost", "out of memory\n"); + return NULL; + } + content->items = NULL; + content->nitems = 0; + + fp = fopen(filepath, "rb"); + if (fp == NULL) + { + mpp_err_msg("ERROR", "ddboost", "could not open credentials file \"%s\" for reading: %s\n", + filepath, strerror(errno)); + return NULL; + } + + /* Parse the file, line by line */ + while (fgets(linebuf, sizeof(linebuf) - 1, fp) != NULL) + { + lockbox_item *item; + + lp = linebuf; + while (*lp == ' ' || *lp == '\t') + lp++; /* ignore leading space */ + + if (*lp == '\0' || *lp == '#') + continue; /* ignore empty lines and comments */ + + kp = keybuf; + for (;;) + { + if (*lp == '\0') + { + mpp_err_msg("ERROR", "ddboost", "missing '=' in credentials file\n"); + goto fail; + } + else if (*lp == '=') + break; + *(kp++) = *(lp++); + } + *kp = '\0'; + + /* skip = */ + lp++; + if (*lp != '\'') + { + mpp_err_msg("ERROR", "ddboost", "missing ' in credentials file\n"); + goto fail; + } + lp++; + + /* parse value, e.g. 'foo''bar' */ + vp = valuebuf; + for (;;) + { + if (*lp == '\0' || *lp == '\n') + { + mpp_err_msg("ERROR", "ddboost", "unexpected end-of-line in credentials file\n"); + goto fail; + } + else if (lp[0] == '\'' && lp[1] == '\'') + { + /* escaped quote character */ + *(vp++) = '\''; + lp += 2; + } + else if (lp[0] == '\'') + { + lp++; + break; /* end of value string */ + } + else + *(vp++) = *(lp++); + } + *vp = '\0'; + + while (*lp == ' ' || *lp == '\t' || *lp == '\n') + lp++; /* ignore trailing space */ + + if (*lp != '\0') + { + mpp_err_msg("ERROR", "ddboost", "extra characters at end of line in credentials file: %s\n"); + goto fail; + } + + content->items = realloc(content->items, (content->nitems + 1) * sizeof(lockbox_item)); + if (content->items == NULL) + { + mpp_err_msg("ERROR", "ddboost", "out of memory\n"); + return NULL; + } + + item = &content->items[content->nitems++]; + + item->key = strdup(keybuf); + item->value = strdup(valuebuf); + if (item->key == NULL || item->value == NULL) + { + mpp_err_msg("ERROR", "ddboost", "out of memory\n"); + return NULL; + } + } + + if (ferror(fp)) + { + mpp_err_msg("ERROR", "ddboost", "error reading credentials file \"%s\"\n", + filepath); + fclose(fp); + return NULL; + } + + if (fclose(fp) != 0) + { + mpp_err_msg("ERROR", "ddboost", "error closing credentials file \"%s\"\n", + strerror(errno)); + return NULL; + } + + return content; + +fail: + fclose(fp); + return NULL; +} + +/* Find item with given key from a lockbox_content object. */ +char * +lb_get_item(lockbox_content *content, const char *key) +{ + int i; + + for (i = 0; i < content->nitems; i++) + { + if (strcmp(content->items[i].key, key) == 0) + return content->items[i].value; + } + + return NULL; /* not found */ +} + +/* Same as lb_get_item(), but reports an error if no item if found */ +char * +lb_get_item_or_error(lockbox_content *content, const char *key, + const char *filepath) +{ + char *value; + + value = lb_get_item(content, key); + if (!value) + mpp_err_msg("ERROR", "ddboost", "'%s' not found in credentials file \"%s\"\n", key, filepath); + + return value; +} diff --git a/src/bin/pg_dump/cdb/cdb_lockbox.h b/src/bin/pg_dump/cdb/cdb_lockbox.h new file mode 100644 index 0000000000000000000000000000000000000000..0b92a2e280bae2af1349e6024b8a2abadb5f1a6e --- /dev/null +++ b/src/bin/pg_dump/cdb/cdb_lockbox.h @@ -0,0 +1,41 @@ +/*------------------------------------------------------------------------- + * + * cdb_lockbox.h + * Definitions for cdb_lockbox.c + * + * Portions Copyright (c) 2016, Pivotal Software Inc + * + * src/bin/pg_dump/cdb/cdb_lockbox.h + * + *------------------------------------------------------------------------- + */ + +#ifndef CDB_LOCKBOX_H +#define CDB_LOCKBOX_H + +typedef struct +{ + char *key; + char *value; +} lockbox_item; + +/* + * A list of key-value pairs, used to hold the contents of lockbox file + * in memory. + */ +typedef struct +{ + int nitems; + lockbox_item *items; +} lockbox_content; + +extern int lb_store(const char *filepath, const lockbox_content *content); +extern lockbox_content *lb_load(const char *filepath); +extern char *lb_get_item(lockbox_content *content, const char *key); +extern char *lb_get_item_or_error(lockbox_content *content, const char *key, + const char *filepath); + +extern char *lb_obfuscate(const char *input); +extern char *lb_deobfuscate(const char *input); + +#endif /* CDB_LOCKBOX_H */ diff --git a/src/bin/pg_dump/cdb/test/Makefile b/src/bin/pg_dump/cdb/test/Makefile index 499d417a53e2dee46be17d1445ebb55e3f859437..2557e5166cec70101568df139125f4862700171c 100644 --- a/src/bin/pg_dump/cdb/test/Makefile +++ b/src/bin/pg_dump/cdb/test/Makefile @@ -2,9 +2,9 @@ subdir=src/bin/pg_dump/cdb top_builddir=../../../../.. include $(top_builddir)/src/Makefile.global -TARGETS=cdb_dump_util cdb_bsa_util +TARGETS=cdb_dump_util cdb_bsa_util cdb_lockbox -override CPPFLAGS+= -I$(top_srcdir)/src/interfaces/libpq -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb +override CPPFLAGS+= -I$(top_srcdir)/src/interfaces/libpq -I$(top_srcdir)/gpAux/ext/$(BLD_ARCH)/include include $(top_builddir)/src/Makefile.mock @@ -15,7 +15,6 @@ LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses -lcurl -lssl ifeq ($(enable_ddboost), yes) DDBOOSTLIB += -lDDBoost -override CFLAGS += -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp -I$(top_builddir)/gpAux/ext/$(BLD_ARCH)/include/rsa_csp/lb endif ifeq ($(enable_netbackup), yes) @@ -27,9 +26,13 @@ endif # The command line here should resemble the one used to build cdb_dump_agent -cdb_dump_util.t: cdb_dump_util_test.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS) +cdb_dump_util.t: cdb_dump_util_test.o ../../dumputils.o ../cdb_lockbox.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS) $(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) -o $@ # The command line here should resemble the one used to build libgpbsa -cdb_bsa_util.t: cdb_bsa_util_test.o ../cdb_dump_util.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS) +cdb_bsa_util.t: cdb_bsa_util_test.o ../cdb_dump_util.o ../cdb_lockbox.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS) + $(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) $(NETBACKUPLIB75) -o $@ + +# The command line here should resemble the one used to build libgpbsa +cdb_lockbox.t: cdb_lockbox_test.o ../cdb_dump_util.o ../../dumputils.o ../../keywords.o ../../kwlookup.o $(CMOCKERY_OBJS) $(CC) $^ $(libpq) $(LDFLAGS) $(LIBS) $(DDBOOSTLIB) $(NETBACKUPLIB75) -o $@ diff --git a/src/bin/pg_dump/cdb/test/cdb_lockbox_test.c b/src/bin/pg_dump/cdb/test/cdb_lockbox_test.c new file mode 100644 index 0000000000000000000000000000000000000000..a07f2a8654cd23f39ae0cd01c5e92f874744e964 --- /dev/null +++ b/src/bin/pg_dump/cdb/test/cdb_lockbox_test.c @@ -0,0 +1,29 @@ +#include +#include +#include +#include "cmockery.h" + +#include "c.h" +#include "../cdb_lockbox.c" + +void +test__obfuscate(void **state) +{ + char *str; + char *back; + + str = lb_obfuscate("foobar"); + back = lb_deobfuscate(str); + assert_string_equal("foobar", back); +} + +int +main(int argc, char* argv[]) +{ + cmockery_parse_arguments(argc, argv); + + const UnitTest tests[] = { + unit_test(test__obfuscate), + }; + return run_tests(tests); +}