diff --git a/concourse/pipelines/gpdb_master-generated.yml b/concourse/pipelines/gpdb_master-generated.yml index 9f6567fe24916c912e0ae9f7a95239e28f727022..d075fe74a45b2fb4055618b9b4ef076ee4db9c48 100644 --- a/concourse/pipelines/gpdb_master-generated.yml +++ b/concourse/pipelines/gpdb_master-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2018-11-19 13:46:58.098114 +## Generated by gen_pipeline.py at: 2018-11-29 09:21:43.022135 ## Template file: gpdb-tpl.yml ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16'] ## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions'] @@ -300,15 +300,6 @@ resources: region_name: {{aws-region}} bucket: gpdb5-pipeline-dynamic-terraform bucket_path: clusters-google/ - -- name: snowflake-simple-database-dump # remove this resource in favor of icw_gporca_centos6_dump once gpexpand works with a dump of ICW - type: s3 - source: - access_key_id: {{bucket-access-key-id}} - bucket: {{test-data-bucket-name}} - secret_access_key: {{bucket-secret-access-key}} - region_name: {{aws-region}} - versioned_file: snowflake-simple-database/dump.sql.xz - name: aix_environments type: pool source: @@ -1832,7 +1823,6 @@ jobs: - get: centos-gpdb-dev-6 - get: icw_gporca_centos6_dump passed: [gate_icw_end] - - get: snowflake-simple-database-dump # remove once icw_gporca_centos6_dump works with gpexpand - put: terraform params: <<: *ccp_default_params @@ -1846,7 +1836,7 @@ jobs: - task: pre_run_test_setup file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml image: centos-gpdb-dev-6 - input_mapping: {sqldump: snowflake-simple-database-dump} # use icw_gporca_centos6_dump when it works with gpexpand + input_mapping: {sqldump: icw_gporca_centos6_dump} - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml params: @@ -1871,7 +1861,7 @@ jobs: trigger: true - get: ccp_src - get: centos-gpdb-dev-6 - - get: snowflake-simple-database-dump # remove once icw_gporca_centos6_dump works with gpexpand + - get: icw_gporca_centos6_dump - put: terraform params: <<: *ccp_default_params @@ -1887,7 +1877,7 @@ jobs: - task: pre_run_test_setup image: centos-gpdb-dev-6 file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml - input_mapping: {sqldump: snowflake-simple-database-dump} # use icw_gporca_centos6_dump when it works with gpexpand + input_mapping: {sqldump: icw_gporca_centos6_dump} - task: run_tests file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 @@ -1912,7 +1902,7 @@ jobs: trigger: true - get: ccp_src - get: centos-gpdb-dev-6 - - get: snowflake-simple-database-dump # remove once icw_gporca_centos6_dump works with gpexpand + - get: icw_gporca_centos6_dump - put: terraform params: <<: *ccp_default_params @@ -1928,7 +1918,7 @@ jobs: - task: pre_run_test_setup image: centos-gpdb-dev-6 file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml - input_mapping: {sqldump: snowflake-simple-database-dump} # use icw_gporca_centos6_dump when it works with gpexpand + input_mapping: {sqldump: icw_gporca_centos6_dump} - task: run_tests file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index b91469aaf7165353c6786a0a91d92063d4b032aa..cf5e49dbda99b87de090acfc3deaaae581ea98c3 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -407,15 +407,6 @@ resources: region_name: {{aws-region}} bucket: gpdb5-pipeline-dynamic-terraform bucket_path: clusters-google/ - -- name: snowflake-simple-database-dump # remove this resource in favor of icw_gporca_centos6_dump once gpexpand works with a dump of ICW - type: s3 - source: - access_key_id: {{bucket-access-key-id}} - bucket: {{test-data-bucket-name}} - secret_access_key: {{bucket-secret-access-key}} - region_name: {{aws-region}} - versioned_file: snowflake-simple-database/dump.sql.xz {% endif %} {% if "aix7" in os_types %} - name: aix_environments @@ -1924,7 +1915,6 @@ jobs: - get: centos-gpdb-dev-6 - get: icw_gporca_centos6_dump passed: [gate_icw_end] - - get: snowflake-simple-database-dump # remove once icw_gporca_centos6_dump works with gpexpand - put: terraform params: <<: *ccp_default_params @@ -1938,7 +1928,7 @@ jobs: - task: pre_run_test_setup file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml image: centos-gpdb-dev-6 - input_mapping: {sqldump: snowflake-simple-database-dump} # use icw_gporca_centos6_dump when it works with gpexpand + input_mapping: {sqldump: icw_gporca_centos6_dump} - task: run_tests file: gpdb_src/concourse/tasks/run_behave.yml params: @@ -1964,7 +1954,7 @@ jobs: trigger: [[ test_trigger ]] - get: ccp_src - get: centos-gpdb-dev-6 - - get: snowflake-simple-database-dump # remove once icw_gporca_centos6_dump works with gpexpand + - get: icw_gporca_centos6_dump - put: terraform params: <<: *ccp_default_params @@ -1980,7 +1970,7 @@ jobs: - task: pre_run_test_setup image: centos-gpdb-dev-6 file: gpdb_src/concourse/tasks/setup_for_gpexpand_to_make_new_gpdb.yml - input_mapping: {sqldump: snowflake-simple-database-dump} # use icw_gporca_centos6_dump when it works with gpexpand + input_mapping: {sqldump: icw_gporca_centos6_dump} - task: run_tests file: gpdb_src/concourse/tasks/run_tinc.yml image: centos-gpdb-dev-6 diff --git a/gpMgmt/bin/gpexpand b/gpMgmt/bin/gpexpand index 8dc72f086f2e8782d09a5686b69198d0f9e7f29b..be5e8fc237e0504626bcf4dafd0720a828543585 100755 --- a/gpMgmt/bin/gpexpand +++ b/gpMgmt/bin/gpexpand @@ -2779,9 +2779,9 @@ def main(options, args, parser): if not _gp_expand.validate_unalterable_tables(): raise ValidationError() if _gp_expand.check_unique_indexes(): - logger.info("Tables with unique indexes exist. Until these tables are successfully") - logger.info("redistributed, unique constraints may be violated. For more information") - logger.info("on this issue, see the Greenplum Database Administrator Guide") + logger.warn("Tables with unique indexes exist. Until these tables are successfully") + logger.warn("redistributed, unique constraints may be violated. For more information") + logger.warn("on this issue, see the Greenplum Database Administrator Guide") if not options.silent: if not ask_yesno(None, "Would you like to continue with System Expansion", 'N'): raise ValidationError() diff --git a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature index 40683bf00f6280b742840395652638c4902f49d8..92f55aa354fc78c27d810dc7c85f773626a57700 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpexpand.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpexpand.feature @@ -5,9 +5,9 @@ Feature: expand the cluster by adding more segments @gpexpand_ranks @gpexpand_timing Scenario: after resuming a duration interrupted redistribution, tables were restored in the user defined order - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -18,7 +18,7 @@ Feature: expand the cluster by adding more segments Then the number of segments have been saved And user has created expansionranktest tables And 4000000 rows are inserted into table "expansionranktest8" in schema "public" with column type list "int" - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then user has fixed the expansion order for tables When the user runs gpexpand against database "gptest" to redistribute with duration "00:00:02" Then gpexpand should print "End time reached. Stopping expansion." to stdout @@ -32,9 +32,9 @@ Feature: expand the cluster by adding more segments @gpexpand_timing @gpexpand_standby Scenario: after a duration interrupted redistribution, state file on standby matches master - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the user runs gpinitstandby with options " " @@ -44,7 +44,7 @@ Feature: expand the cluster by adding more segments When the user runs gpexpand interview to add 2 new segment and 0 new host "ignored.host" Then user has created expansionranktest tables And 4000000 rows are inserted into table "expansionranktest8" in schema "public" with column type list "int" - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" When the user runs gpexpand against database "gptest" to redistribute with duration "00:00:02" Then gpexpand should print "End time reached. Stopping expansion." to stdout @@ -53,9 +53,9 @@ Feature: expand the cluster by adding more segments @gpexpand_ranks @gpexpand_timing Scenario: after resuming an end time interrupted redistribution, tables were restored in the user defined order - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -65,7 +65,7 @@ Feature: expand the cluster by adding more segments Then the number of segments have been saved And user has created expansionranktest tables And 4000000 rows are inserted into table "expansionranktest8" in schema "public" with column type list "int" - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then user has fixed the expansion order for tables When the user runs gpexpand against database "gptest" to redistribute with the --end flag Then gpexpand should print "End time reached. Stopping expansion." to stdout @@ -76,9 +76,9 @@ Feature: expand the cluster by adding more segments @gpexpand_no_mirrors @gpexpand_segment Scenario: expand a cluster that has no mirrors - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -86,15 +86,15 @@ Feature: expand the cluster by adding more segments And the cluster is setup for an expansion on hosts "mdw,sdw1" When the user runs gpexpand interview to add 2 new segment and 0 new host "ignored.host" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 2 new segments @gpexpand_no_mirrors @gpexpand_host Scenario: expand a cluster that has no mirrors with one new hosts - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1,sdw2" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -103,15 +103,15 @@ Feature: expand the cluster by adding more segments And the new host "sdw2" is ready to go When the user runs gpexpand interview to add 0 new segment and 1 new host "sdw2" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 2 new segments @gpexpand_no_mirrors @gpexpand_host_and_segment Scenario: expand a cluster that has no mirrors with one new hosts - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1,sdw2" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -120,15 +120,15 @@ Feature: expand the cluster by adding more segments And the new host "sdw2" is ready to go When the user runs gpexpand interview to add 1 new segment and 1 new host "sdw2" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 4 new segments @gpexpand_mirrors @gpexpand_segment Scenario: expand a cluster that has mirrors - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -141,9 +141,9 @@ Feature: expand the cluster by adding more segments @gpexpand_mirrors @gpexpand_host Scenario: expand a cluster that has mirrors with one new hosts - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1,sdw2,sdw3" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with mirrors on "mdw" and "sdw1" And database "gptest" exists @@ -152,16 +152,16 @@ Feature: expand the cluster by adding more segments And the new host "sdw2,sdw3" is ready to go When the user runs gpexpand interview to add 0 new segment and 2 new host "sdw2,sdw3" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 8 new segments @gpexpand_mirrors @gpexpand_host_and_segment @gpexpand_standby - Scenario: expand a cluster that has mirrors with one new hosts - Given a working directory of the test as '/tmp/gpexpand_behave' + Scenario: expand a cluster that has mirrors with one new host + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1,sdw2,sdw3" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with mirrors on "mdw" and "sdw1" And the user runs gpinitstandby with options " " @@ -171,85 +171,54 @@ Feature: expand the cluster by adding more segments And the new host "sdw2,sdw3" is ready to go When the user runs gpexpand interview to add 1 new segment and 2 new host "sdw2,sdw3" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 14 new segments - @gpexpand_mirrors - @gpexpand_host_and_segment - @gpexpand_standby - Scenario: expand a cluster that has mirrors and verify redistribution - Given a working directory of the test as '/tmp/gpexpand_behave' - And the database is killed on hosts "mdw,sdw1,sdw2,sdw3" - And a temporary directory to expand into - And the database is not running - And a cluster is created with mirrors on "mdw" and "sdw1" - And the user runs gpinitstandby with options " " - And database "gptest" exists - And the user runs psql with "-c 'CREATE TABLE public.redistribute (i int) DISTRIBUTED BY (i)'" against database "gptest" - And the user runs psql with "-c 'INSERT INTO public.redistribute SELECT generate_series(1, 10000)'" against database "gptest" - And distribution information from table "public.redistribute" with data in "gptest" is saved - And there are no gpexpand_inputfiles - And the cluster is setup for an expansion on hosts "mdw,sdw1,sdw2,sdw3" - And the new host "sdw2,sdw3" is ready to go - When the user runs gpexpand interview to add 1 new segment and 2 new host "sdw2,sdw3" - Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile - Then verify that the cluster has 14 new segments - When the user runs gpexpand against database "gptest" to redistribute - # Temporarily comment the verifys until redistribute is fixed. This allows us to commit a resource to get a dump of the ICW dump for other tests to use - # Then distribution information from table "public.redistribute" with data in "gptest" is verified against saved data - - @gpexpand_local - Scenario: Use a hefty database for expansion - Given a working directory of the test as '/tmp/gpexpand_behave' + @gpexpand_verify_redistribution + Scenario: Verify data is correctly redistributed after expansion + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "localhost" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And the cluster is generated with "1" primaries only And database "gptest" exists And the user runs psql with "-c 'CREATE TABLE public.redistribute (i int) DISTRIBUTED BY (i)'" against database "gptest" And the user runs psql with "-c 'INSERT INTO public.redistribute SELECT generate_series(1, 10000)'" against database "gptest" And distribution information from table "public.redistribute" with data in "gptest" is saved - And the user runs psql with "-f test/behave/mgmt_utils/steps/data/test_dump.sql" against database "gptest" And there are no gpexpand_inputfiles And the cluster is setup for an expansion on hosts "localhost" When the user runs gpexpand interview to add 3 new segment and 0 new host "ignored.host" Then the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then verify that the cluster has 3 new segments When the user runs gpexpand against database "gptest" to redistribute - # Temporarily comment the verifys until redistribute is fixed. This allows us to commit a resource to get a dump of the ICW dump for other tests to use - # Then distribution information from table "public.redistribute" with data in "gptest" is verified against saved data + Then distribution information from table "public.redistribute" with data in "gptest" is verified against saved data @gpexpand_icw_db_concourse - Scenario: Use a hefty database for expansion - Given a working directory of the test as '/tmp/gpexpand_behave' + Scenario: Use a dump of the ICW database for expansion + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1,sdw2,sdw3" - And a temporary directory to expand into + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with mirrors on "mdw" and "sdw1" And database "gptest" exists - And the user runs psql with "-c 'CREATE TABLE public.redistribute (i int) DISTRIBUTED BY (i)'" against database "gptest" - And the user runs psql with "-c 'INSERT INTO public.redistribute SELECT generate_series(1, 10000)'" against database "gptest" - And distribution information from table "public.redistribute" with data in "gptest" is saved - # currently using snowflake-simple-database-dump which should be replaced with icw_gporca_centos6_dump once gpexpand works with a dump of ICW And the user runs psql with "-f /home/gpadmin/sqldump/dump.sql" against database "gptest" And there are no gpexpand_inputfiles And the cluster is setup for an expansion on hosts "mdw,sdw1,sdw2,sdw3" And the new host "sdw2,sdw3" is ready to go And the user runs gpexpand interview to add 1 new segment and 2 new host "sdw2,sdw3" And the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" And verify that the cluster has 14 new segments @gpexpand_no_mirrors @gpexpand_no_restart @gpexpand_catalog_copied - Scenario: expand a cluster without restarting db and catalog has been copie - Given a working directory of the test as '/tmp/gpexpand_behave' + Scenario: expand a cluster without restarting db and catalog has been copied + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And the user runs command "rm -rf /tmp/gpexpand_behave/*" - And a temporary directory to expand into + And the user runs command "rm -rf /data/gpdata/gpexpand/*" + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -259,7 +228,7 @@ Feature: expand the cluster by adding more segments And the user runs gpexpand interview to add 2 new segment and 0 new host "ignored.host" And the number of segments have been saved And user has created test table - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then gpexpand should return a return code of 0 And verify that the cluster has 2 new segments And all the segments are running @@ -271,10 +240,10 @@ Feature: expand the cluster by adding more segments @gpexpand_no_restart @gpexpand_conf_copied Scenario: expand a cluster without restarting db and conf has been copie - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And the user runs command "rm -rf /tmp/gpexpand_behave/*" - And a temporary directory to expand into + And the user runs command "rm -rf /data/gpdata/gpexpand/*" + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -283,7 +252,7 @@ Feature: expand the cluster by adding more segments And the cluster is setup for an expansion on hosts "mdw" And the user runs gpexpand interview to add 1 new segment and 0 new host "ignore.host" And the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then gpexpand should return a return code of 0 And verify that the cluster has 1 new segments And all the segments are running @@ -294,10 +263,10 @@ Feature: expand the cluster by adding more segments @gpexpand_no_restart @gpexpand_long_run_read_only Scenario: expand a cluster without restarting db with long-run read-only transaction - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And the user runs command "rm -rf /tmp/gpexpand_behave/*" - And a temporary directory to expand into + And the user runs command "rm -rf /data/gpdata/gpexpand/*" + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -309,7 +278,7 @@ Feature: expand the cluster by adding more segments And the cluster is setup for an expansion on hosts "mdw" And the user runs gpexpand interview to add 1 new segment and 0 new host "ignore.host" And the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then gpexpand should return a return code of 0 And verify that the cluster has 1 new segments And all the segments are running @@ -320,10 +289,10 @@ Feature: expand the cluster by adding more segments @gpexpand_no_restart @gpexpand_change_catalog_abort Scenario: expand a cluster without restarting db and any transaction which wants to change the catalog must be aborted - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And the user runs command "rm -rf /tmp/gpexpand_behave/*" - And a temporary directory to expand into + And the user runs command "rm -rf /data/gpdata/gpexpand/*" + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -333,7 +302,7 @@ Feature: expand the cluster by adding more segments And the cluster is setup for an expansion on hosts "mdw" And the user runs gpexpand interview to add 1 new segment and 0 new host "ignore.host" And the number of segments have been saved - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then gpexpand should return a return code of 0 And verify that the cluster has 1 new segments And all the segments are running @@ -344,10 +313,10 @@ Feature: expand the cluster by adding more segments @gpexpand_no_restart @gpexpand_dml Scenario: expand a cluster without restarting db and any transaction which wants to do dml work well - Given a working directory of the test as '/tmp/gpexpand_behave' + Given a working directory of the test as '/data/gpdata/gpexpand' And the database is killed on hosts "mdw,sdw1" - And the user runs command "rm -rf /tmp/gpexpand_behave/*" - And a temporary directory to expand into + And the user runs command "rm -rf /data/gpdata/gpexpand/*" + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into And the database is not running And a cluster is created with no mirrors on "mdw" and "sdw1" And the master pid has been saved @@ -357,7 +326,7 @@ Feature: expand the cluster by adding more segments And the user runs gpexpand interview to add 1 new segment and 0 new host "ignore.host" And the number of segments have been saved And the transactions are started for dml - When the user runs gpexpand with the latest gpexpand_inputfile + When the user runs gpexpand with the latest gpexpand_inputfile with additional parameters "--silent" Then gpexpand should return a return code of 0 And verify that the cluster has 1 new segments And all the segments are running diff --git a/gpMgmt/test/behave/mgmt_utils/steps/data/test_dump.sql b/gpMgmt/test/behave/mgmt_utils/steps/data/test_dump.sql deleted file mode 100644 index 75fd592dc9b9f778c2a165d726d00b6e64e2eaa5..0000000000000000000000000000000000000000 --- a/gpMgmt/test/behave/mgmt_utils/steps/data/test_dump.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Replace this with a dump of ICW once it works with gpexpand - -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = off; -SET escape_string_warning = 'off'; - - - -CREATE TABLE test ( - name text, - age int -); - -INSERT INTO test (name, age) - VALUES ('a', 1), ('b', 2); diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 608ef9de3b101b1b309668615a9415ed81b6d5b7..c8500d1b6a8e5894996e9aa4f2e5d17b9f4d5837 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -2089,11 +2089,15 @@ def impl(context, num_of_segments, num_of_hosts, hostnames): if num_of_hosts != len(hosts): raise Exception("Incorrect amount of hosts. number of hosts:%s\nhostnames: %s" % (num_of_hosts, hosts)) - temp_base_dir = context.temp_base_dir - primary_dir = os.path.join(temp_base_dir, 'data', 'primary') + base_dir = "/tmp" + if hasattr(context, "temp_base_dir"): + base_dir = context.temp_base_dir + elif hasattr(context, "working_directory"): + base_dir = context.working_directory + primary_dir = os.path.join(base_dir, 'data', 'primary') mirror_dir = '' if context.gpexpand_mirrors_enabled: - mirror_dir = os.path.join(temp_base_dir, 'data', 'mirror') + mirror_dir = os.path.join(base_dir, 'data', 'mirror') directory_pairs = [] # we need to create the tuples for the interview to work. @@ -2112,10 +2116,10 @@ def impl(context, num_of_segments, num_of_hosts, hostnames): def impl(context): map(os.remove, glob.glob("gpexpand_inputfile*")) -@when('the user runs gpexpand with the latest gpexpand_inputfile') -def impl(context): +@when('the user runs gpexpand with the latest gpexpand_inputfile with additional parameters {additional_params}') +def impl(context, additional_params=''): gpexpand = Gpexpand(context, working_directory=context.working_directory, database='gptest') - ret_code, std_err, std_out = gpexpand.initialize_segments() + ret_code, std_err, std_out = gpexpand.initialize_segments(additional_params) if ret_code != 0: raise Exception("gpexpand exited with return code: %d.\nstderr=%s\nstdout=%s" % (ret_code, std_err, std_out)) @@ -2264,10 +2268,14 @@ def impl(context, num_of_segments): @given('the cluster is setup for an expansion on hosts "{hostnames}"') def impl(context, hostnames): hosts = hostnames.split(",") - temp_base_dir = context.temp_base_dir + base_dir = "/tmp" + if hasattr(context, "temp_base_dir"): + base_dir = context.temp_base_dir + elif hasattr(context, "working_directory"): + base_dir = context.working_directory for host in hosts: cmd = Command(name='create data directories for expansion', - cmdStr="mkdir -p %s/data/primary; mkdir -p %s/data/mirror" % (temp_base_dir, temp_base_dir), + cmdStr="mkdir -p %s/data/primary; mkdir -p %s/data/mirror" % (base_dir, base_dir), ctxt=REMOTE, remoteHost=host) cmd.run(validateAfter=True) @@ -2283,8 +2291,10 @@ def impl(context, tmp_base_dir): @given('the new host "{hostnames}" is ready to go') def impl(context, hostnames): hosts = hostnames.split(',') - reset_hosts(hosts, context.working_directory) - reset_hosts(hosts, context.temp_base_dir) + if hasattr(context, "working_directory"): + reset_hosts(hosts, context.working_directory) + if hasattr(context, "temp_base_dir"): + reset_hosts(hosts, context.temp_base_dir) @then('the database is killed on hosts "{hostnames}"') @given('the database is killed on hosts "{hostnames}"') diff --git a/gpMgmt/test/behave_utils/cluster_expand.py b/gpMgmt/test/behave_utils/cluster_expand.py index 9cb95e8e35b87c8644b29c9b55fd40fa746d9d0f..889234f86f3bd2e458647ce992a7515900b35a60 100755 --- a/gpMgmt/test/behave_utils/cluster_expand.py +++ b/gpMgmt/test/behave_utils/cluster_expand.py @@ -63,9 +63,9 @@ class Gpexpand: return output, p1.wait() - def initialize_segments(self): + def initialize_segments(self, additional_params=''): input_files = sorted(glob.glob('%s/gpexpand_inputfile*' % self.working_directory)) - return run_gpcommand(self.context, "gpexpand -D %s -i %s" % (self.database, input_files[-1])) + return run_gpcommand(self.context, "gpexpand -D %s -i %s %s" % (self.database, input_files[-1], additional_params)) def redistribute(self, duration, endtime): # Can flake with "[ERROR]:-End time occurs in the past"