From dd1e154464e2a646392709581dc48074b7623c33 Mon Sep 17 00:00:00 2001 From: Wang Hao Date: Wed, 27 Mar 2019 17:49:25 +0800 Subject: [PATCH] Add gpperfmon test for all supported platforms (#7174) (#7202) For GP 6 beta, the release engineering team is removing the apr-util package from the list of bundled dependencies. Users will be asked to provide their own apr-util package, which can differ in the version on each platform. So it is necessary to verify gpperfmon is workable with the platform provided apr-util on each supported platform. Originally, gpperfmon test is done in CLI test suite, only covers centos6. This commit changed it to a dedicated suite in order to test multiple platforms. Note: on SLES12 does not need to install libapr-util1 to run gpmmon --- concourse/pipelines/gen_pipeline.py | 5 +- concourse/pipelines/gpdb_master-generated.yml | 149 ++++++++++++++-- concourse/pipelines/templates/gpdb-tpl.yml | 162 +++++++++++++++++- concourse/tasks/run_behave_on_ccp_cluster.yml | 12 ++ .../test/behave/mgmt_utils/gpperfmon.feature | 2 +- .../behave/mgmt_utils/steps/mgmt_utils.py | 10 +- 6 files changed, 319 insertions(+), 21 deletions(-) create mode 100644 concourse/tasks/run_behave_on_ccp_cluster.yml diff --git a/concourse/pipelines/gen_pipeline.py b/concourse/pipelines/gen_pipeline.py index 1a5502d18e..dc42d2fef9 100755 --- a/concourse/pipelines/gen_pipeline.py +++ b/concourse/pipelines/gen_pipeline.py @@ -51,6 +51,7 @@ JOBS_THAT_ARE_GATES = ['gate_icw_start', 'gate_icw_end', 'gate_replication_start', 'gate_resource_groups_start', + 'gate_gpperfmon_start', 'gate_cli_start', 'gate_ud_start', 'gate_advanced_analytics_start', @@ -247,7 +248,7 @@ if __name__ == "__main__": PARSER.add_argument('-a', '--test_sections', action='store', dest='test_sections', - choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions'], + choices=['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions', 'Gpperfmon'], default=['ICW'], nargs='+', help='Select tests sections to run') @@ -267,7 +268,7 @@ if __name__ == "__main__": if ARGS.pipeline_type == 'prod': ARGS.os_types = ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16'] - ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions'] + ARGS.test_sections = ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions', 'Gpperfmon'] # if generating a dev pipeline but didn't specify an output, don't overwrite the master pipeline if ARGS.pipeline_type != 'prod' and os.path.basename(ARGS.output_filepath) == default_output_filename: diff --git a/concourse/pipelines/gpdb_master-generated.yml b/concourse/pipelines/gpdb_master-generated.yml index f6a6315170..a1079f0309 100644 --- a/concourse/pipelines/gpdb_master-generated.yml +++ b/concourse/pipelines/gpdb_master-generated.yml @@ -12,10 +12,10 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2019-03-13 16:06:35.154669 +## Generated by gen_pipeline.py at: 2019-03-19 11:16:19.529007 ## Template file: gpdb-tpl.yml ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16'] -## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions'] +## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'AA', 'Extensions', 'Gpperfmon'] ## ====================================================================== ## ====================================================================== @@ -60,13 +60,17 @@ groups: - resource_group_centos7 - resource_group_sles12 ## -------------------------------------------------------------------- + - gate_gpperfmon_start + - gpperfmon_centos6 + - gpperfmon_centos7 + - gpperfmon_sles12 + ## -------------------------------------------------------------------- - gate_cli_start - gprecoverseg - gpcheckcat - gpinitstandby - gpactivatestandby - analyzedb - - gpperfmon - gpinitsystem - gpstate - replication_slots @@ -168,6 +172,18 @@ groups: ## ====================================================================== +- name: Gpperfmon + jobs: + - gate_gpperfmon_start + - gpperfmon_centos6 + - compile_gpdb_centos6 + - gpperfmon_centos7 + - compile_gpdb_centos7 + - gpperfmon_sles12 + - compile_gpdb_sles11 + +## ====================================================================== + - name: CLI jobs: - gate_cli_start @@ -176,7 +192,6 @@ groups: - gpinitstandby - gpactivatestandby - analyzedb - - gpperfmon - gpinitsystem - gpstate - replication_slots @@ -1449,6 +1464,118 @@ jobs: <<: *set_failed + +## ====================================================================== +## __ +## __ _ _ __ _ __ ___ _ __ / _|_ __ ___ ___ _ __ +## / _` | '_ \| '_ \ / _ \ '__| |_| '_ ` _ \ / _ \| '_ \ +## | (_| | |_) | |_) | __/ | | _| | | | | | (_) | | | | +## \__, | .__/| .__/ \___|_| |_| |_| |_| |_|\___/|_| |_| +## |___/|_| |_| +## ====================================================================== + +- name: gate_gpperfmon_start + plan: + - aggregate: + - get: gpdb_src + passed: + - compile_gpdb_centos6 + - compile_gpdb_sles11 + - compile_gpdb_centos7 + trigger: true + - get: bin_gpdb_centos6 + passed: + - compile_gpdb_centos6 + - get: bin_gpdb_centos7 + passed: + - compile_gpdb_centos7 + - get: bin_gpdb_sles11 + passed: + - compile_gpdb_sles11 + - get: binary_swap_gpdb + passed: [compile_gpdb_binary_swap_centos6] + resource: binary_swap_gpdb_centos6 + trigger: true + +- name: gpperfmon_centos6 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: bin_gpdb + resource: bin_gpdb_centos6 + passed: [gate_gpperfmon_start] + trigger: true + - get: gpdb6-centos6-test + - task: gpperfmon + file: gpdb_src/concourse/tasks/behave_gpdb.yml + image: gpdb6-centos6-test + params: + BEHAVE_TAGS: gpperfmon + +- name: gpperfmon_centos7 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: bin_gpdb + resource: bin_gpdb_centos7 + passed: [gate_gpperfmon_start] + trigger: true + - get: gpdb6-centos7-test + - task: gpperfmon + file: gpdb_src/concourse/tasks/behave_gpdb.yml + image: gpdb6-centos7-test + params: + BEHAVE_TAGS: gpperfmon + +- name: gpperfmon_sles12 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: gpdb_binary + resource: bin_gpdb_sles11 + passed: [gate_gpperfmon_start] + trigger: true + - get: ccp_src + - get: ccp-image + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + PLATFORM: sles12 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + # PLATFORM is necessary to generate correct ssh config + PLATFORM: sles12 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: run_tests + file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml + image: ccp-image + params: + BEHAVE_FLAGS: --tags=gpperfmon + TEST_OS: sles12 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + + + ## ====================================================================== ## ____ _ ___ ## / ___| | |_ _| @@ -1686,15 +1813,6 @@ jobs: params: BEHAVE_TAGS: analyzedb -- name: gpperfmon - plan: - - aggregate: *gets_for_behave - - task: gpperfmon - file: gpdb_src/concourse/tasks/behave_gpdb.yml - image: gpdb6-centos6-test - params: - BEHAVE_TAGS: gpperfmon - - name: gpinitsystem plan: - aggregate: *gets_for_behave @@ -2066,13 +2184,15 @@ jobs: ## - icw_extensions_gpcloud_ubuntu16 - resource_group_centos6 - resource_group_centos7 + - gpperfmon_centos6 + - gpperfmon_centos7 + - gpperfmon_sles12 ## - resource_group_sles12 - gprecoverseg - gpcheckcat - gpinitstandby - gpactivatestandby - analyzedb - - gpperfmon - gpinitsystem - gpstate - replication_slots @@ -2105,7 +2225,6 @@ jobs: - gpinitstandby - gpactivatestandby - analyzedb - - gpperfmon - gpinitsystem - gpstate - replication_slots diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index 8db9d489ba..14638f60c5 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -4,7 +4,6 @@ 'gpinitstandby', 'gpactivatestandby'] %} {% set CLI_2_suites = ['analyzedb', - 'gpperfmon', 'gpinitsystem', 'gpstate', 'replication_slots'] %} @@ -83,6 +82,17 @@ groups: - resource_group_sles12 {% endif %} {% endif %} +{% if "Gpperfmon" in test_sections %} + ## -------------------------------------------------------------------- + - gate_gpperfmon_start + - gpperfmon_centos6 +{% if "centos7" in os_types %} + - gpperfmon_centos7 +{% endif %} +{% if "sles" in os_types %} + - gpperfmon_sles12 +{% endif %} +{% endif %} {% if "CLI" in test_sections %} ## -------------------------------------------------------------------- - gate_cli_start @@ -241,6 +251,24 @@ groups: - compile_gpdb_sles11 {% endif %} +{% endif %} +{% if "Gpperfmon" in test_sections %} +## ====================================================================== + +- name: Gpperfmon + jobs: + - gate_gpperfmon_start + - gpperfmon_centos6 + - compile_gpdb_centos6 +{% if "centos7" in os_types %} + - gpperfmon_centos7 + - compile_gpdb_centos7 +{% endif %} +{% if "sles" in os_types %} + - gpperfmon_sles12 + - compile_gpdb_sles11 +{% endif %} + {% endif %} {% if "CLI" in test_sections %} ## ====================================================================== @@ -310,6 +338,7 @@ resource_types: resources: {% if ("ICW" in test_sections and "sles" in os_types) or "ResourceGroups" in test_sections or + ("Gpperfmon" in test_sections and "sles" in os_types) or "CLI" in test_sections %} - name: ccp_src type: git @@ -1646,6 +1675,134 @@ jobs: {% endif %} {% endif %} + +{% if "Gpperfmon" in test_sections %} +## ====================================================================== +## __ +## __ _ _ __ _ __ ___ _ __ / _|_ __ ___ ___ _ __ +## / _` | '_ \| '_ \ / _ \ '__| |_| '_ ` _ \ / _ \| '_ \ +## | (_| | |_) | |_) | __/ | | _| | | | | | (_) | | | | +## \__, | .__/| .__/ \___|_| |_| |_| |_| |_|\___/|_| |_| +## |___/|_| |_| +## ====================================================================== + +- name: gate_gpperfmon_start + plan: + - aggregate: + - get: gpdb_src + passed: + - compile_gpdb_centos6 +{% if "sles" in os_types %} + - compile_gpdb_sles11 +{% endif %} +{% if "centos7" in os_types %} + - compile_gpdb_centos7 +{% endif %} + trigger: true + - get: bin_gpdb_centos6 + passed: + - compile_gpdb_centos6 +{% if "centos7" in os_types %} + - get: bin_gpdb_centos7 + passed: + - compile_gpdb_centos7 +{% endif %} +{% if "sles" in os_types %} + - get: bin_gpdb_sles11 + passed: + - compile_gpdb_sles11 +{% endif %} +{% if "centos6" in os_types or "centos7" in os_types %} + - get: binary_swap_gpdb + passed: [compile_gpdb_binary_swap_centos6] + resource: binary_swap_gpdb_centos6 + trigger: [[ test_trigger ]] +{% endif %} + +- name: gpperfmon_centos6 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: bin_gpdb + resource: bin_gpdb_centos6 + passed: [gate_gpperfmon_start] + trigger: true + - get: gpdb6-centos6-test + - task: gpperfmon + file: gpdb_src/concourse/tasks/behave_gpdb.yml + image: gpdb6-centos6-test + params: + BEHAVE_TAGS: gpperfmon + +{% if "centos7" in os_types %} +- name: gpperfmon_centos7 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: bin_gpdb + resource: bin_gpdb_centos7 + passed: [gate_gpperfmon_start] + trigger: true + - get: gpdb6-centos7-test + - task: gpperfmon + file: gpdb_src/concourse/tasks/behave_gpdb.yml + image: gpdb6-centos7-test + params: + BEHAVE_TAGS: gpperfmon + +{% endif %} +{% if "sles" in os_types %} +- name: gpperfmon_sles12 + plan: + - aggregate: + - get: gpdb_src + params: + submodules: + - gpMgmt/bin/pythonSrc/ext + passed: [gate_gpperfmon_start] + - get: gpdb_binary + resource: bin_gpdb_sles11 + passed: [gate_gpperfmon_start] + trigger: true + - get: ccp_src + - get: ccp-image + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + PLATFORM: sles12 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + # PLATFORM is necessary to generate correct ssh config + PLATFORM: sles12 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: run_tests + file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml + image: ccp-image + params: + BEHAVE_FLAGS: --tags=gpperfmon + TEST_OS: sles12 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +{% endif %} + +{% endif %} + {% if "CLI" in test_sections %} ## ====================================================================== ## ____ _ ___ @@ -2146,6 +2303,9 @@ jobs: ## - icw_extensions_gpcloud_ubuntu16 - resource_group_centos6 - resource_group_centos7 + - gpperfmon_centos6 + - gpperfmon_centos7 + - gpperfmon_sles12 ## - resource_group_sles12 {% for test_name in CLI_1_suites + CLI_2_suites %} diff --git a/concourse/tasks/run_behave_on_ccp_cluster.yml b/concourse/tasks/run_behave_on_ccp_cluster.yml new file mode 100644 index 0000000000..318707f0c2 --- /dev/null +++ b/concourse/tasks/run_behave_on_ccp_cluster.yml @@ -0,0 +1,12 @@ +platform: linux +inputs: + - name: ccp_src + - name: cluster_env_files +run: + path: bash + args: + - -c + - | + set -ex + ccp_src/scripts/setup_ssh_to_cluster.sh + ssh -t mdw "$CUSTOM_ENV bash /home/gpadmin/gpdb_src/concourse/scripts/run_behave_test.sh \"$BEHAVE_FLAGS\"" diff --git a/gpMgmt/test/behave/mgmt_utils/gpperfmon.feature b/gpMgmt/test/behave/mgmt_utils/gpperfmon.feature index 31d076c15e..bf4a0ff626 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpperfmon.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpperfmon.feature @@ -15,7 +15,7 @@ Feature: gpperfmon Scenario: install gpperfmon Given the environment variable "PGDATABASE" is set to "template1" Given the database "gpperfmon" does not exist - When the user runs "gpperfmon_install --port 15432 --enable --password foo" + When the user installs gpperfmon Then gpperfmon_install should return a return code of 0 Then verify that the last line of the file "postgresql.conf" in the master data directory contains the string "gpperfmon_log_alert_level=warning" Then verify that the last line of the file "pg_hba.conf" in the master data directory contains the string "host all gpmon ::1/128 md5" diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 807cc9aa33..eab838f58f 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -1748,17 +1748,23 @@ def impl(context, table_name, db_name): dbconn.execSQL(conn, index_qry) conn.commit() +@when("the user installs gpperfmon") +def impl(context): + master_port = os.getenv('PGPORT', 15432) + cmd = "gpperfmon_install --port {master_port} --enable --password foo".format(master_port=master_port) + run_command(context, cmd) @given('gpperfmon is configured and running in qamode') @then('gpperfmon is configured and running in qamode') def impl(context): + master_port = os.getenv('PGPORT', 15432) target_line = 'qamode = 1' gpperfmon_config_file = "%s/gpperfmon/conf/gpperfmon.conf" % os.getenv("MASTER_DATA_DIRECTORY") if not check_db_exists("gpperfmon", "localhost"): context.execute_steps(u''' - When the user runs "gpperfmon_install --port 15432 --enable --password foo" + When the user runs "gpperfmon_install --port {master_port} --enable --password foo" Then gpperfmon_install should return a return code of 0 - ''') + '''.format(master_port=master_port)) if not file_contains_line(gpperfmon_config_file, target_line): context.execute_steps(u''' -- GitLab