resgroup: backward compatibility for memory auditor

Bring back the resgroup memory auditor feature: - 4354d336 - 8ede074c - 140d4d2e Memory auditor was a new feature introduced to allow external components (e.g. pl/container) managed by resource group. This feature requires a new gpdb dir to be created in cgroup memory controller, however on 5X branch unless the users created this new dir manually then the upgrade from a previous version would fail. In this commit we provide backward compatibility by checking the release version: - on 6X and master branches the memory auditor feature is always enabled so the new gpdb dir is mandatory; - on 5X branch only if the new gpdb dir is created with proper permissions the memory auditor feature could be enabled, when it's disabled `CREATE RESOURCE GROUP WITH (memory_auditor='cgroup') will fail with guide information on how to enable it; Binary swap tests are also provided to verify backward compatibility in future releases. As cgroup need to be configured to enable resgroup we split the resgroup binary swap tests into two parts: - resqueue mode only tests which can be triggered in the icw_gporca_centos6 pipeline job after the ICW tests, these parts have no requirements on cgroup; - complete resqueue & resgroup modes tests which can be triggered in the mpp_resource_group_centos{6,7} pipeline jobs after the resgroup tests, these parts need cgroup to be properly configured;

resgroup: backward compatibility for memory auditor
Bring back the resgroup memory auditor feature: - 4354d336 - 8ede074c - 140d4d2e Memory auditor was a new feature introduced to allow external components (e.g. pl/container) managed by resource group. This feature requires a new gpdb dir to be created in cgroup memory controller, however on 5X branch unless the users created this new dir manually then the upgrade from a previous version would fail. In this commit we provide backward compatibility by checking the release version: - on 6X and master branches the memory auditor feature is always enabled so the new gpdb dir is mandatory; - on 5X branch only if the new gpdb dir is created with proper permissions the memory auditor feature could be enabled, when it's disabled `CREATE RESOURCE GROUP WITH (memory_auditor='cgroup') will fail with guide information on how to enable it; Binary swap tests are also provided to verify backward compatibility in future releases. As cgroup need to be configured to enable resgroup we split the resgroup binary swap tests into two parts: - resqueue mode only tests which can be triggered in the icw_gporca_centos6 pipeline job after the ICW tests, these parts have no requirements on cgroup; - complete resqueue & resgroup modes tests which can be triggered in the mpp_resource_group_centos{6,7} pipeline jobs after the resgroup tests, these parts need cgroup to be properly configured;
23cd8b1e · Ning Yu · GitHub · 1cb7a49d · 23cd8b1e · 23cd8b1e
48 changed file
--- a/concourse/pipelines/5X_STABLE-generated.yml
+++ b/concourse/pipelines/5X_STABLE-generated.yml
@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2018-04-10 15:19:50.474125
+## Generated by gen_pipeline.py at: 2018-04-19 13:53:01.195453
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
 ## Test Sections: ['ICW', 'CS', 'MPP', 'MM', 'DPM', 'UD', 'FileRep', 'AA']
@@ -2151,6 +2151,9 @@ jobs:
    - get: bin_gpdb_centos7
      passed:
      - gate_compile_end
+    - get: binary_swap_gpdb_centos6
+      passed:
+      - gate_compile_end
 - name: mpp_interconnect
  plan:
@@ -2192,6 +2195,10 @@ jobs:
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      resource: binary_swap_gpdb_centos6
+      passed: [gate_mpp_start]
+      trigger: true
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2228,6 +2235,10 @@ jobs:
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-7
+    - get: binary_swap_gpdb
+      resource: binary_swap_gpdb_centos6
+      passed: [gate_mpp_start]
+      trigger: true
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2314,6 +2325,10 @@ jobs:
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      passed: [compile_gpdb_binary_swap_centos6]
+      resource: binary_swap_gpdb_centos6
+      trigger: true
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2350,6 +2365,10 @@ jobs:
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      passed: [compile_gpdb_binary_swap_centos6]
+      resource: binary_swap_gpdb_centos6
+      trigger: true
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2386,6 +2405,10 @@ jobs:
      trigger: true
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      passed: [compile_gpdb_binary_swap_centos6]
+      resource: binary_swap_gpdb_centos6
+      trigger: true
  - put: terraform
    params:
      <<: *ccp_default_params

--- a/concourse/pipelines/templates/gpdb-tpl.yml
+++ b/concourse/pipelines/templates/gpdb-tpl.yml
@@ -1919,6 +1919,9 @@ jobs:
    - get: bin_gpdb_centos7
      passed:
      - gate_compile_end
+    - get: binary_swap_gpdb_centos6
+      passed:
+      - gate_compile_end
 - name: mpp_interconnect
  plan:
@@ -1960,6 +1963,10 @@ jobs:
      trigger: [[ test_trigger ]]
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      resource: binary_swap_gpdb_centos6
+      passed: [gate_mpp_start]
+      trigger: [[ test_trigger ]]
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -1996,6 +2003,10 @@ jobs:
      trigger: [[ test_trigger ]]
    - get: ccp_src
    - get: centos-gpdb-dev-7
+    - get: binary_swap_gpdb
+      resource: binary_swap_gpdb_centos6
+      passed: [gate_mpp_start]
+      trigger: [[ test_trigger ]]
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2087,6 +2098,10 @@ jobs:
      trigger: [[ test_trigger ]]
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      passed: [compile_gpdb_binary_swap_centos6]
+      resource: binary_swap_gpdb_centos6
+      trigger: [[ test_trigger ]]
  - put: terraform
    params:
      <<: *ccp_default_params
@@ -2124,6 +2139,10 @@ jobs:
      trigger: [[ test_trigger ]]
    - get: ccp_src
    - get: centos-gpdb-dev-6
+    - get: binary_swap_gpdb
+      passed: [compile_gpdb_binary_swap_centos6]
+      resource: binary_swap_gpdb_centos6
+      trigger: [[ test_trigger ]]
  - put: terraform
    params:
      <<: *ccp_default_params

--- a/concourse/scripts/ic_gpdb_resgroup.bash
+++ b/concourse/scripts/ic_gpdb_resgroup.bash
@@ -89,6 +89,50 @@ EOF1
 EOF
 }
+keep_minimal_cgroup_dirs() {
+    local gpdb_master_alias=$1
+    local basedir=/cgroup
+    if [ "$TEST_OS" = "centos7" ]; then basedir=/sys/fs/cgroup; fi
+    ssh -t $gpdb_master_alias sudo bash -ex <<EOF
+        rmdir $basedir/memory/gpdb/*/ || :
+        rmdir $basedir/memory/gpdb
+EOF
+}
+setup_binary_swap_test() {
+    local gpdb_master_alias=$1
+    if [ "${TEST_BINARY_SWAP}" != "true" ]; then
+        return 0
+    fi
+    ssh $gpdb_master_alias mkdir -p /tmp/local/greenplum-db-devel
+    ssh $gpdb_master_alias tar -zxf - -C /tmp/local/greenplum-db-devel \
+        < binary_swap_gpdb/bin_gpdb.tar.gz
+    ssh $gpdb_master_alias sed -i -e "s@/usr/local@/tmp/local@" \
+        /tmp/local/greenplum-db-devel/greenplum_path.sh
+}
+run_binary_swap_test() {
+    local gpdb_master_alias=$1
+    if [ "${TEST_BINARY_SWAP}" != "true" ]; then
+        return 0
+    fi
+    scp -r /tmp/build/*/binary_swap_gpdb/ $gpdb_master_alias:/home/gpadmin/
+    ssh $gpdb_master_alias bash -ex <<EOF
+        source /usr/local/greenplum-db-devel/greenplum_path.sh
+        export PGPORT=5432
+        export MASTER_DATA_DIRECTORY=/data/gpdata/master/gpseg-1
+        export BINARY_SWAP_VARIANT=_resgroup
+        cd /home/gpadmin
+        time ./gpdb_src/concourse/scripts/test_binary_swap_gpdb.bash
+EOF
+}
 validate_env ccp-${CLUSTER_NAME}-0
 validate_env ccp-${CLUSTER_NAME}-1
 mount_cgroups ccp-${CLUSTER_NAME}-0
@@ -96,3 +140,16 @@ mount_cgroups ccp-${CLUSTER_NAME}-1
 make_cgroups_dir ccp-${CLUSTER_NAME}-0
 make_cgroups_dir ccp-${CLUSTER_NAME}-1
 run_resgroup_test mdw
+#
+# below is for binary swap test
+#
+# remove cgroup dirs for new features such as the memory auditor
+# the purpose is to ensure the backward compatibilities
+keep_minimal_cgroup_dirs ccp-${CLUSTER_NAME}-0
+keep_minimal_cgroup_dirs ccp-${CLUSTER_NAME}-1
+# deploy the binaries for binary swap test
+setup_binary_swap_test sdw1
+# run it
+run_binary_swap_test mdw
--- a/concourse/scripts/test_binary_swap_gpdb.bash
+++ b/concourse/scripts/test_binary_swap_gpdb.bash
@@ -2,15 +2,37 @@
 set -eox pipefail
+if [ "$USER" = gpadmin ]; then
+    tmpdir=/tmp
+else
+    tmpdir=/opt
+fi
 function gen_env() {
-    cat > /opt/run_test.sh <<-EOF
+    cat > $tmpdir/run_test.sh <<-EOF
        source /usr/local/greenplum-db-devel/greenplum_path.sh
+        [ -e \${1}/gpdb_src/gpAux/gpdemo/gpdemo-env.sh ] && \
        source \${1}/gpdb_src/gpAux/gpdemo/gpdemo-env.sh
        cd "\${1}/gpdb_src/src/test/binary_swap"
-        ./test_binary_swap.sh -b /tmp/local/greenplum-db-devel
+        ./test_binary_swap.sh -b /tmp/local/greenplum-db-devel \
+            -v "${BINARY_SWAP_VARIANT}" || (
+            errcode=\$?
+            find . -name regression.diffs \
+            | while read diff; do
+                cat <<EOF1
+======================================================================
+DIFF FILE: \$diff
+----------------------------------------------------------------------
+EOF1
+                cat \$diff
+              done
+            exit \$errcode
+        )
 EOF
-    chmod a+x /opt/run_test.sh
+    chmod a+x $tmpdir/run_test.sh
 }
 function install_gpdb_binary_swap() {
@@ -21,7 +43,11 @@ function install_gpdb_binary_swap() {
 }
 function run_test() {
-    su - gpadmin -c "bash /opt/run_test.sh $(pwd)"
+    if [ "$USER" = gpadmin ]; then
+        bash $tmpdir/run_test.sh $(pwd)
+    else
+        su - gpadmin -c "bash $tmpdir/run_test.sh $(pwd)"
+    fi
 }
 function _main() {

--- a/concourse/tasks/ic_gpdb_resgroup.yml
+++ b/concourse/tasks/ic_gpdb_resgroup.yml
@@ -9,12 +9,14 @@ inputs:
  - name: gpdb_src
  - name: ccp_src
  - name: cluster_env_files
+  - name: binary_swap_gpdb
 outputs:
 params:
  MAKE_TEST_COMMAND: ""
  BLDWRAP_POSTGRES_CONF_ADDONS: ""
  TEST_OS: ""
+  TEST_BINARY_SWAP: true
  CONFIGURE_FLAGS: ""
 run:

--- a/gpMgmt/bin/gpcheckresgroupimpl
+++ b/gpMgmt/bin/gpcheckresgroupimpl
@@ -6,6 +6,8 @@
 import os
 import sys
 import psutil
+from gppylib.commands import gp
+from gppylib import gpversion
 class dummy(object):
@@ -55,6 +57,16 @@ class cgroup(object):
        self.validate_permission("memory/memory.limit_in_bytes", "r")
+        # resgroup memory auditor is introduced in 6.0 devel and backported
+        # to 5.x branch since 5.6.1.  To provide backward compatibilities
+        # memory permissions are only checked since 6.0.
+        gpverstr = gp.GpVersion.local("", os.getenv("GPHOME"))
+        gpver = gpversion.GpVersion(gpverstr)
+        if gpver.version >= [6, 0, 0]:
+            self.validate_permission("memory/gpdb/", "rwx")
+            self.validate_permission("memory/gpdb/memory.limit_in_bytes", "rw")
+            self.validate_permission("memory/gpdb/memory.usage_in_bytes", "r")
    def die(self, msg):
        exit(self.impl + self.error_prefix + msg)

--- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckresgroupimpl.py
+++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckresgroupimpl.py
@@ -14,6 +14,11 @@ gpcheckresgroupimpl_path = os.path.abspath('gpcheckresgroupimpl')
 gpcheckresgroupimpl = imp.load_source('gpcheckresgroupimpl', gpcheckresgroupimpl_path)
 import gpcheckresgroupimpl
+from gppylib.commands import gp
+from gppylib import gpversion
+gpverstr = gp.GpVersion.local("", os.getenv("GPHOME"))
+gpver = gpversion.GpVersion(gpverstr)
 @unittest.skipUnless(sys.platform.startswith("linux"), "requires linux")
 class GpCheckResGroupImplCGroup(unittest.TestCase):
    cgroup_mntpnt = None
@@ -44,6 +49,10 @@ class GpCheckResGroupImplCGroup(unittest.TestCase):
        self.touch(os.path.join(self.cgroup_mntpnt, "memory", "memory.limit_in_bytes"), 0400)
        self.touch(os.path.join(self.cgroup_mntpnt, "memory", "memory.memsw.limit_in_bytes"), 0400)
+        os.mkdir(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"), 0700)
+        self.touch(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.limit_in_bytes"), 0600)
+        self.touch(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.usage_in_bytes"), 0400)
    def tearDown(self):
        shutil.rmtree(self.cgroup_mntpnt)
        self.cgroup = None
@@ -160,6 +169,61 @@ class GpCheckResGroupImplCGroup(unittest.TestCase):
        with self.assertRaisesRegexp(AssertionError, "file '.*/memory/memory.limit_in_bytes' does not exist"):
            self.cgroup.validate_all()
+    def test_when_memory_limit_in_bytes_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "memory.limit_in_bytes"), 0100)
+        with self.assertRaisesRegexp(AssertionError, "file '.*/memory/memory.limit_in_bytes' permission denied: require permission 'r'"):
+            self.cgroup.validate_all()
+    def test_when_memory_gpdb_dir_missing(self):
+        shutil.rmtree(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"))
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "directory '.*/memory/gpdb/' does not exist"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
+    def test_when_memory_gpdb_dir_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"), 0500)
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "directory '.*/memory/gpdb/' permission denied: require permission 'rwx'"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
+        # restore permission for the dir to be removed in tearDown()
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb"), 0700)
+    def test_when_memory_gpdb_limit_in_bytes_missing(self):
+        os.unlink(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.limit_in_bytes"))
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "file '.*/memory/gpdb/memory.limit_in_bytes' does not exist"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
+    def test_when_memory_gpdb_limit_in_bytes_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.limit_in_bytes"), 0100)
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "file '.*/memory/gpdb/memory.limit_in_bytes' permission denied: require permission 'rw'"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
+    def test_when_memory_gpdb_usage_in_bytes_missing(self):
+        os.unlink(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.usage_in_bytes"))
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "file '.*/memory/gpdb/memory.usage_in_bytes' does not exist"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
+    def test_when_memory_gpdb_usage_in_bytes_bad_permission(self):
+        os.chmod(os.path.join(self.cgroup_mntpnt, "memory", "gpdb", "memory.usage_in_bytes"), 0100)
+        if gpver.version >= [6, 0, 0]:
+            with self.assertRaisesRegexp(AssertionError, "file '.*/memory/gpdb/memory.usage_in_bytes' permission denied: require permission 'r'"):
+                self.cgroup.validate_all()
+        else:
+            self.cgroup.validate_all()
 if __name__ == '__main__':
    unittest.main()
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -1240,7 +1240,14 @@ gpvars_assign_gp_resource_manager_policy(const char *newval, bool doit, GucSourc
 {
 	ResourceManagerPolicy newtype = RESOURCE_MANAGER_POLICY_QUEUE;
-	if (newval == NULL || newval[0] == 0 )
+	/*
+	 * Probe resgroup configurations even not in resgroup mode,
+	 * variables like gp_resource_group_enable_cgroup_memory need to
+	 * be properly set in all modes.
+	 */
+	ResGroupOps_Probe();
+	if (newval == NULL || newval[0] == 0)
 		newtype = RESOURCE_MANAGER_POLICY_QUEUE;
 	else if (!pg_strcasecmp("queue", newval))
 		newtype = RESOURCE_MANAGER_POLICY_QUEUE;

--- a/src/backend/commands/resgroupcmds.c
+++ b/src/backend/commands/resgroupcmds.c
@@ -42,6 +42,9 @@
 #define RESGROUP_DEFAULT_MEM_SHARED_QUOTA (20)
 #define RESGROUP_DEFAULT_MEM_SPILL_RATIO (20)
+#define RESGROUP_DEFAULT_MEM_AUDITOR (RESGROUP_MEMORY_AUDITOR_VMTRACKER)
+#define RESGROUP_INVALID_MEM_AUDITOR (-1)
 #define RESGROUP_MIN_CONCURRENCY	(0)
 #define RESGROUP_MAX_CONCURRENCY	(MaxConnections)
@@ -57,20 +60,31 @@
 #define RESGROUP_MIN_MEMORY_SPILL_RATIO		(0)
 #define RESGROUP_MAX_MEMORY_SPILL_RATIO		(100)
+/*
+ * The names must be in the same order as ResGroupMemAuditorType.
+ */
+static const char *ResGroupMemAuditorName[] =
+{
+	"vmtracker",	// RESGROUP_MEMORY_AUDITOR_VMTRACKER
+	"cgroup"		// RESGROUP_MEMORY_AUDITOR_CGROUP
+};
 /*
 * The context to pass to callback in ALTER resource group
 */
 typedef struct {
 	Oid		groupid;
-	int		limittype;
+	ResGroupLimitType	limittype;
 	ResGroupCaps	caps;
+	ResGroupCap		memLimitGap;
 } ResourceGroupAlterCallbackContext;
 static int str2Int(const char *str, const char *prop);
 static ResGroupLimitType getResgroupOptionType(const char* defname);
-static ResGroupCap getResgroupOptionValue(DefElem *defel);
+static ResGroupCap getResgroupOptionValue(DefElem *defel, int type);
 static const char *getResgroupOptionName(ResGroupLimitType type);
 static void checkResgroupCapLimit(ResGroupLimitType type, ResGroupCap value);
+static void checkResgroupMemAuditor(ResGroupCaps *caps);
 static void parseStmtOptions(CreateResourceGroupStmt *stmt, ResGroupCaps *caps);
 static void validateCapabilities(Relation rel, Oid groupid, ResGroupCaps *caps, bool newGroup);
 static void insertResgroupCapabilityEntry(Relation rel, Oid groupid, uint16 type, char *value);
@@ -84,6 +98,7 @@ static void checkAuthIdForDrop(Oid groupId);
 static void createResgroupCallback(XactEvent event, void *arg);
 static void dropResgroupCallback(XactEvent event, void *arg);
 static void alterResgroupCallback(XactEvent event, void *arg);
+static int getResGroupMemAuditor(char *name);
 /*
 * CREATE RESOURCE GROUP
@@ -220,6 +235,7 @@ CreateResourceGroup(CreateResourceGroupStmt *stmt)
 		/* Create os dependent part for this resource group */
 		ResGroupOps_CreateGroup(groupid);
 		ResGroupOps_SetCpuRateLimit(groupid, caps.cpuRateLimit);
+		ResGroupOps_SetMemoryLimit(groupid, caps.memLimit);
 	}
 	else if (Gp_role == GP_ROLE_DISPATCH)
 		ereport(WARNING,
@@ -357,7 +373,7 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt)
 				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
 				 errmsg("option \"%s\" not recognized", defel->defname)));
-	value = getResgroupOptionValue(defel);
+	value = getResgroupOptionValue(defel, limitType);
 	checkResgroupCapLimit(limitType, value);
 	/*
@@ -396,6 +412,8 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt)
 	oldValue = capArray[limitType];
 	capArray[limitType] = value;
+	checkResgroupMemAuditor(&caps);
 	if ((limitType == RESGROUP_LIMIT_TYPE_CPU ||
 		 limitType == RESGROUP_LIMIT_TYPE_MEMORY) &&
 		oldValue < value)
@@ -431,6 +449,8 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt)
 		callbackCtx->groupid = groupid;
 		callbackCtx->limittype = limitType;
 		callbackCtx->caps = caps;
+		callbackCtx->memLimitGap = (limitType == RESGROUP_LIMIT_TYPE_MEMORY) ?
+			(oldValue - value) : 0;
 		RegisterXactCallbackOnce(alterResgroupCallback, (void *)callbackCtx);
 	}
 }
@@ -484,9 +504,11 @@ GetResGroupCapabilities(Relation rel, Oid groupId, ResGroupCaps *resgroupCaps)
 		type = (ResGroupLimitType) DatumGetInt16(typeDatum);
 		Assert(type > RESGROUP_LIMIT_TYPE_UNKNOWN);
-		Assert(type < RESGROUP_LIMIT_TYPE_COUNT);
 		Assert(!(mask & (1 << type)));
+		if (type >= RESGROUP_LIMIT_TYPE_COUNT)
+			continue;
 		mask |= 1 << type;
 		proposedDatum = heap_getattr(tuple, Anum_pg_resgroupcapability_proposed,
@@ -506,6 +528,22 @@ GetResGroupCapabilities(Relation rel, Oid groupId, ResGroupCaps *resgroupCaps)
 	}
 }
+/*
+ * GetResGroupMemAuditorForId -- Return the resource group memory auditor
+ * for a groupId
+ */
+int32
+GetResGroupMemAuditorForId(Oid groupId, LOCKMODE lockmode)
+{
+	ResGroupCaps		caps;
+	Relation pg_resgroupcapability_rel = heap_open(
+			ResGroupCapabilityRelationId, lockmode);
+	GetResGroupCapabilities(pg_resgroupcapability_rel, groupId, &caps);
+	heap_close(pg_resgroupcapability_rel, lockmode);
+	return caps.memAuditor;
+}
 /*
 * Get resource group id for a role in pg_authid.
 *
@@ -643,6 +681,29 @@ GetResGroupNameForId(Oid oid, LOCKMODE lockmode)
 	return name;
 }
+/*
+ * Check to see if the group can be assigned with role
+ */
+void
+ResGroupCheckForRole(Oid groupId)
+{
+	Relation pg_resgroupcapability_rel;
+	ResGroupCaps caps;
+	pg_resgroupcapability_rel = heap_open(ResGroupCapabilityRelationId,
+										  AccessShareLock);
+	/* Load current resource group capabilities */
+	GetResGroupCapabilities(pg_resgroupcapability_rel, groupId, &caps);
+	if (caps.memAuditor == RESGROUP_MEMORY_AUDITOR_CGROUP)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("You cannot assign a role to this resource group because "
+					    "the memory_auditor property for this group is not the default.")));
+	heap_close(pg_resgroupcapability_rel, AccessShareLock);
+}
 /*
 * Get the option type from a name string.
 *
@@ -663,6 +724,8 @@ getResgroupOptionType(const char* defname)
 		return RESGROUP_LIMIT_TYPE_MEMORY_SHARED_QUOTA;
 	else if (strcmp(defname, "memory_spill_ratio") == 0)
 		return RESGROUP_LIMIT_TYPE_MEMORY_SPILL_RATIO;
+	else if (strcmp(defname, "memory_auditor") == 0)
+		return RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR;
 	else
 		return RESGROUP_LIMIT_TYPE_UNKNOWN;
 }
@@ -671,9 +734,20 @@ getResgroupOptionType(const char* defname)
 * Get capability value from DefElem, convert from int64 to int
 */
 static ResGroupCap
-getResgroupOptionValue(DefElem *defel)
+getResgroupOptionValue(DefElem *defel, int type)
 {
-	int64 value = defGetInt64(defel);
+	int64 value;
+	if (type == RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR)
+	{
+		char *auditor_name = defGetString(defel);
+		value = getResGroupMemAuditor(auditor_name);
+	}
+	else
+	{
+		value = defGetInt64(defel);
+	}
 	if (value < INT_MIN || value > INT_MAX)
 		ereport(ERROR,
 				(errcode(ERRCODE_SYNTAX_ERROR),
@@ -765,12 +839,60 @@ checkResgroupCapLimit(ResGroupLimitType type, int value)
 								   RESGROUP_MAX_MEMORY_SPILL_RATIO)));
 				break;
+			case RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR:
+				if (value != RESGROUP_MEMORY_AUDITOR_VMTRACKER &&
+					value != RESGROUP_MEMORY_AUDITOR_CGROUP)
+					ereport(ERROR,
+							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+							errmsg("memory_auditor should be \"%s\" or \"%s\"",
+								   ResGroupMemAuditorName[RESGROUP_MEMORY_AUDITOR_VMTRACKER],
+								   ResGroupMemAuditorName[RESGROUP_MEMORY_AUDITOR_CGROUP])));
+				break;
 			default:
 				Assert(!"unexpected options");
 				break;
 		}
 }
+/*
+ * Check to see if concurrency is not zero for resource group
+ * with cgroup memory auditor.
+ */
+static void
+checkResgroupMemAuditor(ResGroupCaps *caps)
+{
+	if (caps->memAuditor == RESGROUP_MEMORY_AUDITOR_CGROUP &&
+		caps->concurrency != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				errmsg("resource group concurrency must be 0 when group memory_auditor is %s",
+					ResGroupMemAuditorName[RESGROUP_MEMORY_AUDITOR_CGROUP])));
+	if (caps->memAuditor == RESGROUP_MEMORY_AUDITOR_CGROUP &&
+		!gp_resource_group_enable_cgroup_memory)
+	{
+		/*
+		 * Suppose the user has reconfigured the cgroup dirs by following
+		 * the gpdb documents, could it take effect at runtime (e.g. create
+		 * the resgroup again) without restart the cluster?
+		 *
+		 * It's possible but might not be reliable, as the user might
+		 * introduced unwanted changes to other cgroup dirs during the
+		 * reconfiguration (e.g. changed the permissions, moved processes
+		 * in/out).
+		 *
+		 * So we do not recheck the permissions here.
+		 */
+		ereport(ERROR,
+				(errcode(ERRCODE_GP_FEATURE_NOT_CONFIGURED),
+				 errmsg("cgroup is not properly configured for the 'cgroup' memory auditor"),
+				 errhint("Extra cgroup configurations are required to enable this feature, "
+						 "please refer to the Greenplum Documentations for details")));
+	}
+}
 /*
 * Parse a statement and store the settings in options.
 *
@@ -803,7 +925,7 @@ parseStmtOptions(CreateResourceGroupStmt *stmt, ResGroupCaps *caps)
 		else
 			mask |= 1 << type;
-		value = getResgroupOptionValue(defel);
+		value = getResgroupOptionValue(defel, type);
 		checkResgroupCapLimit(type, value);
 		capArray[type] = value;
@@ -823,6 +945,11 @@ parseStmtOptions(CreateResourceGroupStmt *stmt, ResGroupCaps *caps)
 	if (!(mask & (1 << RESGROUP_LIMIT_TYPE_MEMORY_SPILL_RATIO)))
 		caps->memSpillRatio = RESGROUP_DEFAULT_MEM_SPILL_RATIO;
+	if (!(mask & (1 << RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR)))
+		caps->memAuditor = RESGROUP_DEFAULT_MEM_AUDITOR;
+	checkResgroupMemAuditor(caps);
 }
 /*
@@ -875,7 +1002,8 @@ alterResgroupCallback(XactEvent event, void *arg)
 		(ResourceGroupAlterCallbackContext *) arg;
 	if (event == XACT_EVENT_COMMIT)
-		ResGroupAlterOnCommit(ctx->groupid, ctx->limittype, &ctx->caps);
+		ResGroupAlterOnCommit(ctx->groupid, ctx->limittype, &ctx->caps,
+				ctx->memLimitGap);
 	pfree(arg);
 }
@@ -918,6 +1046,10 @@ insertResgroupCapabilities(Relation rel, Oid groupId, ResGroupCaps *caps)
 	sprintf(value, "%d", caps->memSpillRatio);
 	insertResgroupCapabilityEntry(rel, groupId,
 								  RESGROUP_LIMIT_TYPE_MEMORY_SPILL_RATIO, value);
+	sprintf(value, "%d", caps->memAuditor);
+	insertResgroupCapabilityEntry(rel, groupId,
+								  RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR, value);
 }
 /*
@@ -1180,3 +1312,20 @@ str2Int(const char *str, const char *prop)
 	return floor(val);
 }
+/*
+ * Get memory auditor from auditor name.
+ */
+static int
+getResGroupMemAuditor(char *name)
+{
+	int index;
+	for (index = 0; index < RESGROUP_MEMORY_AUDITOR_COUNT; index ++)
+	{
+		if (strcmp(ResGroupMemAuditorName[index], name) == 0)
+			return index;
+	}
+	return RESGROUP_INVALID_MEM_AUDITOR;
+}
--- a/src/backend/commands/user.c
+++ b/src/backend/commands/user.c
@@ -544,6 +544,8 @@ CreateRole(CreateRoleStmt *stmt)
 					(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
 					 errmsg("only superuser can be assigned to admin resgroup")));
+		ResGroupCheckForRole(rsgid);
 		new_record[Anum_pg_authid_rolresgroup - 1] = ObjectIdGetDatum(rsgid);
 		if (!IsResGroupActivated() && Gp_role == GP_ROLE_DISPATCH)
 			ereport(WARNING,
@@ -1203,6 +1205,7 @@ AlterRole(AlterRoleStmt *stmt)
 			ereport(ERROR,
 					(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
 					 errmsg("only superuser can be assigned to admin resgroup")));
+		ResGroupCheckForRole(rsgid);
 		new_record[Anum_pg_authid_rolresgroup - 1] =
 			ObjectIdGetDatum(rsgid);
 		new_record_repl[Anum_pg_authid_rolresgroup - 1] = true;

--- a/src/backend/utils/mmgr/vmem_tracker.c
+++ b/src/backend/utils/mmgr/vmem_tracker.c
@@ -370,6 +370,13 @@ VmemTracker_ConvertVmemChunksToBytes(int chunks)
 	return CHUNKS_TO_BYTES(chunks);
 }
+/* Converts bytes to chunks */
+int32
+VmemTracker_ConvertVmemBytesToChunks(int64 bytes)
+{
+	return BYTES_TO_CHUNKS(bytes);
+}
 /*
 * Returns the maximum vmem consumed by current process in "chunks" unit.
 */

--- a/src/backend/utils/resgroup/resgroup-ops-dummy.c
+++ b/src/backend/utils/resgroup/resgroup-ops-dummy.c
@@ -36,6 +36,19 @@ ResGroupOps_Name(void)
 	return "unsupported";
 }
+/*
+ * Probe the configuration for the OS group implementation.
+ *
+ * Return true if everything is OK, or false is some requirements are not
+ * satisfied.  Will not fail in either case.
+ */
+bool
+ResGroupOps_Probe(void)
+{
+	unsupported_system();
+	return false;
+}
 /* Check whether the OS group implementation is available and useable */
 void
 ResGroupOps_Bless(void)
@@ -69,10 +82,11 @@ ResGroupOps_CreateGroup(Oid group)
 /*
 * Destroy the OS group for group.
 *
- * Fail if any process is running under it.
+ * One OS group can not be dropped if there are processes running under it,
+ * if migrate is true these processes will be moved out automatically.
 */
 void
-ResGroupOps_DestroyGroup(Oid group)
+ResGroupOps_DestroyGroup(Oid group, bool migrate)
 {
 	unsupported_system();
 }
@@ -101,7 +115,7 @@ ResGroupOps_AssignGroup(Oid group, int pid)
 * ResGroupOps_UnLockGroup() to unblock it.
 */
 int
-ResGroupOps_LockGroup(Oid group, bool block)
+ResGroupOps_LockGroup(Oid group, const char *comp, bool block)
 {
 	unsupported_system();
 	return -1;
@@ -129,6 +143,28 @@ ResGroupOps_SetCpuRateLimit(Oid group, int cpu_rate_limit)
 	unsupported_system();
 }
+/*
+ * Set the memory limit for the OS group by rate.
+ *
+ * memory_limit should be within [0, 100].
+ */
+void
+ResGroupOps_SetMemoryLimit(Oid group, int memory_limit)
+{
+	unsupported_system();
+}
+/*
+ * Set the memory limit for the OS group by value.
+ *
+ * memory_limit is the limit value in chunks
+ */
+void
+ResGroupOps_SetMemoryLimitByValue(Oid group, int32 memory_limit)
+{
+	unsupported_system();
+}
 /*
 * Get the cpu usage of the OS group, that is the total cpu time obtained
 * by this OS group, in nano seconds.
@@ -140,6 +176,30 @@ ResGroupOps_GetCpuUsage(Oid group)
 	return 0;
 }
+/*
+ * Get the memory usage of the OS group
+ *
+ * memory usage is returned in chunks
+ */
+int32
+ResGroupOps_GetMemoryUsage(Oid group)
+{
+	unsupported_system();
+	return 0;
+}
+/*
+ * Get the memory limit of the OS group
+ *
+ * memory limit is returned in chunks
+ */
+int32
+ResGroupOps_GetMemoryLimit(Oid group)
+{
+	unsupported_system();
+	return 0;
+}
 /*
 * Get the count of cpu cores on the system.
 */

--- a/src/backend/utils/resgroup/resgroup-ops-linux.c
+++ b/src/backend/utils/resgroup/resgroup-ops-linux.c
@@ -42,34 +42,135 @@
 * So far these operations are mainly for CPU rate limitation and accounting.
 */
-#define CGROUP_ERROR_PREFIX "cgroup is not properly configured: "
+#define CGROUP_ERROR(...) elog(ERROR, __VA_ARGS__)
-#define CGROUP_ERROR(...) do { \
+#define CGROUP_CONFIG_ERROR(...) \
-	elog(ERROR, CGROUP_ERROR_PREFIX __VA_ARGS__); \
+	CGROUP_ERROR("cgroup is not properly configured: " __VA_ARGS__)
-} while (false)
 #define PROC_MOUNTS "/proc/self/mounts"
 #define MAX_INT_STRING_LEN 20
 #define MAX_RETRY 10
+/*
+ * cgroup memory permission is only mandatory on 6.x and master;
+ * on 5.x we need to make it optional to provide backward compatibilities.
+ */
+#define CGROUP_MEMORY_IS_OPTIONAL (GP_VERSION_NUM < 60000)
+typedef struct PermItem PermItem;
+typedef struct PermList PermList;
+struct PermItem
+{
+	const char	*comp;
+	const char	*prop;
+	int			perm;
+};
+struct PermList
+{
+	const PermItem	*items;
+	bool			optional;
+	bool			*presult;
+};
+#define foreach_perm_list(i, lists) \
+	for ((i) = 0; (lists)[(i)].items; (i)++)
+#define foreach_perm_item(i, items) \
+	for ((i) = 0; (items)[(i)].comp; (i)++)
 static char * buildPath(Oid group, const char *base, const char *comp, const char *prop, char *path, size_t pathsize);
 static int lockDir(const char *path, bool block);
 static void unassignGroup(Oid group, const char *comp, int fddir);
 static bool createDir(Oid group, const char *comp);
-static bool removeDir(Oid group, const char *comp, bool unassign);
+static bool removeDir(Oid group, const char *comp, const char *prop, bool unassign);
 static int getCpuCores(void);
 static size_t readData(const char *path, char *data, size_t datasize);
 static void writeData(const char *path, char *data, size_t datasize);
 static int64 readInt64(Oid group, const char *base, const char *comp, const char *prop);
 static void writeInt64(Oid group, const char *base, const char *comp, const char *prop, int64 x);
+static bool permListCheck(const PermList *permlist, Oid group, bool report);
 static bool checkPermission(Oid group, bool report);
 static void getMemoryInfo(unsigned long *ram, unsigned long *swap);
 static void getCgMemoryInfo(uint64 *cgram, uint64 *cgmemsw);
 static int getOvercommitRatio(void);
-static void detectCgroupMountPoint(void);
+static bool detectCgroupMountPoint(void);
 static Oid currentGroupIdInCGroup = InvalidOid;
 static char cgdir[MAXPGPATH];
+bool gp_resource_group_enable_cgroup_memory = false;
+bool gp_resource_group_enable_cgroup_swap = false;
+/*
+ * These checks should keep in sync with gpMgmt/bin/gpcheckresgroupimpl
+ */
+static const PermItem perm_items_cpu[] =
+{
+	{ "cpu", "", R_OK | W_OK | X_OK },
+	{ "cpu", "cgroup.procs", R_OK | W_OK },
+	{ "cpu", "cpu.cfs_period_us", R_OK | W_OK },
+	{ "cpu", "cpu.cfs_quota_us", R_OK | W_OK },
+	{ "cpu", "cpu.shares", R_OK | W_OK },
+	{ NULL, NULL, 0 }
+};
+static const PermItem perm_items_cpu_acct[] =
+{
+	{ "cpuacct", "", R_OK | W_OK | X_OK },
+	{ "cpuacct", "cgroup.procs", R_OK | W_OK },
+	{ "cpuacct", "cpuacct.usage", R_OK },
+	{ "cpuacct", "cpuacct.stat", R_OK },
+	{ NULL, NULL, 0 }
+};
+static const PermItem perm_items_memory[] =
+{
+	{ "memory", "", R_OK | W_OK | X_OK },
+	{ "memory", "memory.limit_in_bytes", R_OK | W_OK },
+	{ "memory", "memory.usage_in_bytes", R_OK },
+	{ NULL, NULL, 0 }
+};
+static const PermItem perm_items_swap[] =
+{
+	{ "memory", "", R_OK | W_OK | X_OK },
+	{ "memory", "memory.memsw.limit_in_bytes", R_OK | W_OK },
+	{ "memory", "memory.memsw.usage_in_bytes", R_OK },
+	{ NULL, NULL, 0 }
+};
+/*
+ * Permission groups.
+ */
+static const PermList permlists[] =
+{
+	/*
+	 * swap permissions are optional.
+	 *
+	 * cgroup/memory/memory.memsw.* is only available if
+	 * - CONFIG_MEMCG_SWAP_ENABLED=on in kernel config, or
+	 * - swapaccount=1 in kernel cmdline.
+	 *
+	 * Without these interfaces the swap usage can not be limited or accounted
+	 * via cgroup.
+	 */
+	{ perm_items_swap, true, &gp_resource_group_enable_cgroup_swap },
+	/*
+	 * memory permissions can be mandatory or optional depends on the switch.
+	 *
+	 * resgroup memory auditor is introduced in 6.0 devel and backported
+	 * to 5.x branch since 5.6.1.  To provide backward compatibilities memory
+	 * permissions are optional on 5.x branch.
+	 */
+	{ perm_items_memory, CGROUP_MEMORY_IS_OPTIONAL,
+		&gp_resource_group_enable_cgroup_memory },
+	/* cpu/cpuacct permissions are mandatory */
+	{ perm_items_cpu, false, NULL },
+	{ perm_items_cpu_acct, false, NULL },
+	{ NULL, false, NULL }
+};
 /*
 * Build path string with parameters.
 * - if base is NULL, use default value "gpdb"
@@ -240,7 +341,7 @@ lockDir(const char *path, bool block)
 	}
 	int flags = LOCK_EX;
-	if (block)
+	if (!block)
 		flags |= LOCK_NB;
 	while (flock(fddir, flags))
@@ -318,11 +419,11 @@ createDir(Oid group, const char *comp)
 * - if unassign is true then unassign all the processes first before removal;
 */
 static bool
-removeDir(Oid group, const char *comp, bool unassign)
+removeDir(Oid group, const char *comp, const char *prop, bool unassign)
 {
 	char path[MAXPGPATH];
 	size_t pathsize = sizeof(path);
-	int retry = 0;
+	int retry = unassign ? 0 : MAX_RETRY - 1;
 	int fddir;
 	buildPath(group, NULL, comp, "", path, pathsize);
@@ -338,6 +439,12 @@ removeDir(Oid group, const char *comp, bool unassign)
 		return true;
 	}
+	/*
+	 * Reset the corresponding control file to zero
+	 */
+	if (prop)
+		writeInt64(group, NULL, comp, prop, 0);
 	while (++retry <= MAX_RETRY)
 	{
 		if (unassign)
@@ -498,54 +605,71 @@ writeInt64(Oid group, const char *base, const char *comp, const char *prop, int6
 }
 /*
- * Check permissions on group's cgroup dir & interface files.
+ * Check a list of permissions on group.
 *
- * - if report is true then raise an error on and bad permission,
+ * - if all the permissions are met then return true;
- *   otherwise only return false;
+ * - otherwise:
+ *   - raise an error if report is true and permlist is not optional;
+ *   - or return false;
 */
 static bool
-checkPermission(Oid group, bool report)
+permListCheck(const PermList *permlist, Oid group, bool report)
 {
 	char path[MAXPGPATH];
 	size_t pathsize = sizeof(path);
-	const char *comp;
+	int i;
-#define __CHECK(prop, perm) do { \
+	if (group == RESGROUP_ROOT_ID && permlist->presult)
-	buildPath(group, NULL, comp, prop, path, pathsize); \
+		*permlist->presult = false;
-	if (access(path, perm)) \
-	{ \
-		if (report) \
-		{ \
-			CGROUP_ERROR("can't access %s '%s': %s", \
-						 prop[0] ? "file" : "directory", \
-						 path, \
-						 strerror(errno)); \
-		} \
-		return false; \
-	} \
-} while (0)
-    /*
+	foreach_perm_item(i, permlist->items)
-     * These checks should keep in sync with
+	{
-     * gpMgmt/bin/gpcheckresgroupimpl
+		const char	*comp = permlist->items[i].comp;
-     */
+		const char	*prop = permlist->items[i].prop;
+		int			perm = permlist->items[i].perm;
-	comp = "cpu";
+		buildPath(group, NULL, comp, prop, path, pathsize);
-	__CHECK("", R_OK | W_OK | X_OK);
+		if (access(path, perm))
-	__CHECK("cgroup.procs", R_OK | W_OK);
+		{
-	__CHECK("cpu.cfs_period_us", R_OK | W_OK);
+			/* No such file or directory / Permission denied */
-	__CHECK("cpu.cfs_quota_us", R_OK | W_OK);
-	__CHECK("cpu.shares", R_OK | W_OK);
-	comp = "cpuacct";
+			if (report && !permlist->optional)
+			{
+				CGROUP_CONFIG_ERROR("can't access %s '%s': %s",
+									prop[0] ? "file" : "directory",
+									path,
+									strerror(errno));
+			}
+			return false;
+		}
+	}
-	__CHECK("", R_OK | W_OK | X_OK);
+	if (group == RESGROUP_ROOT_ID && permlist->presult)
-	__CHECK("cgroup.procs", R_OK | W_OK);
+		*permlist->presult = true;
-	__CHECK("cpuacct.usage", R_OK);
-	__CHECK("cpuacct.stat", R_OK);
-#undef __CHECK
+	return true;
+}
+/*
+ * Check permissions on group's cgroup dir & interface files.
+ *
+ * - if report is true then raise an error if any mandatory permission
+ *   is not met;
+ * - otherwise only return false;
+ */
+static bool
+checkPermission(Oid group, bool report)
+{
+	int i;
+	foreach_perm_list(i, permlists)
+	{
+		const PermList *permlist = &permlists[i];
+		if (!permListCheck(permlist, group, report) && !permlist->optional)
+			return false;
+	}
 	return true;
 }
@@ -565,20 +689,9 @@ getMemoryInfo(unsigned long *ram, unsigned long *swap)
 static void
 getCgMemoryInfo(uint64 *cgram, uint64 *cgmemsw)
 {
-	char path[MAXPGPATH];
-	size_t pathsize = sizeof(path);
 	*cgram = readInt64(RESGROUP_ROOT_ID, "", "memory", "memory.limit_in_bytes");
-	/*
+	if (gp_resource_group_enable_cgroup_swap)
-	 * cgroup/memory/memory.memsw.limit_in_bytes is only available if
-	 * CONFIG_MEMCG_SWAP_ENABLED is on in kernel config or
-	 * swapaccount=1 in cmdline. Without this file we have to assume swap is
-	 * unlimited in container.
-	 */
-	buildPath(RESGROUP_ROOT_ID, "",
-			  "memory", "memory.memsw.limit_in_bytes", path, pathsize);
-	if (access(path, R_OK) == 0)
 	{
 		*cgmemsw = readInt64(RESGROUP_ROOT_ID, "",
 							 "memory", "memory.memsw.limit_in_bytes");
@@ -608,18 +721,18 @@ getOvercommitRatio(void)
 }
 /* detect cgroup mount point */
-static void
+static bool
 detectCgroupMountPoint(void)
 {
 	struct mntent *me;
 	FILE *fp;
 	if (cgdir[0])
-		return;
+		return true;
 	fp = setmntent(PROC_MOUNTS, "r");
 	if (fp == NULL)
-		CGROUP_ERROR("can not open '%s' for read", PROC_MOUNTS);
+		CGROUP_CONFIG_ERROR("can not open '%s' for read", PROC_MOUNTS);
 	while ((me = getmntent(fp)))
@@ -633,7 +746,7 @@ detectCgroupMountPoint(void)
 		p = strrchr(cgdir, '/');
 		if (p == NULL)
-			CGROUP_ERROR("cgroup mount point parse error: %s", cgdir);
+			CGROUP_CONFIG_ERROR("cgroup mount point parse error: %s", cgdir);
 		else
 			*p = 0;
 		break;
@@ -641,8 +754,7 @@ detectCgroupMountPoint(void)
 	endmntent(fp);
-	if (!cgdir[0])
+	return !!cgdir[0];
-		CGROUP_ERROR("can not find cgroup mount point");
 }
 /* Return the name for the OS group implementation */
@@ -652,6 +764,39 @@ ResGroupOps_Name(void)
 	return "cgroup";
 }
+/*
+ * Probe the configuration for the OS group implementation.
+ *
+ * Return true if everything is OK, or false is some requirements are not
+ * satisfied.  Will not fail in either case.
+ */
+bool
+ResGroupOps_Probe(void)
+{
+	/*
+	 * We only have to do these checks and initialization once on each host,
+	 * so only let postmaster do the job.
+	 */
+	if (IsUnderPostmaster)
+		return true;
+	/*
+	 * Ignore the error even if cgroup mount point can not be successfully
+	 * probed, the error will be reported in Bless() later.
+	 */
+	if (!detectCgroupMountPoint())
+		return false;
+	/*
+	 * Probe for optional features like the 'cgroup' memory auditor,
+	 * do not raise any errors.
+	 */
+	if (!checkPermission(RESGROUP_ROOT_ID, false))
+		return false;
+	return true;
+}
 /* Check whether the OS group implementation is available and useable */
 void
 ResGroupOps_Bless(void)
@@ -663,7 +808,18 @@ ResGroupOps_Bless(void)
 	if (IsUnderPostmaster)
 		return;
-	detectCgroupMountPoint();
+	/*
+	 * We should have already detected for cgroup mount point in Probe(),
+	 * it was not an error if the detection failed at that step.  But once
+	 * we call Bless() we know we want to make use of cgroup then we must
+	 * know the mount point, otherwise it's a critical error.
+	 */
+	if (!cgdir[0])
+		CGROUP_CONFIG_ERROR("can not find cgroup mount point");
+	/*
+	 * Check again, this time we will fail on unmet requirements.
+	 */
 	checkPermission(RESGROUP_ROOT_ID, true);
 	/*
@@ -725,7 +881,10 @@ ResGroupOps_CreateGroup(Oid group)
 {
 	int retry = 0;
-	if (!createDir(group, "cpu") || !createDir(group, "cpuacct"))
+	if (!createDir(group, "cpu")
+		|| !createDir(group, "cpuacct")
+		|| (gp_resource_group_enable_cgroup_memory &&
+			!createDir(group, "memory")))
 	{
 		CGROUP_ERROR("can't create cgroup for resgroup '%d': %s",
 					 group, strerror(errno));
@@ -751,12 +910,16 @@ ResGroupOps_CreateGroup(Oid group)
 /*
 * Destroy the OS group for group.
 *
- * Fail if any process is running under it.
+ * One OS group can not be dropped if there are processes running under it,
+ * if migrate is true these processes will be moved out automatically.
 */
 void
-ResGroupOps_DestroyGroup(Oid group)
+ResGroupOps_DestroyGroup(Oid group, bool migrate)
 {
-	if (!removeDir(group, "cpu", true) || !removeDir(group, "cpuacct", true))
+	if (!removeDir(group, "cpu", "cpu.shares", migrate)
+		|| !removeDir(group, "cpuacct", NULL, migrate)
+		|| (gp_resource_group_enable_cgroup_memory &&
+			!removeDir(group, "memory", "memory.limit_in_bytes", migrate)))
 	{
 		CGROUP_ERROR("can't remove cgroup for resgroup '%d': %s",
 			 group, strerror(errno));
@@ -779,6 +942,10 @@ ResGroupOps_AssignGroup(Oid group, int pid)
 	writeInt64(group, NULL, "cpu", "cgroup.procs", pid);
 	writeInt64(group, NULL, "cpuacct", "cgroup.procs", pid);
+	/*
+	 * Do not assign the process to cgroup/memory for now.
+	 */
 	currentGroupIdInCGroup = group;
 }
@@ -793,12 +960,12 @@ ResGroupOps_AssignGroup(Oid group, int pid)
 * ResGroupOps_UnLockGroup() to unblock it.
 */
 int
-ResGroupOps_LockGroup(Oid group, bool block)
+ResGroupOps_LockGroup(Oid group, const char *comp, bool block)
 {
 	char path[MAXPGPATH];
 	size_t pathsize = sizeof(path);
-	buildPath(group, NULL, "cpu", "", path, pathsize);
+	buildPath(group, NULL, comp, "", path, pathsize);
 	return lockDir(path, block);
 }
@@ -831,6 +998,84 @@ ResGroupOps_SetCpuRateLimit(Oid group, int cpu_rate_limit)
 	writeInt64(group, NULL, comp, "cpu.shares", shares * cpu_rate_limit / 100);
 }
+/*
+ * Set the memory limit for the OS group by rate.
+ *
+ * memory_limit should be within [0, 100].
+ */
+void
+ResGroupOps_SetMemoryLimit(Oid group, int memory_limit)
+{
+	int fd;
+	int32 memory_limit_in_chunks;
+	memory_limit_in_chunks = ResGroupGetVmemLimitChunks() * memory_limit / 100;
+	memory_limit_in_chunks *= ResGroupGetSegmentNum();
+	fd = ResGroupOps_LockGroup(group, "memory", true);
+	ResGroupOps_SetMemoryLimitByValue(group, memory_limit_in_chunks);
+	ResGroupOps_UnLockGroup(group, fd);
+}
+/*
+ * Set the memory limit for the OS group by value.
+ *
+ * memory_limit is the limit value in chunks
+ *
+ * If cgroup supports memory swap, we will write the same limit to
+ * memory.memsw.limit and memory.limit.
+ */
+void
+ResGroupOps_SetMemoryLimitByValue(Oid group, int32 memory_limit)
+{
+	const char *comp = "memory";
+	int64 memory_limit_in_bytes;
+	if (!gp_resource_group_enable_cgroup_memory)
+		return;
+	memory_limit_in_bytes = VmemTracker_ConvertVmemChunksToBytes(memory_limit);
+	/* Is swap interfaces enabled? */
+	if (!gp_resource_group_enable_cgroup_swap)
+	{
+		/* No, then we only need to setup the memory limit */
+		writeInt64(group, NULL, comp, "memory.limit_in_bytes",
+				memory_limit_in_bytes);
+	}
+	else
+	{
+		/* Yes, then we have to setup both the memory and mem+swap limits */
+		int64 memory_limit_in_bytes_old;
+		/*
+		 * Memory limit should always <= mem+swap limit, then the limits
+		 * must be set in a proper order depending on the relation between
+		 * new and old limits.
+		 */
+		memory_limit_in_bytes_old = readInt64(group, NULL,
+				comp, "memory.limit_in_bytes");
+		if (memory_limit_in_bytes > memory_limit_in_bytes_old)
+		{
+			/* When new value > old memory limit, write mem+swap limit first */
+			writeInt64(group, NULL, comp, "memory.memsw.limit_in_bytes",
+					memory_limit_in_bytes);
+			writeInt64(group, NULL, comp, "memory.limit_in_bytes",
+					memory_limit_in_bytes);
+		}
+		else if (memory_limit_in_bytes < memory_limit_in_bytes_old)
+		{
+			/* When new value < old memory limit,  write memory limit first */
+			writeInt64(group, NULL, comp, "memory.limit_in_bytes",
+					memory_limit_in_bytes);
+			writeInt64(group, NULL, comp, "memory.memsw.limit_in_bytes",
+					memory_limit_in_bytes);
+		}
+	}
+}
 /*
 * Get the cpu usage of the OS group, that is the total cpu time obtained
 * by this OS group, in nano seconds.
@@ -843,6 +1088,52 @@ ResGroupOps_GetCpuUsage(Oid group)
 	return readInt64(group, NULL, comp, "cpuacct.usage");
 }
+/*
+ * Get the memory usage of the OS group
+ *
+ * memory usage is returned in chunks
+ */
+int32
+ResGroupOps_GetMemoryUsage(Oid group)
+{
+	const char *comp = "memory";
+	int64 memory_usage_in_bytes;
+	char *prop;
+	/* Report 0 if cgroup memory is not enabled */
+	if (!gp_resource_group_enable_cgroup_memory)
+		return 0;
+	prop = gp_resource_group_enable_cgroup_swap
+		? "memory.memsw.usage_in_bytes"
+		: "memory.usage_in_bytes";
+	memory_usage_in_bytes = readInt64(group, NULL, comp, prop);
+	return VmemTracker_ConvertVmemBytesToChunks(memory_usage_in_bytes);
+}
+/*
+ * Get the memory limit of the OS group
+ *
+ * memory limit is returned in chunks
+ */
+int32
+ResGroupOps_GetMemoryLimit(Oid group)
+{
+	const char *comp = "memory";
+	int64 memory_limit_in_bytes;
+	/* Report unlimited (max int32) if cgroup memory is not enabled */
+	if (!gp_resource_group_enable_cgroup_memory)
+		return (int32) ((1U << 31) - 1);
+	memory_limit_in_bytes = readInt64(group, NULL,
+			comp, "memory.limit_in_bytes");
+	return VmemTracker_ConvertVmemBytesToChunks(memory_limit_in_bytes);
+}
 /*
 * Get the count of cpu cores on the system.
 */

--- a/src/backend/utils/resgroup/resgroup.c
+++ b/src/backend/utils/resgroup/resgroup.c
@@ -146,6 +146,21 @@ struct ResGroupSlotData
 	ResGroupSlotData	*next;
 };
+/*
+ * Resource group operations for memory.
+ *
+ * Groups with different memory auditor will have different
+ * operations.
+ */
+typedef struct ResGroupMemOperations
+{
+	void (*group_mem_on_create) (Oid groupId, ResGroupData *group);
+	void (*group_mem_on_alter) (Oid groupId, ResGroupData *group);
+	void (*group_mem_on_drop) (Oid groupId, ResGroupData *group);
+	void (*group_mem_on_notify) (ResGroupData *group);
+	void (*group_mem_on_dump) (ResGroupData *group, StringInfo str);
+} ResGroupMemOperations;
 /*
 * Resource group information.
 */
@@ -161,6 +176,15 @@ struct ResGroupData
 	bool		lockedForDrop;  /* true if resource group is dropped but not committed yet */
+	/*
+	 * memGap is calculated as:
+	 * 	(memory limit (before alter) - memory expected (after alter))
+	 *
+	 * It stands for how many memory (in chunks) this group should
+	 * give back to MEM POOL.
+	 */
+	int32       memGap;
 	int32		memExpected;		/* expected memory chunks according to current caps */
 	int32		memQuotaGranted;	/* memory chunks for quota part */
 	int32		memSharedGranted;	/* memory chunks for shared part */
@@ -174,6 +198,11 @@ struct ResGroupData
 	 */
 	int32		memUsage;
 	int32		memSharedUsage;
+	/*
+	 * operation functions for resource group
+	 */
+	const ResGroupMemOperations *groupMemOps;
 };
 struct ResGroupControl
@@ -232,14 +261,15 @@ static int32 slotGetMemQuotaExpected(const ResGroupCaps *caps);
 static int32 slotGetMemQuotaOnQE(const ResGroupCaps *caps, ResGroupData *group);
 static int32 slotGetMemSpill(const ResGroupCaps *caps);
 static void wakeupSlots(ResGroupData *group, bool grant);
-static void wakeupGroups(Oid skipGroupId);
+static void notifyGroupsOnMem(Oid skipGroupId);
 static int32 mempoolAutoRelease(ResGroupData *group);
 static int32 mempoolAutoReserve(ResGroupData *group, const ResGroupCaps *caps);
 static ResGroupData *groupHashNew(Oid groupId);
 static ResGroupData *groupHashFind(Oid groupId, bool raise);
-static void groupHashRemove(Oid groupId);
+static ResGroupData *groupHashRemove(Oid groupId);
 static void waitOnGroup(ResGroupData *group);
 static ResGroupData *createGroup(Oid groupId, const ResGroupCaps *caps);
+static void removeGroup(Oid groupId);
 static void AtProcExit_ResGroup(int code, Datum arg);
 static void groupWaitCancel(void);
 static int32 groupReserveMemQuota(ResGroupData *group);
@@ -295,6 +325,19 @@ static void sessionSetSlot(ResGroupSlotData *slot);
 static ResGroupSlotData *sessionGetSlot(void);
 static void sessionResetSlot(void);
+static void bindGroupOperation(ResGroupData *group);
+static void groupMemOnAlterForVmtracker(Oid groupId, ResGroupData *group);
+static void groupMemOnDropForVmtracker(Oid groupId, ResGroupData *group);
+static void groupMemOnNotifyForVmtracker(ResGroupData *group);
+static void groupMemOnDumpForVmtracker(ResGroupData *group, StringInfo str);
+static void groupMemOnAlterForCgroup(Oid groupId, ResGroupData *group);
+static void groupMemOnDropForCgroup(Oid groupId, ResGroupData *group);
+static void groupMemOnNotifyForCgroup(ResGroupData *group);
+static void groupMemOnDumpForCgroup(ResGroupData *group, StringInfo str);
+static void groupApplyCgroupMemInc(ResGroupData *group);
+static void groupApplyCgroupMemDec(ResGroupData *group);
 #ifdef USE_ASSERT_CHECKING
 static bool selfHasGroup(void);
 static bool selfHasSlot(void);
@@ -305,6 +348,28 @@ static bool groupIsNotDropped(const ResGroupData *group);
 static bool groupWaitQueueFind(ResGroupData *group, const PGPROC *proc);
 #endif /* USE_ASSERT_CHECKING */
+/*
+ * Operations of memory for resource groups with vmtracker memory auditor.
+ */
+static const ResGroupMemOperations resgroup_memory_operations_vmtracker = {
+	.group_mem_on_create	= NULL,
+	.group_mem_on_alter		= groupMemOnAlterForVmtracker,
+	.group_mem_on_drop		= groupMemOnDropForVmtracker,
+	.group_mem_on_notify	= groupMemOnNotifyForVmtracker,
+	.group_mem_on_dump		= groupMemOnDumpForVmtracker,
+};
+/*
+ * Operations of memory for resource groups with cgroup memory auditor.
+ */
+static const ResGroupMemOperations resgroup_memory_operations_cgroup = {
+	.group_mem_on_create	= NULL,
+	.group_mem_on_alter		= groupMemOnAlterForCgroup,
+	.group_mem_on_drop		= groupMemOnDropForCgroup,
+	.group_mem_on_notify	= groupMemOnNotifyForCgroup,
+	.group_mem_on_dump		= groupMemOnDumpForCgroup,
+};
 /*
 * Estimate size the resource group structures will need in
 * shared memory.
@@ -492,6 +557,7 @@ InitResGroups(void)
 		ResGroupOps_CreateGroup(groupId);
 		ResGroupOps_SetCpuRateLimit(groupId, cpuRateLimit);
+		ResGroupOps_SetMemoryLimit(groupId, caps.memLimit);
 		numGroups++;
 		Assert(numGroups <= MaxResourceGroups);
@@ -580,8 +646,14 @@ ResGroupDropFinish(Oid groupId, bool isCommit)
 		if (isCommit)
 		{
-			groupHashRemove(groupId);
+			bool		migrate;
-			ResGroupOps_DestroyGroup(groupId);
+			removeGroup(groupId);
+			/* Only migrate processes out of vmtracker groups */
+			migrate = group->caps.memAuditor == RESGROUP_MEMORY_AUDITOR_VMTRACKER;
+			ResGroupOps_DestroyGroup(groupId, migrate);
 		}
 	}
 	PG_CATCH();
@@ -617,9 +689,9 @@ ResGroupCreateOnAbort(Oid groupId)
 	PG_TRY();
 	{
 		savedInterruptHoldoffCount = InterruptHoldoffCount;
-		groupHashRemove(groupId);
+		removeGroup(groupId);
 		/* remove the os dependent part for this resource group */
-		ResGroupOps_DestroyGroup(groupId);
+		ResGroupOps_DestroyGroup(groupId, true);
 	}
 	PG_CATCH();
 	{
@@ -644,10 +716,10 @@ ResGroupCreateOnAbort(Oid groupId)
 void
 ResGroupAlterOnCommit(Oid groupId,
 					  ResGroupLimitType limittype,
-					  const ResGroupCaps *caps)
+					  const ResGroupCaps *caps,
+					  ResGroupCap memLimitGap)
 {
 	ResGroupData	*group;
-	bool			shouldWakeUp;
 	volatile int	savedInterruptHoldoffCount;
 	Assert(caps != NULL);
@@ -667,11 +739,12 @@ ResGroupAlterOnCommit(Oid groupId,
 		}
 		else if (limittype != RESGROUP_LIMIT_TYPE_MEMORY_SPILL_RATIO)
 		{
-			shouldWakeUp = groupApplyMemCaps(group);
+			Assert(pResGroupControl->totalChunks > 0);
+			group->memGap += pResGroupControl->totalChunks * memLimitGap / 100;
-			wakeupSlots(group, true);
+			Assert(group->groupMemOps != NULL);
-			if (shouldWakeUp)
+			if (group->groupMemOps->group_mem_on_alter)
-				wakeupGroups(groupId);
+				group->groupMemOps->group_mem_on_alter(groupId, group);
 		}
 	}
 	PG_CATCH();
@@ -761,6 +834,15 @@ ResGroupGetStat(Oid groupId, ResGroupStatType type)
 	return result;
 }
+/*
+ * Get the number of primary segments on this host
+ */
+int
+ResGroupGetSegmentNum()
+{
+	return (Gp_role == GP_ROLE_EXECUTE ? host_segments : pResGroupControl->segmentsOnMaster);
+}
 static char *
 groupDumpMemUsage(ResGroupData *group)
 {
@@ -768,33 +850,9 @@ groupDumpMemUsage(ResGroupData *group)
 	initStringInfo(&memUsage);
-	appendStringInfo(&memUsage, "{");
+	Assert(group->groupMemOps != NULL);
-	appendStringInfo(&memUsage, "\"used\":%d, ",
+	if (group->groupMemOps->group_mem_on_dump)
-					 VmemTracker_ConvertVmemChunksToMB(group->memUsage));
+		group->groupMemOps->group_mem_on_dump(group, &memUsage);
-	appendStringInfo(&memUsage, "\"available\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(
-						group->memQuotaGranted + group->memSharedGranted - group->memUsage));
-	appendStringInfo(&memUsage, "\"quota_used\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(group->memQuotaUsed));
-	appendStringInfo(&memUsage, "\"quota_available\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(
-						group->memQuotaGranted - group->memQuotaUsed));
-	appendStringInfo(&memUsage, "\"quota_granted\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(group->memQuotaGranted));
-	appendStringInfo(&memUsage, "\"quota_proposed\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(
-						groupGetMemQuotaExpected(&group->caps)));
-	appendStringInfo(&memUsage, "\"shared_used\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(group->memSharedUsage));
-	appendStringInfo(&memUsage, "\"shared_available\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(
-						group->memSharedGranted - group->memSharedUsage));
-	appendStringInfo(&memUsage, "\"shared_granted\":%d, ",
-					 VmemTracker_ConvertVmemChunksToMB(group->memSharedGranted));
-	appendStringInfo(&memUsage, "\"shared_proposed\":%d",
-					 VmemTracker_ConvertVmemChunksToMB(
-						groupGetMemSharedExpected(&group->caps)));
-	appendStringInfo(&memUsage, "}");
 	return memUsage.data;
 }
@@ -957,6 +1015,28 @@ ResourceGroupGetQueryMemoryLimit(void)
 	return memSpill << VmemTracker_GetChunkSizeInBits();
 }
+/*
+ * removeGroup -- remove resource group from share memory and
+ * reclaim the group's memory back to MEM POOL.
+ */
+static void
+removeGroup(Oid groupId)
+{
+	ResGroupData *group;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	Assert(OidIsValid(groupId));
+	group = groupHashRemove(groupId);
+	Assert(group->groupMemOps != NULL);
+	if (group->groupMemOps->group_mem_on_drop)
+		group->groupMemOps->group_mem_on_drop(groupId, group);
+	group->groupId = InvalidOid;
+	notifyGroupsOnMem(groupId);
+}
 /*
 * createGroup -- initialize the elements for a resource group.
 *
@@ -982,9 +1062,11 @@ createGroup(Oid groupId, const ResGroupCaps *caps)
 	ProcQueueInit(&group->waitProcs);
 	group->totalExecuted = 0;
 	group->totalQueued = 0;
+	group->memGap = 0;
 	group->memUsage = 0;
 	group->memSharedUsage = 0;
 	group->memQuotaUsed = 0;
+	group->groupMemOps = NULL;
 	memset(&group->totalQueuedTime, 0, sizeof(group->totalQueuedTime));
 	group->lockedForDrop = false;
@@ -995,9 +1077,29 @@ createGroup(Oid groupId, const ResGroupCaps *caps)
 	chunks = mempoolReserve(groupId, group->memExpected);
 	groupRebalanceQuota(group, chunks, caps);
+	bindGroupOperation(group);
 	return group;
 }
+/*
+ * Bind operation to resource group according to memory auditor.
+ */
+static void
+bindGroupOperation(ResGroupData *group)
+{
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	if (group->caps.memAuditor == RESGROUP_MEMORY_AUDITOR_VMTRACKER)
+		group->groupMemOps = &resgroup_memory_operations_vmtracker;
+	else if (group->caps.memAuditor == RESGROUP_MEMORY_AUDITOR_CGROUP)
+		group->groupMemOps = &resgroup_memory_operations_cgroup;
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid memory auditor: %d", group->caps.memAuditor)));
+}
 /*
 * Add chunks into group and slot memory usage.
 *
@@ -1256,7 +1358,7 @@ groupPutSlot(ResGroupData *group, ResGroupSlotData *slot)
 	/* And finally release the overused memory quota */
 	released = mempoolAutoRelease(group);
 	if (released > 0)
-		wakeupGroups(group->groupId);
+		notifyGroupsOnMem(group->groupId);
 	/*
 	 * Once we have waken up other groups then the slot we just released
@@ -1699,23 +1801,22 @@ wakeupSlots(ResGroupData *group, bool grant)
 }
 /*
- * When a group returns chunks to MEM POOL, we need to wake up
+ * When a group returns chunks to MEM POOL, we need to:
- * the transactions waiting on other groups for memory quota.
+ * 1. For groups with vmtracker memory auditor, wake up the
+ *    transactions waiting on them for memory quota.
+ * 2. For groups with cgroup memory auditor, increase their
+ *    memory limit if needed.
 */
 static void
-wakeupGroups(Oid skipGroupId)
+notifyGroupsOnMem(Oid skipGroupId)
 {
 	int				i;
-	if (Gp_role != GP_ROLE_DISPATCH)
-		return;
 	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
 	for (i = 0; i < MaxResourceGroups; i++)
 	{
 		ResGroupData	*group = &pResGroupControl->groups[i];
-		int32			delta;
 		if (group->groupId == InvalidOid)
 			continue;
@@ -1723,17 +1824,9 @@ wakeupGroups(Oid skipGroupId)
 		if (group->groupId == skipGroupId)
 			continue;
-		if (group->lockedForDrop)
+		Assert(group->groupMemOps != NULL);
-			continue;
+		if (group->groupMemOps->group_mem_on_notify)
+			group->groupMemOps->group_mem_on_notify(group);
-		if (groupWaitQueueIsEmpty(group))
-			continue;
-		delta = group->memExpected - group->memQuotaGranted - group->memSharedGranted;
-		if (delta <= 0)
-			continue;
-		wakeupSlots(group, true);
 		if (!pResGroupControl->freeChunks)
 			break;
@@ -2079,6 +2172,8 @@ UnassignResGroup(void)
 	}
 	else if (slot->nProcs == 0)
 	{
+		int32 released;
 		Assert(Gp_role == GP_ROLE_EXECUTE);
 		group->memQuotaUsed -= slot->memQuota;
@@ -2093,7 +2188,10 @@ UnassignResGroup(void)
 		group->nRunning--;
 		/* And finally release the overused memory quota */
-		mempoolAutoRelease(group);
+		released = mempoolAutoRelease(group);
+		if (released > 0)
+			notifyGroupsOnMem(group->groupId);
 	}
 	LWLockRelease(ResGroupLock);
@@ -2297,7 +2395,7 @@ groupHashFind(Oid groupId, bool raise)
 *	The resource group lightweight lock (ResGroupLock) *must* be held for
 *	this operation.
 */
-static void
+static ResGroupData *
 groupHashRemove(Oid groupId)
 {
 	bool		found;
@@ -2317,12 +2415,8 @@ groupHashRemove(Oid groupId)
 						groupId)));
 	group = &pResGroupControl->groups[entry->index];
-	mempoolRelease(groupId, group->memQuotaGranted + group->memSharedGranted);
-	group->memQuotaGranted = 0;
-	group->memSharedGranted = 0;
-	group->groupId = InvalidOid;
-	wakeupGroups(groupId);
+	return group;
 }
 /* Process exit without waiting for slot or received SIGTERM */
@@ -3110,3 +3204,230 @@ sessionResetSlot(void)
 	MySessionState->resGroupSlot = NULL;
 }
+/*
+ * Operation for resource groups with vmtracker memory auditor
+ * when alter its memory limit.
+ */
+static void
+groupMemOnAlterForVmtracker(Oid groupId, ResGroupData *group)
+{
+	bool shouldNotify;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	shouldNotify = groupApplyMemCaps(group);
+	wakeupSlots(group, true);
+	if (shouldNotify)
+		notifyGroupsOnMem(groupId);
+}
+/*
+ * Operation for resource groups with vmtracker memory auditor
+ * when reclaiming its memory back to MEM POOL.
+ */
+static void
+groupMemOnDropForVmtracker(Oid groupId, ResGroupData *group)
+{
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	mempoolRelease(groupId, group->memQuotaGranted + group->memSharedGranted);
+	group->memQuotaGranted = 0;
+	group->memSharedGranted = 0;
+}
+/*
+ * Operation for resource groups with vmtracker memory auditor
+ * when memory in MEM POOL is increased.
+ */
+static void
+groupMemOnNotifyForVmtracker(ResGroupData *group)
+{
+	int32			delta;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	if (Gp_role != GP_ROLE_DISPATCH)
+		return;
+	if (group->lockedForDrop)
+		return;
+	if (groupWaitQueueIsEmpty(group))
+		return;
+	delta = group->memExpected - group->memQuotaGranted - group->memSharedGranted;
+	if (delta <= 0)
+		return;
+	wakeupSlots(group, true);
+}
+/*
+ * Operation for resource groups with vmtracker memory auditor
+ * when dump memory statistics.
+ */
+static void
+groupMemOnDumpForVmtracker(ResGroupData *group, StringInfo str)
+{
+	appendStringInfo(str, "{");
+	appendStringInfo(str, "\"used\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(group->memUsage));
+	appendStringInfo(str, "\"available\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(
+				group->memQuotaGranted + group->memSharedGranted - group->memUsage));
+	appendStringInfo(str, "\"quota_used\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(group->memQuotaUsed));
+	appendStringInfo(str, "\"quota_available\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(
+				group->memQuotaGranted - group->memQuotaUsed));
+	appendStringInfo(str, "\"quota_granted\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(group->memQuotaGranted));
+	appendStringInfo(str, "\"quota_proposed\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(
+				groupGetMemQuotaExpected(&group->caps)));
+	appendStringInfo(str, "\"shared_used\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(group->memSharedUsage));
+	appendStringInfo(str, "\"shared_available\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(
+				group->memSharedGranted - group->memSharedUsage));
+	appendStringInfo(str, "\"shared_granted\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(group->memSharedGranted));
+	appendStringInfo(str, "\"shared_proposed\":%d",
+			VmemTracker_ConvertVmemChunksToMB(
+				groupGetMemSharedExpected(&group->caps)));
+	appendStringInfo(str, "}");
+}
+/*
+ * Operation for resource groups with cgroup memory auditor
+ * when alter its memory limit.
+ */
+static void
+groupMemOnAlterForCgroup(Oid groupId, ResGroupData *group)
+{
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	/*
+	 * If memGap is positive, it indicates this group should
+	 * give back these many memory back to MEM POOL.
+	 *
+	 * If memGap is negative, it indicates this group should
+	 * retrieve these many memory from MEM POOL.
+	 *
+	 * If memGap is zero, this group is holding the same memory
+	 * as it expects.
+	 */
+	if (group->memGap == 0)
+		return;
+	if (group->memGap > 0)
+		groupApplyCgroupMemDec(group);
+	else
+		groupApplyCgroupMemInc(group);
+}
+/*
+ * Increase a resource group's cgroup memory limit
+ *
+ * This may not take effect immediately.
+ */
+static void
+groupApplyCgroupMemInc(ResGroupData *group)
+{
+	int32 memory_limit;
+	int32 memory_inc;
+	int fd;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	Assert(group->memGap < 0);
+	memory_inc = mempoolReserve(group->groupId, group->memGap * -1);
+	if (memory_inc <= 0)
+		return;
+	fd = ResGroupOps_LockGroup(group->groupId, "memory", true);
+	memory_limit = ResGroupOps_GetMemoryLimit(group->groupId);
+	ResGroupOps_SetMemoryLimitByValue(group->groupId, memory_limit + memory_inc);
+	ResGroupOps_UnLockGroup(group->groupId, fd);
+	group->memGap += memory_inc;
+}
+/*
+ * Decrease a resource group's cgroup memory limit
+ *
+ * This will take effect immediately for now.
+ */
+static void
+groupApplyCgroupMemDec(ResGroupData *group)
+{
+	int32 memory_limit;
+	int32 memory_dec;
+	int fd;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	Assert(group->memGap > 0);
+	fd = ResGroupOps_LockGroup(group->groupId, "memory", true);
+	memory_limit = ResGroupOps_GetMemoryLimit(group->groupId);
+	Assert(memory_limit > group->memGap);
+	memory_dec = group->memGap;
+	ResGroupOps_SetMemoryLimitByValue(group->groupId, memory_limit - memory_dec);
+	ResGroupOps_UnLockGroup(group->groupId, fd);
+	mempoolRelease(group->groupId, memory_dec);
+	notifyGroupsOnMem(group->groupId);
+	group->memGap -= memory_dec;
+}
+/*
+ * Operation for resource groups with cgroup memory auditor
+ * when reclaiming its memory back to MEM POOL.
+ */
+static void
+groupMemOnDropForCgroup(Oid groupId, ResGroupData *group)
+{
+	int32 memory_expected;
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	memory_expected = groupGetMemExpected(&group->caps);
+	mempoolRelease(groupId, memory_expected + group->memGap);
+}
+/*
+ * Operation for resource groups with cgroup memory auditor
+ * when memory in MEM POOL is increased.
+ */
+static void
+groupMemOnNotifyForCgroup(ResGroupData *group)
+{
+	Assert(LWLockHeldExclusiveByMe(ResGroupLock));
+	if (group->memGap < 0)
+		groupApplyCgroupMemInc(group);
+}
+/*
+ * Operation for resource groups with cgroup memory auditor
+ * when dump memory statistics.
+ */
+static void
+groupMemOnDumpForCgroup(ResGroupData *group, StringInfo str)
+{
+	appendStringInfo(str, "{");
+	appendStringInfo(str, "\"usage\":%d, ",
+			VmemTracker_ConvertVmemChunksToMB(
+				ResGroupOps_GetMemoryUsage(group->groupId) / ResGroupGetSegmentNum()));
+	appendStringInfo(str, "\"limit\":%d",
+			VmemTracker_ConvertVmemChunksToMB(
+				ResGroupOps_GetMemoryLimit(group->groupId) / ResGroupGetSegmentNum()));
+	appendStringInfo(str, "}");
+}
--- a/src/include/catalog/pg_resgroup.h
+++ b/src/include/catalog/pg_resgroup.h
@@ -73,6 +73,7 @@ typedef enum ResGroupLimitType
 	RESGROUP_LIMIT_TYPE_MEMORY,
 	RESGROUP_LIMIT_TYPE_MEMORY_SHARED_QUOTA,
 	RESGROUP_LIMIT_TYPE_MEMORY_SPILL_RATIO,
+	RESGROUP_LIMIT_TYPE_MEMORY_AUDITOR,
 	RESGROUP_LIMIT_TYPE_COUNT,
 } ResGroupLimitType;
@@ -121,6 +122,8 @@ DATA(insert ( 6437, 4, 50, 50 ));
 DATA(insert ( 6437, 5, 20, 20 ));
+DATA(insert ( 6437, 6, 0, 0 ));
 DATA(insert ( 6438, 1, 10, 10 ));
 DATA(insert ( 6438, 2, 10, 10 ));
@@ -131,4 +134,6 @@ DATA(insert ( 6438, 4, 50, 50 ));
 DATA(insert ( 6438, 5, 20, 20 ));
+DATA(insert ( 6438, 6, 0, 0 ));
 #endif   /* PG_RESGROUP_H */
--- a/src/include/commands/resgroupcmds.h
+++ b/src/include/commands/resgroupcmds.h
@@ -19,6 +19,14 @@
 #include "utils/resgroup.h"
 #include "utils/relcache.h"
+typedef enum ResGroupMemAuditorType
+{
+	RESGROUP_MEMORY_AUDITOR_VMTRACKER = 0,
+	RESGROUP_MEMORY_AUDITOR_CGROUP,
+	RESGROUP_MEMORY_AUDITOR_COUNT,
+} ResGroupMemAuditorType;
 extern void CreateResourceGroup(CreateResourceGroupStmt *stmt);
 extern void DropResourceGroup(DropResourceGroupStmt *stmt);
 extern void AlterResourceGroup(AlterResourceGroupStmt *stmt);
@@ -30,4 +38,8 @@ extern Oid GetResGroupIdForRole(Oid roleid);
 extern void GetResGroupCapabilities(Relation rel,
 									Oid groupId,
 									ResGroupCaps *resgroupCaps);
+extern void ResGroupCheckForRole(Oid groupId);
+extern int32 GetResGroupMemAuditorForId(Oid groupId, LOCKMODE lockmode);
 #endif   /* RESGROUPCMDS_H */
--- a/src/include/utils/resgroup-ops.h
+++ b/src/include/utils/resgroup-ops.h
@@ -20,16 +20,21 @@
 */
 extern const char * ResGroupOps_Name(void);
+extern bool ResGroupOps_Probe(void);
 extern void ResGroupOps_Bless(void);
 extern void ResGroupOps_Init(void);
 extern void ResGroupOps_AdjustGUCs(void);
 extern void ResGroupOps_CreateGroup(Oid group);
-extern void ResGroupOps_DestroyGroup(Oid group);
+extern void ResGroupOps_DestroyGroup(Oid group, bool migrate);
 extern void ResGroupOps_AssignGroup(Oid group, int pid);
-extern int ResGroupOps_LockGroup(Oid group, bool block);
+extern int ResGroupOps_LockGroup(Oid group, const char *comp, bool block);
 extern void ResGroupOps_UnLockGroup(Oid group, int fd);
 extern void ResGroupOps_SetCpuRateLimit(Oid group, int cpu_rate_limit);
+extern void ResGroupOps_SetMemoryLimit(Oid group, int memory_limit);
+extern void ResGroupOps_SetMemoryLimitByValue(Oid group, int32 memory_limit);
 extern int64 ResGroupOps_GetCpuUsage(Oid group);
+extern int32 ResGroupOps_GetMemoryUsage(Oid group);
+extern int32 ResGroupOps_GetMemoryLimit(Oid group);
 extern int ResGroupOps_GetCpuCores(void);
 extern int ResGroupOps_GetTotalMemory(void);

--- a/src/include/utils/resgroup.h
+++ b/src/include/utils/resgroup.h
@@ -54,6 +54,7 @@ typedef struct ResGroupCaps
 	ResGroupCap		memLimit;
 	ResGroupCap		memSharedQuota;
 	ResGroupCap		memSpillRatio;
+	ResGroupCap		memAuditor;
 } ResGroupCaps;
 /*
@@ -69,6 +70,12 @@ extern int gp_resource_group_cpu_priority;
 extern double gp_resource_group_cpu_limit;
 extern double gp_resource_group_memory_limit;
+/*
+ * Non-GUC global variables.
+ */
+extern bool gp_resource_group_enable_cgroup_memory;
+extern bool gp_resource_group_enable_cgroup_swap;
 /*
 * Resource Group assignment hook.
 *
@@ -127,7 +134,8 @@ extern void ResGroupDropFinish(Oid groupId, bool isCommit);
 extern void ResGroupCreateOnAbort(Oid groupId);
 extern void ResGroupAlterOnCommit(Oid groupId,
 								  ResGroupLimitType limittype,
-								  const ResGroupCaps *caps);
+								  const ResGroupCaps *caps,
+								  ResGroupCap memLimitGap);
 extern void ResGroupCheckForDrop(Oid groupId, char *name);
 extern int32 ResGroupGetVmemLimitChunks(void);
@@ -141,6 +149,8 @@ extern int64 ResourceGroupGetQueryMemoryLimit(void);
 extern void ResGroupDumpInfo(StringInfo str);
+extern int ResGroupGetSegmentNum(void);
 #define LOG_RESGROUP_DEBUG(...) \
 	do {if (Debug_resource_group) elog(__VA_ARGS__); } while(false);

--- a/src/include/utils/vmem_tracker.h
+++ b/src/include/utils/vmem_tracker.h
@@ -43,6 +43,7 @@ extern int runaway_detector_activation_percent;
 extern int32 VmemTracker_ConvertVmemChunksToMB(int chunks);
 extern int32 VmemTracker_ConvertVmemMBToChunks(int mb);
 extern int64 VmemTracker_ConvertVmemChunksToBytes(int chunks);
+extern int32 VmemTracker_ConvertVmemBytesToChunks(int64 bytes);
 extern int32 VmemTracker_GetReservedVmemChunks(void);
 extern int64 VmemTracker_GetReservedVmemBytes(void);
 extern int64 VmemTracker_GetMaxReservedVmemChunks(void);

--- a/src/test/binary_swap/expected/resgroup_cleanup.out
+++ b/src/test/binary_swap/expected/resgroup_cleanup.out
+-- start_ignore
+drop resource group rg1;
+drop resource group rg2;
+\! gpconfig -c gp_resource_manager -v queue
+20180409:14:08:27:030072 gpconfig:nyu-vm-centos:gpadmin-[INFO]:-completed successfully with parameters '-c gp_resource_manager -v queue'
+-- end_ignore
+\! echo $?
+0
+-- start_ignore
+\! gpstop -rai
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Starting gpstop with args: -rai
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Gathering information and validating the environment...
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Greenplum Master catalog information
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Segment details from master...
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 5.6.1+dev.26.g8353723c5c build dev'
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-There are 1 connections to the database
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode='immediate'
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master host=nyu-vm-centos
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode=immediate
+20180409:14:08:27:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master segment instance directory=/home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:08:28:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Attempting forceful termination of any leftover master process
+20180409:14:08:28:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Terminating processes for segment /home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:08:28:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Stopping master standby host nyu-vm-centos mode=fast
+20180409:14:08:29:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown standby process on nyu-vm-centos
+20180409:14:08:29:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Targeting dbid [2, 5, 3, 6, 4, 7] for shutdown
+20180409:14:08:29:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel primary segment instance shutdown, please wait...
+20180409:14:08:29:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:08:31:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:08:31:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel mirror segment instance shutdown, please wait...
+20180409:14:08:31:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments stopped successfully      = 6
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments with errors during stop   = 0
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown 6 of 6 segment instances 
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Database successfully shutdown with no errors reported
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpmmon process
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpmmon process found
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpsmon processes
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpsmon processes on some hosts. not attempting forceful termination on these hosts
+20180409:14:08:32:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover shared memory
+20180409:14:08:33:030207 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Restarting System...
+-- end_ignore
+\! echo $?
+0
--- a/src/test/binary_swap/expected/resgroup_cleanup_basic.out
+++ b/src/test/binary_swap/expected/resgroup_cleanup_basic.out
+-- start_ignore
+drop resource group rg1;
+drop resource group rg2;
+-- end_ignore
--- a/src/test/binary_swap/expected/resgroup_current_1_group.out
+++ b/src/test/binary_swap/expected/resgroup_current_1_group.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ group
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+ rg1           |      0
+ rg2           |      0
+(4 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+    rsgname    
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/expected/resgroup_current_1_queue.out
+++ b/src/test/binary_swap/expected/resgroup_current_1_queue.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ queue
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+(2 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+(2 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+ rsgname 
+---------
+(0 rows)
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+WARNING:  resource group is disabled
+HINT:  To enable set gp_resource_manager=group
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+WARNING:  resource group is disabled
+HINT:  To enable set gp_resource_manager=group
+create resource group rg2 with (cpu_rate_limit=10, memory_limit=10);
+WARNING:  resource group is disabled
+HINT:  To enable set gp_resource_manager=group
--- a/src/test/binary_swap/expected/resgroup_current_3_group.out
+++ b/src/test/binary_swap/expected/resgroup_current_3_group.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ group
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+ rg1           |      0
+ rg2           |      0
+(4 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+    rsgname    
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/expected/resgroup_current_3_queue.out
+++ b/src/test/binary_swap/expected/resgroup_current_3_queue.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ queue
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+ rg1           |      0
+ rg2           |      0
+(4 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+    rsgname    
+---------
+(0 rows)
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+WARNING:  resource group is disabled
+HINT:  To enable set gp_resource_manager=group
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
--- a/src/test/binary_swap/expected/resgroup_other_2_group.out
+++ b/src/test/binary_swap/expected/resgroup_other_2_group.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ group
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+ rg1           |      0
+ rg2           |      0
+(4 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+    rsgname    
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/expected/resgroup_other_2_queue.out
+++ b/src/test/binary_swap/expected/resgroup_other_2_queue.out
+show gp_resource_manager;
+ gp_resource_manager 
+---------------------
+ queue
+(1 row)
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+    rsgname    | parent 
+---------------+--------
+ default_group |      0
+ admin_group   |      0
+ rg1           |      0
+ rg2           |      0
+(4 rows)
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+ avg 
+-----
+   1
+(1 row)
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+   groupname   
+---------------
+ default_group
+ admin_group
+ rg1
+ rg2
+(4 rows)
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+    rsgname    
+---------
+(0 rows)
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+WARNING:  resource group is disabled
+HINT:  To enable set gp_resource_manager=group
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
--- a/src/test/binary_swap/expected/resgroup_switch_group.out
+++ b/src/test/binary_swap/expected/resgroup_switch_group.out
+-- start_ignore
+\! gpconfig -c gp_resource_manager -v group
+20180409:14:08:40:031018 gpconfig:nyu-vm-centos:gpadmin-[INFO]:-completed successfully with parameters '-c gp_resource_manager -v group'
+-- end_ignore
+\! echo $?
+0
+-- start_ignore
+\! gpstop -rai
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Starting gpstop with args: -rai
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Gathering information and validating the environment...
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Greenplum Master catalog information
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Segment details from master...
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 5.6.1+dev.26.g8353723c5c build dev'
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-There are 1 connections to the database
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode='immediate'
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master host=nyu-vm-centos
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode=immediate
+20180409:14:08:41:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master segment instance directory=/home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:08:42:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Attempting forceful termination of any leftover master process
+20180409:14:08:42:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Terminating processes for segment /home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:08:42:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Stopping master standby host nyu-vm-centos mode=fast
+20180409:14:08:43:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown standby process on nyu-vm-centos
+20180409:14:08:43:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Targeting dbid [2, 5, 3, 6, 4, 7] for shutdown
+20180409:14:08:43:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel primary segment instance shutdown, please wait...
+20180409:14:08:43:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:08:44:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:08:44:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel mirror segment instance shutdown, please wait...
+20180409:14:08:44:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments stopped successfully      = 6
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments with errors during stop   = 0
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown 6 of 6 segment instances 
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Database successfully shutdown with no errors reported
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpmmon process
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpmmon process found
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpsmon processes
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpsmon processes on some hosts. not attempting forceful termination on these hosts
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover shared memory
+20180409:14:08:46:031194 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Restarting System...
+-- end_ignore
+\! echo $?
+0
--- a/src/test/binary_swap/expected/resgroup_switch_queue.out
+++ b/src/test/binary_swap/expected/resgroup_switch_queue.out
+-- start_ignore
+\! gpconfig -c gp_resource_manager -v queue
+20180409:14:57:19:021894 gpconfig:nyu-vm-centos:gpadmin-[INFO]:-completed successfully with parameters '-c gp_resource_manager -v queue'
+-- end_ignore
+\! echo $?
+0
+-- start_ignore
+\! gpstop -rai
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Starting gpstop with args: -rai
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Gathering information and validating the environment...
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Greenplum Master catalog information
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Obtaining Segment details from master...
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 5.6.1+dev.26.g8353723c5c build dev'
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-There are 1 connections to the database
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode='immediate'
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master host=nyu-vm-centos
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing Master instance shutdown with mode=immediate
+20180409:14:57:20:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Master segment instance directory=/home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:57:21:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Attempting forceful termination of any leftover master process
+20180409:14:57:21:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Terminating processes for segment /home/gpadmin/src/gpdb5.git/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
+20180409:14:57:21:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Stopping master standby host nyu-vm-centos mode=fast
+20180409:14:57:22:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown standby process on nyu-vm-centos
+20180409:14:57:22:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Targeting dbid [2, 5, 3, 6, 4, 7] for shutdown
+20180409:14:57:22:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel primary segment instance shutdown, please wait...
+20180409:14:57:22:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:57:24:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:57:24:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Commencing parallel mirror segment instance shutdown, please wait...
+20180409:14:57:24:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-0.00% of jobs completed
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-100.00% of jobs completed
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments stopped successfully      = 6
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-   Segments with errors during stop   = 0
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-----------------------------------------------------
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Successfully shutdown 6 of 6 segment instances 
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Database successfully shutdown with no errors reported
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpmmon process
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpmmon process found
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover gpsmon processes
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-No leftover gpsmon processes on some hosts. not attempting forceful termination on these hosts
+20180409:14:57:25:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Cleaning up leftover shared memory
+20180409:14:57:26:022030 gpstop:nyu-vm-centos:gpadmin-[INFO]:-Restarting System...
+-- end_ignore
+\! echo $?
+0
--- a/src/test/binary_swap/schedule1
+++ b/src/test/binary_swap/schedule1
 test: pg_dumpall_current
 test: gpcheckcat
+test: resgroup_cleanup_basic
+test: resgroup_current_1_queue
--- a/src/test/binary_swap/schedule1_resgroup
+++ b/src/test/binary_swap/schedule1_resgroup
+test: resgroup_cleanup
+test: pg_dumpall_current
+test: gpcheckcat
+test: resgroup_current_1_queue
+test: resgroup_switch_group
+test: resgroup_current_1_group
--- a/src/test/binary_swap/schedule2
+++ b/src/test/binary_swap/schedule2
@@ -3,3 +3,5 @@ test: diff_dumps
 test: inserts
 test: pg_dumpall_other
 test: gpcheckcat
+test: resgroup_other_2_queue
--- a/src/test/binary_swap/schedule2_resgroup
+++ b/src/test/binary_swap/schedule2_resgroup
+test: resgroup_switch_queue
+test: pg_dumpall_other
+test: diff_dumps
+test: pg_dumpall_other
+test: gpcheckcat
+test: resgroup_other_2_queue
+test: resgroup_switch_group
+test: resgroup_other_2_group
--- a/src/test/binary_swap/schedule3
+++ b/src/test/binary_swap/schedule3
@@ -3,3 +3,6 @@ test: diff_dumps
 test: inserts
 test: pg_dumpall_current
 test: gpcheckcat
+test: resgroup_current_3_queue
+test: resgroup_cleanup_basic
--- a/src/test/binary_swap/schedule3_resgroup
+++ b/src/test/binary_swap/schedule3_resgroup
+test: resgroup_switch_queue
+test: pg_dumpall_current
+test: diff_dumps
+test: pg_dumpall_current
+test: gpcheckcat
+test: resgroup_current_3_queue
+test: resgroup_switch_group
+test: resgroup_current_3_group
+test: resgroup_cleanup
--- a/src/test/binary_swap/sql/resgroup_cleanup.sql
+++ b/src/test/binary_swap/sql/resgroup_cleanup.sql
+-- start_ignore
+drop resource group rg1;
+drop resource group rg2;
+\! gpconfig -c gp_resource_manager -v queue
+-- end_ignore
+\! echo $?
+-- start_ignore
+\! gpstop -rai
+-- end_ignore
+\! echo $?
--- a/src/test/binary_swap/sql/resgroup_cleanup_basic.sql
+++ b/src/test/binary_swap/sql/resgroup_cleanup_basic.sql
+-- start_ignore
+drop resource group rg1;
+drop resource group rg2;
+-- end_ignore
--- a/src/test/binary_swap/sql/resgroup_current_1_group.sql
+++ b/src/test/binary_swap/sql/resgroup_current_1_group.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/sql/resgroup_current_1_queue.sql
+++ b/src/test/binary_swap/sql/resgroup_current_1_queue.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+create resource group rg2 with (cpu_rate_limit=10, memory_limit=10);
--- a/src/test/binary_swap/sql/resgroup_current_3_group.sql
+++ b/src/test/binary_swap/sql/resgroup_current_3_group.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/sql/resgroup_current_3_queue.sql
+++ b/src/test/binary_swap/sql/resgroup_current_3_queue.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
--- a/src/test/binary_swap/sql/resgroup_other_2_group.sql
+++ b/src/test/binary_swap/sql/resgroup_other_2_group.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
+-- TODO: change to the resgroup
--- a/src/test/binary_swap/sql/resgroup_other_2_queue.sql
+++ b/src/test/binary_swap/sql/resgroup_other_2_queue.sql
+show gp_resource_manager;
+select * from pg_resgroup
+  where rsgname not like 'rg_dump_test%'
+  order by oid;
+select avg(reslimittype)
+  from pg_resgroupcapability
+  where reslimittype <= 1;
+select groupname from gp_toolkit.gp_resgroup_config
+  where groupname not like 'rg_dump_test%'
+  order by groupid;
+select rsgname from gp_toolkit.gp_resgroup_status
+  where rsgname not like 'rg_dump_test%'
+  order by groupid;
+alter resource group rg1 set cpu_rate_limit 20;
+alter resource group rg1 set cpu_rate_limit 10;
+drop resource group rg1;
+create resource group rg1 with (cpu_rate_limit=10, memory_limit=10);
+alter resource group rg2 set cpu_rate_limit 20;
+alter resource group rg2 set cpu_rate_limit 10;
--- a/src/test/binary_swap/sql/resgroup_switch_group.sql
+++ b/src/test/binary_swap/sql/resgroup_switch_group.sql
+-- start_ignore
+\! gpconfig -c gp_resource_manager -v group
+-- end_ignore
+\! echo $?
+-- start_ignore
+\! gpstop -rai
+-- end_ignore
+\! echo $?
--- a/src/test/binary_swap/sql/resgroup_switch_queue.sql
+++ b/src/test/binary_swap/sql/resgroup_switch_queue.sql
+-- start_ignore
+\! gpconfig -c gp_resource_manager -v queue
+-- end_ignore
+\! echo $?
+-- start_ignore
+\! gpstop -rai
+-- end_ignore
+\! echo $?
--- a/src/test/binary_swap/test_binary_swap.sh
+++ b/src/test/binary_swap/test_binary_swap.sh
@@ -54,10 +54,11 @@ usage()
    echo " -c <dir>   Greenplum install path for current binary to test upgrade/downgrade to (Default: \$GPHOME)"
    echo " -m <dir>   Greenplum Master Data Directory (Default: \$MASTER_DATA_DIRECTORY)"
    echo " -p <port>  Greenplum Master Port (Default: \$PGPORT)"
+    echo " -v <variant> Variant of the test plan (Default: '')"
    exit 0
 }
-while getopts ":c:b:m:p" opt; do
+while getopts ":c:b:m:p:v:" opt; do
    case ${opt} in
        c)
            GPHOME_CURRENT=$OPTARG
@@ -71,6 +72,9 @@ while getopts ":c:b:m:p" opt; do
        p)
            PGPORT_CURRENT=$OPTARG
            ;;
+        v)
+            VARIANT=$OPTARG
+            ;;
        *)
            usage
            ;;
@@ -105,6 +109,13 @@ if [ "${PGPORT_CURRENT}x" == "x" ]; then
    exit 1
 fi
+if ! [ -e schedule1${VARIANT} -a \
+       -e schedule2${VARIANT} -a \
+       -e schedule3${VARIANT} ]; then
+    echo "Use -v to provide a valid variant of the test plan (Default: '')"
+    exit 1
+fi
 ## Grab the Greenplum versions of each binary for display
 CURRENT_VERSION=`$GPHOME_CURRENT/bin/gpstart --version | awk '{ for (i=3; i<NF; i++) printf $i " "; print $NF }'`
 OTHER_VERSION=`$GPHOME_OTHER/bin/gpstart --version | awk '{ for (i=3; i<NF; i++) printf $i " "; print $NF }'`
@@ -115,6 +126,7 @@ echo "Current binaries: ${CURRENT_VERSION}"
 echo "                  ${GPHOME_CURRENT}"
 echo "  Other binaries: ${OTHER_VERSION}"
 echo "                  ${GPHOME_OTHER}"
+echo "         Variant: ${VARIANT}"
 echo "=================================================="
 ## Clean our directory of any previous test output
@@ -122,7 +134,7 @@ clean_output
 ## Start/restart current Greenplum and do initial dump to compare against
 start_binary $GPHOME_CURRENT
-run_tests schedule1
+run_tests schedule1${VARIANT}
 ## Change the binary, dump, and then compare the two dumps generated
 ## by both binaries. Then we do some inserts and dump again. We source
@@ -134,13 +146,13 @@ run_tests schedule1
 ## test.
 start_binary $GPHOME_OTHER
 source $GPHOME_CURRENT/greenplum_path.sh
-run_tests schedule2
+run_tests schedule2${VARIANT}
 ## Change the binary back, dump, and then compare the two new dumps
 ## generated by both binaries. Then we do some inserts and check to see
 ## if dump still works fine.
 start_binary $GPHOME_CURRENT
-run_tests schedule3
+run_tests schedule3${VARIANT}
 ## Print unnecessary success output
 echo "SUCCESS! Provided binaries are swappable."
--- a/src/test/isolation2/expected/resgroup/resgroup_syntax.out
+++ b/src/test/isolation2/expected/resgroup/resgroup_syntax.out
@@ -169,6 +169,12 @@ CREATE RESOURCE GROUP rg_test_group WITH (concurrency=-1, cpu_rate_limit=10, mem
 ERROR:  concurrency range is [0, 'max_connections']
 CREATE RESOURCE GROUP rg_test_group WITH (concurrency=26, cpu_rate_limit=10, memory_limit=10);
 ERROR:  concurrency range is [0, 'max_connections']
+-- negative: memory_auditor should be 'vmtracker' or 'cgroup'
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="randomtext");
+ERROR:  memory_auditor should be "vmtracker" or "cgroup"
+-- negative: concurrency should be zero for cgroup audited resource group
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=1, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
+ERROR:  resource group concurrency must be 0 when group memory_auditor is cgroup
 -- memory_spill_ratio range is [0, 100]
 -- no limit on the sum of memory_shared_quota and memory_spill_ratio
@@ -239,6 +245,11 @@ DROP RESOURCE GROUP rg1_test_group;
 DROP
 DROP RESOURCE GROUP rg2_test_group;
 DROP
+-- positive: concurrency should be zero for cgroup audited resource group
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
+CREATE
+DROP RESOURCE GROUP rg_test_group;
+DROP
 -- memory_spill_ratio range is [0, 100]
 -- no limit on the sum of memory_shared_quota and memory_spill_ratio
@@ -368,3 +379,18 @@ DROP
 DROP RESOURCE GROUP rg2_test_group;
 DROP
+CREATE RESOURCE GROUP cgroup_audited_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
+CREATE
+-- negative: memory_auditor cannot be altered
+ALTER RESOURCE GROUP cgroup_audited_group SET MEMORY_AUDITOR "default";
+ERROR:  syntax error at or near "MEMORY_AUDITOR"
+LINE 1: ALTER RESOURCE GROUP cgroup_audited_group SET MEMORY_AUDITOR...
+                                                      ^
+-- negative: concurrency should be zero for cgroup audited resource group
+ALTER RESOURCE GROUP cgroup_audited_group SET CONCURRENCY 10;
+ERROR:  resource group concurrency must be 0 when group memory_auditor is cgroup
+-- negative: role should not be assigned to a cgroup audited resource group
+CREATE ROLE cgroup_audited_role RESOURCE GROUP cgroup_audited_group;
+ERROR:  You cannot assign a role to this resource group because the memory_auditor property for this group is not the default.
+DROP RESOURCE GROUP cgroup_audited_group;
+DROP
--- a/src/test/isolation2/sql/resgroup/resgroup_syntax.sql
+++ b/src/test/isolation2/sql/resgroup/resgroup_syntax.sql
@@ -90,6 +90,10 @@ CREATE RESOURCE GROUP rg_test_group WITH (cpu_rate_limit=10, memory_limit=1.9);
 -- negative: concurrency should be in [1, max_connections]
 CREATE RESOURCE GROUP rg_test_group WITH (concurrency=-1, cpu_rate_limit=10, memory_limit=10);
 CREATE RESOURCE GROUP rg_test_group WITH (concurrency=26, cpu_rate_limit=10, memory_limit=10);
+-- negative: memory_auditor should be 'vmtracker' or 'cgroup'
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="randomtext");
+-- negative: concurrency should be zero for cgroup audited resource group
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=1, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
 -- memory_spill_ratio range is [0, 100]
 -- no limit on the sum of memory_shared_quota and memory_spill_ratio
@@ -128,6 +132,9 @@ CREATE RESOURCE GROUP rg1_test_group WITH (concurrency=1, cpu_rate_limit=30, mem
 CREATE RESOURCE GROUP rg2_test_group WITH (concurrency=1, cpu_rate_limit=30, memory_limit=30);
 DROP RESOURCE GROUP rg1_test_group;
 DROP RESOURCE GROUP rg2_test_group;
+-- positive: concurrency should be zero for cgroup audited resource group
+CREATE RESOURCE GROUP rg_test_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
+DROP RESOURCE GROUP rg_test_group;
 -- memory_spill_ratio range is [0, 100]
 -- no limit on the sum of memory_shared_quota and memory_spill_ratio
@@ -189,3 +196,11 @@ ALTER RESOURCE GROUP rg2_test_group SET CPU_RATE_LIMIT 30;
 DROP RESOURCE GROUP rg1_test_group;
 DROP RESOURCE GROUP rg2_test_group;
+CREATE RESOURCE GROUP cgroup_audited_group WITH (concurrency=0, cpu_rate_limit=10, memory_limit=10, memory_auditor="cgroup");
+-- negative: memory_auditor cannot be altered
+ALTER RESOURCE GROUP cgroup_audited_group SET MEMORY_AUDITOR "default";
+-- negative: concurrency should be zero for cgroup audited resource group
+ALTER RESOURCE GROUP cgroup_audited_group SET CONCURRENCY 10;
+-- negative: role should not be assigned to a cgroup audited resource group
+CREATE ROLE cgroup_audited_role RESOURCE GROUP cgroup_audited_group;
+DROP RESOURCE GROUP cgroup_audited_group;