[MultiTenant] Added CPU throttling feature

Summary: ported cgroup-based cpu throttling facilities

Test Plan: jdk/test/multi-tenant

Reviewed-by: yuleil, superajun-wsj

Issue: https://github.com/alibaba/dragonwell8/issues/84

make/CompileLaunchers.gmk

@@ -432,6 +432,11 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
       -DJAVA_ARGS='{ "-J-ms8m"$(COMMA) "sun.security.krb5.internal.tools.Ktab"$(COMMA) }'))
 endif
 
+ifeq ($(OPENJDK_TARGET_OS), linux)
+    $(eval $(call SetupLauncher,jgroup, \
+        -DJAVA_ARGS='{ "-J-ms8m"$(COMMA) "com.alibaba.tenant.JGroupMain"$(COMMA) }'))
+endif
+
 ##########################################################################################
 # The order of the object files on the link command line affects the size of the resulting
 # binary (at least on linux) which causes the size to differ between old and new build.

make/CompileNativeLibraries.gmk

@@ -87,6 +87,8 @@ include lib/Awt2dLibraries.gmk
 
 include lib/SoundLibraries.gmk
 
+include lib/JGroupLibraries.gmk
+
 # Include the corresponding custom file, if present. 
 -include $(CUSTOM_MAKE_DIR)/CompileNativeLibraries.gmk
 

make/CopyIntoClasses.gmk

@@ -74,6 +74,12 @@ COPY_FILES += \
 COPY_FILES += \
     $(JDK_TOPDIR)/src/share/classes/sun/net/idn/uidna.spp
 
+# Copy jgroup init script to compilation output dir
+ifeq ($(OPENJDK_TARGET_OS), linux)
+    JGROUP_RESOURCE_DIR = $(JDK_TOPDIR)/src/share/classes/com/alibaba/tenant
+    COPY_FILES += $(wildcard $(JGROUP_RESOURCE_DIR)/JGroupInitializer.sh)
+endif
+
 #
 # Swing plaf resources
 #

make/CreateJars.gmk

@@ -213,7 +213,8 @@ RT_JAR_EXCLUDES += \
     $(LOCALEDATA_INCLUDES) \
     com/oracle/jrockit/jfr \
     oracle/jrockit/jfr \
-    jdk/jfr
+    jdk/jfr \
+    com/alibaba/tenant/JGroupMain.class
 
 # Find all files in the classes dir to use as dependencies. This could be more fine granular.
 ALL_FILES_IN_CLASSES := $(call not-containing, _the., $(filter-out %javac_state, \
@@ -523,7 +524,8 @@ $(eval $(call SetupArchive,BUILD_TOOLS_JAR, , \
         META-INF/services/com.sun.jdi.connect.spi.TransportService \
         META-INF/services/com.sun.tools.attach.spi.AttachProvider \
         META-INF/services/com.sun.tools.internal.ws.wscompile.Plugin \
-        META-INF/services/com.sun.tools.internal.xjc.Plugin, \
+        META-INF/services/com.sun.tools.internal.xjc.Plugin \
+        com/alibaba/tenant/JGroupMain.class, \
     JAR := $(IMAGES_OUTPUTDIR)/lib/tools.jar, \
     SKIP_METAINF := true, \
     CHECK_COMPRESS_JAR := true))

make/Images.gmk

@@ -128,6 +128,10 @@ ifeq ($(PROFILE), )
       jhat$(EXE_SUFFIX) \
       clhsdb$(EXE_SUFFIX) \
       hsdb$(EXE_SUFFIX)
+  # jgroup is not included in JRE build
+  ifeq ($(OPENJDK_TARGET_OS), linux)
+    NOT_JRE_BIN_FILES += jgroup$(EXE_SUFFIX)
+  endif
 endif
 
 WINDOWS_JDK_BIN_FILES = \

make/lib/JGroupLibraries.gmk

+#
+# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Alibaba designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+LIBJGROUP_SRC_DIRS := $(JDK_TOPDIR)/src/linux/native/jgroup/com/alibaba/tenant
+
+LIBJGROUP_CFLAGS := $(foreach dir, $(LIBJGROUP_SRC_DIRS), -I$(dir))
+
+LIBJGROUP_EXCLUDE_FILES :=
+
+$(eval $(call SetupNativeCompilation,BUILD_LIBJGROUP, \
+    LIBRARY := jgroup, \
+    OUTPUT_DIR := $(INSTALL_LIBRARIES_HERE), \
+    SRC := $(LIBJGROUP_SRC_DIRS), \
+    EXCLUDE_FILES := $(LIBJGROUP_EXCLUDE_FILES), \
+    LANG := C, \
+    OPTIMIZATION := HIGH, \
+    CFLAGS := $(CFLAGS_JDKLIB) \
+        $(LIBJGROUP_CFLAGS), \
+    MAPFILE := $(JDK_TOPDIR)/make/mapfiles/libjgroup/mapfile-vers, \
+    LDFLAGS := $(LDFLAGS_JDKLIB) \
+        $(call SET_SHARED_LIBRARY_ORIGIN), \
+    LDFLAGS_SUFFIX_macosx := , \
+    LDFLAGS_SUFFIX_solaris := , \
+    LDFLAGS_SUFFIX_linux := -ljava, \
+    LDFLAGS_SUFFIX_aix := ,\
+    LDFLAGS_SUFFIX_windows := , \
+    OBJECT_DIR := $(JDK_OUTPUTDIR)/objs/libjgroup, \
+    DEBUG_SYMBOLS := $(DEBUG_ALL_BINARIES)))
+
+$(BUILD_LIBJGROUP): $(BUILD_LIBJAVA)
+
+BUILD_LIBRARIES += $(BUILD_LIBJGROUP)
+

make/mapfiles/libjgroup/mapfile-vers

+#
+# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Alibaba designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+SUNWprivate_1.1 {
+    global:
+        Java_com_alibaba_tenant_NativeDispatcher_init0;
+        Java_com_alibaba_tenant_NativeDispatcher_createCgroup;
+        Java_com_alibaba_tenant_NativeDispatcher_moveToCgroup;
+
+    local:
+        *;
+};

src/linux/native/jgroup/com/alibaba/tenant/jgroup.c

+//
+// Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation. Alibaba designates this
+// particular file as subject to the "Classpath" exception as provided
+// by Oracle in the LICENSE file that accompanied this code.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <ctype.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/syscall.h>
+#include <sys/utsname.h>
+#include <mntent.h>
+#include "jni.h"
+#include "jni_util.h"
+#include "com_alibaba_tenant_NativeDispatcher.h"
+
+// enum index of cgroup controllers
+enum CGroupController {
+    CG_CPU,
+    CG_CPUSET,
+    CG_CPUACCT,
+    CG_MAX_SUPPORTED, /* supported by here */
+    /* unsupported controllers */
+    CG_BLKIO,
+    CG_DEVICES,
+    CG_FREEZER,
+    CG_MEMORY, /* supported by -XX:+TenantHeapThrottling */
+    CG_NET_CLS,
+    CG_NET_PRIO,
+    CG_NS,
+    CG_PERF_EVENT
+};
+
+// supported controller names, must be access with index < CG_MAX_SUPPORTED
+static const char* cg_controller_names[] = {
+    "cpu",
+    "cpuset",
+    "cpuacct",
+};
+
+// mountpoints of each cgroup controller, dynamically allocated in initialization,
+// and never reclaimed
+static const char* cg_mount_points[CG_MAX_SUPPORTED] = { NULL };
+
+// Array to indicate if a controller is enabled
+static unsigned char cg_controller_enabled[CG_MAX_SUPPORTED] = { 0 };
+
+// set controller enabling status at startup
+#define ENABLE_CONTROLLER(idx)              \
+    if (idx < CG_MAX_SUPPORTED) {           \
+        cg_controller_enabled[idx] = 1;     \
+    }
+// returns zero if controller disabled; otherwise non-zero
+#define IS_CONTROLLER_ENABLED(idx)      ((idx < CG_MAX_SUPPORTED) ? cg_controller_enabled[idx] : 0)
+
+// To control if extra debugging msg should be printed
+static jboolean jgroup_debug = JNI_FALSE;
+
+// Execute statement only in debugging mode
+#define JGROUP_DEBUG(stmt)  do {                \
+            if (JNI_TRUE == jgroup_debug) {     \
+                stmt;                           \
+            }                                   \
+        } while(0)
+
+// Environment variable name to turn on debugging output of JGroup native code
+#define JGROUP_DEBUG_ENV_KEY    "JGROUP_DEBUG"
+
+// logging support
+#define JGROUP_LOG(stream, format, ...)  do {           \
+            if (jgroup_debug == JNI_TRUE) {             \
+                fprintf(stream, format, __VA_ARGS__);   \
+            }                                           \
+        } while (0)
+
+#define JGROUP_MSG(msg)             JGROUP_LOG(stdout, "[JGROUP][INFO] %s\n", msg)
+#define JGROUP_INFO(format, ...)    JGROUP_LOG(stdout, "[JGROUP][INFO]" format "\n", __VA_ARGS__)
+#define JGROUP_WARN(format, ...)    JGROUP_LOG(stderr, "[JGROUP][WARN]" format "\n", __VA_ARGS__)
+#define JGROUP_ERROR(format, ...)   JGROUP_LOG(stderr, "[JGROUP][ERROR]" format "\n", __VA_ARGS__)
+
+#define SIG_STRING              "Ljava/lang/String;"
+#define MAX_PATH_LEN            2048
+#define CG_MAX_CTRL_NAME_LEN    128
+
+//--------------------- Utility methods ----------------------
+
+// Returns thread ID of current thread
+static pid_t get_tid() {
+    return syscall(SYS_gettid);
+}
+
+// returns JNI_TRUE if a 'path' is a file (not a directory); otherwise JNI_FALSE
+static jboolean
+is_file(const char* path) {
+    if (path != NULL) {
+        struct stat st;
+        memset(&st, 0, sizeof(struct stat));
+        if (0 == stat(path, &st)) {
+            return S_ISREG(st.st_mode) ? JNI_TRUE : JNI_FALSE;
+        }
+    }
+    return JNI_FALSE;
+}
+
+//--------------------- JNI impl ----------------------
+
+// Detect if cgroup is enabled by reading /proc/cgroups
+static int
+detect_cgroup_enabled() {
+    char subsys_name[CG_MAX_CTRL_NAME_LEN];
+    int res = 0, err = 0;
+    int hierarchy = 0, num_cgroups = 0, enabled = 0;
+    const char* proc_cgroups_path = "/proc/cgroups";
+    int cg_index = 0;
+    FILE* proc_cgroups = fopen(proc_cgroups_path, "re");
+
+    if (proc_cgroups != NULL) {
+        while (!feof(proc_cgroups)) {
+            err = fscanf(proc_cgroups, "%s %d %d %d", subsys_name,
+                    &hierarchy, &num_cgroups, &enabled);
+            if (err < 0) {
+                break;
+            }
+
+            for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) {
+                if (!strcmp(subsys_name, cg_controller_names[cg_index]) && enabled) {
+                    ENABLE_CONTROLLER(cg_index);
+                    JGROUP_INFO("cgroup controller %s is enabled!", cg_controller_names[cg_index]);
+                    break;
+                }
+            }
+        }
+        JGROUP_INFO("Opening %s to detect enabled controllers", proc_cgroups_path);
+    } else {
+        JGROUP_ERROR("Failed to open %s", proc_cgroups_path);
+        res = -1;
+    }
+
+    return res;
+}
+
+// Detect cgroup mountpoints
+static int
+detect_cgroup_mount_points() {
+    int res = 0;
+    struct mntent *ent = NULL;
+    char mntent_buffer[4 * MAX_PATH_LEN];
+    int cg_index = 0;
+    FILE* proc_mounts = NULL;
+    struct mntent* temp_ent = NULL;
+
+    temp_ent = (struct mntent *) malloc(sizeof(struct mntent));
+    if (temp_ent == NULL) {
+        JGROUP_MSG("Failed to allocate temp struct mntent");
+        res = -1;
+    } else {
+        proc_mounts = fopen("/proc/mounts", "re");
+        if (proc_mounts != NULL) {
+            while ((ent = getmntent_r(proc_mounts,
+                                      temp_ent,
+                                      mntent_buffer,
+                                      sizeof(mntent_buffer))) != NULL) {
+                // skip non-cgroup mounts
+                if (strcmp(ent->mnt_type, "cgroup")) {
+                    continue;
+                }
+                for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) {
+                    if (NULL != hasmntopt(ent, cg_controller_names[cg_index])) {
+                        cg_mount_points[cg_index] = strdup(ent->mnt_dir);
+                        JGROUP_INFO("found mountpoint for %s at %s",
+                                    cg_controller_names[cg_index],
+                                    cg_mount_points[cg_index]);
+                    }
+                }
+            }
+        } else {
+            JGROUP_MSG("failed to open /proc/mounts");
+            res = -1;
+        }
+
+        free(temp_ent);
+
+        // double check
+        for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) {
+            if (IS_CONTROLLER_ENABLED(cg_index) && cg_mount_points[cg_index] == NULL) {
+                JGROUP_ERROR("Cannot determine mountpoint for controller %s", cg_controller_names[cg_index]);
+                res = -1;
+            }
+        }
+    }
+
+    return res;
+}
+
+// initialize static fields of class 'com.alibaba.tenant.NativeDispatcher'
+// returns 0 on success; non-zero otherwise
+static int
+init_Java_globals(JNIEnv* env, jclass clazz) {
+    int         res                 = 0;
+    char*       mp_field_names[]    = { "CG_MP_CPU", "CG_MP_CPUSET", "CG_MP_CPUACCT" };
+    size_t      idx                 = 0;
+    jfieldID    fid                 = NULL;
+
+    // reflect to Java level
+    // initialize mountpoints
+    for (idx = 0; idx < CG_MAX_SUPPORTED; ++idx) {
+        fid = (*env)->GetStaticFieldID(env, clazz, mp_field_names[idx], SIG_STRING);
+        jstring mp = (*env)->NewStringUTF(env, cg_mount_points[idx]);
+        if (mp != NULL) {
+            (*env)->SetStaticObjectField(env, clazz, fid, mp);
+        }
+    }
+
+    // initialize flags of if a controller is enabled
+    if (IS_CONTROLLER_ENABLED(CG_CPU)) {
+        char buf[MAX_PATH_LEN];
+        // cpu.shares
+        snprintf(buf, MAX_PATH_LEN, "%s/cpu.shares", cg_mount_points[CG_CPU]);
+        if (is_file(buf) == JNI_TRUE) {
+            fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPU_SHARES_ENABLED", "Z");
+            (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE);
+        }
+
+        // cpu.cfs_*_us
+        snprintf(buf, MAX_PATH_LEN, "%s/cpu.cfs_period_us", cg_mount_points[CG_CPU]);
+        if (is_file(buf) == JNI_TRUE) {
+            fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPU_CFS_ENABLED", "Z");
+            (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE);
+        }
+        JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPU], cg_mount_points[CG_CPU]);
+    }
+
+    // cpuset.*
+    if (IS_CONTROLLER_ENABLED(CG_CPUSET)) {
+        fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPUSET_ENABLED", "Z");
+        (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE);
+        JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPUSET], cg_mount_points[CG_CPUSET]);
+    }
+
+    // cpuacct.*
+    if (IS_CONTROLLER_ENABLED(CG_CPUACCT)) {
+        fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPUACCT_ENABLED", "Z");
+        (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE);
+        JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPUACCT], cg_mount_points[CG_CPUACCT]);
+    }
+
+    return res;
+}
+
+// invoked once in static initializer
+JNIEXPORT jint JNICALL
+Java_com_alibaba_tenant_NativeDispatcher_init0(JNIEnv* env, jclass clazz) {
+    jint res = 0;
+
+    // Detect and initialize native configuration
+    if (0 != (res = detect_cgroup_enabled())) {
+        JGROUP_MSG("Failed to detect cgroup status!");
+        return res;
+    }
+
+    if (0 != (res = detect_cgroup_mount_points())) {
+        JGROUP_MSG("Failed to detect cgroup mountpoints!");
+        return res;
+    }
+
+    // detect debugging options from environment variables
+    char* true_str = "TRUE"; // will be compared in uppercase
+    char* env_val  = getenv(JGROUP_DEBUG_ENV_KEY);
+    if (env_val != NULL && strlen(true_str) == strlen(env_val)) {
+        size_t idx = 0;
+        jgroup_debug = JNI_TRUE;
+        for (; idx < strlen(true_str); ++idx) {
+            if (toupper(env_val[idx]) != true_str[idx]) {
+                jgroup_debug = JNI_FALSE;
+                break;
+            }
+        }
+    }
+
+    JGROUP_INFO("jgroup debugging mode = %s", jgroup_debug == JNI_TRUE ? "true" : "false");
+
+    // initialize cgroup
+    if (0 != (res = init_Java_globals(env, clazz))) {
+        JGROUP_MSG("Failed to init Java globals!");
+    } else {
+        JGROUP_MSG("cgroup initialized successfully");
+    }
+
+    return res;
+}
+
+//
+// Create the cgroup dir, and initialize some mandatory values
+// @param group_path   relative path of target CGROUP
+JNIEXPORT jint JNICALL
+Java_com_alibaba_tenant_NativeDispatcher_createCgroup(JNIEnv* env,
+                                                       jobject ignored,
+                                                       jstring group_path) {
+    int res = 0;
+    int cg_index = 0;
+    struct stat st;
+    if (group_path == NULL) {
+        JGROUP_MSG("null group path to create");
+        res = -1;
+    } else {
+        const char* path_str = (*env)->GetStringUTFChars(env, group_path, NULL);
+        if (path_str != NULL || strlen(path_str) > 0) {
+            char path_buf[MAX_PATH_LEN];
+            for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) {
+                snprintf(path_buf, MAX_PATH_LEN, "%s/%s", cg_mount_points[cg_index], path_str);
+
+                // if exists, just reuse
+                if (stat(path_buf, &st) == 0 && S_ISDIR(st.st_mode)) {
+                    JGROUP_INFO("Reuse existing directory %s", path_buf);
+                    continue;
+                }
+
+                res = mkdir(path_buf, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);  // modes are copied from libcg
+                if (res == 0) {
+                    JGROUP_INFO("created cgroup dir %s", path_buf);
+                } else {
+                    JGROUP_ERROR("failed to create dir %s (%s)", path_buf, strerror(errno));
+                    break;
+                }
+            }
+            (*env)->ReleaseStringUTFChars(env, group_path, path_str);
+        } else {
+            JGROUP_MSG("Empty cgroup name");
+            res = -1;
+        }
+    }
+    return res;
+}
+
+JNIEXPORT jint JNICALL
+Java_com_alibaba_tenant_NativeDispatcher_moveToCgroup(JNIEnv* env,
+                                                     jobject ignored,
+                                                     jstring group_path) {
+
+    int res = 0;
+    if (group_path == NULL) {
+        JGROUP_MSG("null group path to create");
+        res = -1;
+    } else {
+        const char* path_str = (*env)->GetStringUTFChars(env, group_path, NULL);
+        if (path_str != NULL && strlen(path_str) > 0) {
+            char path_buf[MAX_PATH_LEN];
+            snprintf(path_buf, MAX_PATH_LEN, "%s/%s/tasks", cg_mount_points[CG_CPU], path_str);
+
+            FILE* fp_tasks = fopen(path_buf, "we");
+            if (fp_tasks != NULL) {
+
+                JGROUP_INFO("Open file %s (we)", path_buf);
+
+                pid_t tid = get_tid();
+                int sz = fprintf(fp_tasks, "%d", (int)tid);
+                fclose(fp_tasks);
+
+                if (sz > 0) {
+                    JGROUP_INFO("PID %d has been writen to %s", (int)tid, path_buf);
+                } else {
+                    JGROUP_ERROR("Failed to write to %s", path_buf);
+                }
+
+                JGROUP_INFO("Close file %s (we)", path_buf);
+            } else {
+                JGROUP_ERROR("Failed to open %s (%s)", path_buf, strerror(errno));
+                res = -1;
+            }
+
+            (*env)->ReleaseStringUTFChars(env, group_path, path_str);
+        } else {
+            JGROUP_MSG("Empty cgroup name");
+            res = -1;
+        }
+    }
+    return res;
+}

src/share/classes/com/alibaba/tenant/JGroup.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import com.alibaba.rcm.Constraint;
+import sun.security.action.GetPropertyAction;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.AccessController;
+import java.util.ArrayList;
+import java.util.List;
+import static com.alibaba.tenant.NativeDispatcher.*;
+
+/**
+ * JGroup is a java mirror of Linux control group
+ */
+class JGroup {
+
+    // to dispatch calls to libcgroup and file system
+    private static NativeDispatcher nd;
+
+    /*
+     * The cgroup hierarchy is:
+     *
+     * OS/Container ROOT group
+     * |
+     * |__ JDK Group
+     *     |
+     *     |__ JVM group of process 1
+     *     |    |___ TenantContainer1's group
+     *     |    |___ TenantContainer2's group
+     *     |    |___ ...
+     *     |
+     *     |__ JVM group of process 2
+     *     |
+     *     |__ ...
+     *
+     * The cgroup path identifying TenantContainer 't0' is in following format
+     * /<cgroup mountpoint>/<controller name>/<optional user specified ROOT path/<JDK path>/<JVM path>/<TenantContainer group path>
+     *
+     * e.g.
+     * /sys/fs/cgroup/cpuset,cpu,cpuacct/ajdk_multi_tenant/12345/t0
+     * /sys/fs/cgroup/cpuset/system.slice/docker-0fdsnjk23njkfnkfnwe/ajdk_multi_tenant/12345/t0
+     *
+     */
+
+    // OS/Container ROOT group, will be appended to each cgroup controller's root path
+    static final String ROOT_GROUP_PATH;
+
+    // JDK group must be initialized before running any JVM process with CPU throttling/accounting enabled
+    static final String JDK_GROUP_PATH;
+
+    // Current JVM process's cgroup path, parent of all non-ROOT TenantContainers' groups
+    static final String JVM_GROUP_PATH;
+
+    // deprecated constants
+    static final int DEFAULT_WEIGHT = 1024;
+
+    static final int DEFAULT_MAXCPU = 100;
+
+    // unique top level groups
+    // rootGroup and jdkGroup are static, just to serve 'parent' API
+    private static JGroup rootGroup;
+    private static JGroup jdkGroup;
+    // jvm group is the top level active cgroup
+    private static JGroup jvmGroup;
+
+    static JGroup jvmGroup() {
+        return jvmGroup;
+    }
+
+    static JGroup jdkGroup() {
+        return jdkGroup;
+    }
+
+    static JGroup rootGroup() {
+        return rootGroup;
+    }
+
+    /*
+     * CGroup path of this cgroup
+     */
+    private String groupPath;
+
+    /*
+     * parent group
+     */
+    private JGroup parentGroup;
+
+    // corresponding TenantResourceContainer object
+    private TenantResourceContainer resourceContainer;
+
+    // Create a JGroup and initialize necessary underlying cgroup configurations
+    JGroup(TenantResourceContainer container) {
+        if (container == null) {
+            throw new IllegalArgumentException("Must provide a non-null TenantContainer to start with");
+        }
+        this.resourceContainer = container;
+        assert resourceContainer.getTenant() != null;
+        init(resourceContainer, getTenantGroupPath(resourceContainer));
+    }
+
+    //
+    // only directly called by initializeJGroupClass to initialize jvmGroup and jdkGroup
+    //
+    private JGroup(TenantResourceContainer container, String path) {
+        init(container, path);
+    }
+
+    private void init(TenantResourceContainer container, String path) {
+        this.groupPath = path;
+        if (container == null) {
+            this.parentGroup = null;
+            if (JVM_GROUP_PATH.equals(path)) {
+                this.parentGroup = jdkGroup();
+                initSystemGroup(this);
+            } else if (JDK_GROUP_PATH.equals(path)) {
+                this.parentGroup = rootGroup();
+                // JDK group has already been initialized by JGroupMain
+            } else if (!ROOT_GROUP_PATH.equals(path)) {
+                System.err.println("Should not call this with path=" + path);
+            }
+        } else {
+            this.parentGroup = container.getParent() == null ? jvmGroup() : container.getParent().getJGroup();
+            initUserGroup(this, container.getConstraints());
+        }
+    }
+
+    //
+    // To generate group path for non-ROOT tenant containers
+    // current pattern:
+    //      "JVM_GROUP_PATH/t<tenant id>"
+    //
+    private String getTenantGroupPath(TenantResourceContainer container) {
+        TenantResourceContainer parent = container.getParent();
+        return (parent == null ? JVM_GROUP_PATH : parent.getJGroup().groupPath)
+                + File.separator + "t" + container.getTenant().getTenantId();
+    }
+
+    // returns the relative cgroup path of this JGroup
+    private String groupPath() {
+        return groupPath;
+    }
+
+    // returns parent JGroup of this one, null if current is the top most JGroup
+    JGroup parent() {
+        return parentGroup;
+    }
+
+    /*
+     * Destory this jgroup
+     */
+    void destory() {
+        Path parentTasks = configPathOf(parent(), CG_TASKS);
+        try {
+            // move all tasks of children group to parent!
+            Files.walk(Paths.get(rootPathOf(CG_CPU), groupPath()), FileVisitOption.FOLLOW_LINKS)
+                    .filter(path -> path.endsWith(CG_TASKS))
+                    .forEach(path -> {
+                        try {
+                            for (String pidLine : Files.readAllLines(path)) {
+                                Files.write(parentTasks, pidLine.getBytes());
+                                debug("destroyCgroup: move " + pidLine + " from " + path
+                                        + " to " + parentTasks);
+                            }
+                        } catch (IOException e) {
+                            e.printStackTrace();
+                        }
+                    });
+
+            // delete all directories
+            Files.walk(Paths.get(rootPathOf(CG_CPU), groupPath()), FileVisitOption.FOLLOW_LINKS)
+                    .map(Path::toFile)
+                    .filter(File::isDirectory)
+                    .forEach(File::delete);
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    // Initialize a cgroup with user specified limits after JVM bootstrapping,
+    // for non-ROOT tenant containers
+    private void initUserGroup(JGroup jgroup, Iterable<Constraint> constraints) {
+        try {
+            initCgroupCommon(jgroup);
+
+            // set value with user specified values
+            // ignoring non-cgroup resources, like ResourceType.MEMORY
+            if (constraints != null) {
+                for (Constraint c : constraints) {
+                    if (c.getResourceType() instanceof TenantResourceType) {
+                        TenantResourceType type = (TenantResourceType)c.getResourceType();
+                        if (type.isJGroupResource()) {
+                            ((JGroupConstraint)c).sync(jgroup);
+                        }
+                    }
+                }
+            }
+        } catch (RuntimeException e) {
+            throw e;
+        } catch (Throwable t) {
+            throw new RuntimeException(t);
+        }
+    }
+
+    // Initialize system cgroups, used by JGroup#initializeJGroupClass
+    // during JVM initialization to create the JDK and JVM group.
+    // NOTE: never throw exception here, will cause VM code to crash, very rude!
+    private void initSystemGroup(JGroup jgroup) {
+        try {
+            initCgroupCommon(jgroup);
+        } catch (Throwable t) {
+            t.printStackTrace();
+            System.exit(127);
+        }
+    }
+
+    // create and initialize underlying cgroups
+    private void initCgroupCommon(JGroup jgroup) {
+        if (jgroup == null || jgroup.groupPath() == null
+                || jgroup.groupPath().isEmpty()) {
+            throw new IllegalArgumentException("Bad argument to createCGroup()");
+        }
+
+        if (nd.createCgroup(jgroup.groupPath()) == 0) {
+            // init two mandatory fields
+            if (IS_CPUSET_ENABLED) {
+                copyValueFromParentGroup(jgroup, CG_CPUSET_CPUS);
+                copyValueFromParentGroup(jgroup, CG_CPUSET_MEMS);
+            }
+            debug("Created group with standard controllers");
+        } else {
+            throw new RuntimeException("Failed to create cgroup at " + jgroup.groupPath());
+        }
+    }
+
+    // Copy config value from parent group
+    private void copyValueFromParentGroup(JGroup jgroup, String configName) {
+        String parentValue = getValue(jgroup.parent(), configName);
+        setValue(jgroup, configName, parentValue);
+        debug("Set group " + jgroup.groupPath()+ "'s config " + configName
+                + " with parent group's value: " + parentValue);
+    }
+
+
+    // Not a frequent operation, using Java implementation
+    private String getValue(JGroup group, String key) {
+        if (group == null || group.groupPath() == null || group.groupPath().isEmpty()
+                || key == null || key.isEmpty() || !key.contains(".")) {
+            return null;
+        }
+        String ctrlName = key.split("\\.")[0];
+        Path configPath = Paths.get(rootPathOf(ctrlName), group.groupPath(), key);
+        if (Files.exists(configPath)) {
+            try {
+                String value = new String(Files.readAllBytes(configPath)).trim();
+                debug(key + "=" + value);
+                return value;
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            debug("getValue: config path " + configPath + " does not exist");
+        }
+        return null;
+    }
+
+    // Not a frequent operation, using Java implementation
+    private void setValue(JGroup group, String key, String value) {
+        if (group == null || group.groupPath() == null || group.groupPath().isEmpty()
+                || key == null || key.isEmpty()
+                || value == null || value.isEmpty()
+                || !key.contains(".")) {
+            throw new IllegalArgumentException("Cannot set " + group.groupPath()
+                    + File.separator + key + "=" + value);
+        }
+        String controllerName = key.split("\\.")[0];
+        Path configPath = Paths.get(rootPathOf(controllerName), group.groupPath(), key);
+
+        if (Files.exists(configPath)) {
+            try {
+                debug("Set value: " + configPath + "=" + value);
+                Files.write(configPath, value.getBytes());
+            } catch (IOException e) {
+                throw new RuntimeException("Failed to set " + group.groupPath()
+                        + File.separator + key + " = " + value, e);
+            }
+        } else {
+            debug("setValue: config path " + configPath + " does not exist");
+        }
+    }
+
+    // used by injected test case
+    static String rootPathOf(String controllerName) {
+        switch (controllerName) {
+            case CG_CPU     : return CG_MP_CPU;
+            case CG_CPUSET  : return CG_MP_CPUSET;
+            case CG_CPUACCT : return CG_MP_CPUACCT;
+            default:
+                throw new IllegalArgumentException("Unsupported controller : " + controllerName);
+        }
+    }
+
+    private static void debug(String... messages) {
+        if (debugJGroup) {
+            System.out.print("[JGroupDispatcher] ");
+            for (String msg : messages) {
+                System.out.print(msg);
+                System.out.print(" ");
+            }
+            System.out.println();
+        }
+    }
+
+    /*
+     * Attach the current thread into this jgroup.
+     * @return 0 if successful
+     */
+    void attach() {
+        if (nd.moveToCgroup(groupPath()) != 0) {
+            throw new IllegalStateException("Cannot attach to group " + this);
+        } else {
+            debug("Attached to cgroup: " + groupPath());
+        }
+    }
+
+    /**
+     * Detach the current thread from this jgroup
+     * @return 0 if successful
+     */
+    void detach() {
+        jvmGroup().attach();
+    }
+
+    /**
+     * Get cpuacct usage of this group
+     * @return cpu usage in nano seconds
+     */
+    long getCpuTime() {
+        try {
+            return Long.parseLong(getValue(this, "cpuacct.usage"));
+        } catch (Exception e) {
+            System.err.println("Exception from JGroup.getCpuTime()");
+            e.printStackTrace();
+        }
+        return -1;
+    }
+
+    /*
+     * Set the value of a cgroup attribute
+     */
+    synchronized void setValue(String key, String name) {
+        setValue(this, key, name);
+    }
+
+    /*
+     * Get the value of a cgroup attribute
+     */
+    synchronized String getValue(String key) {
+        return getValue(this, key);
+    }
+
+    /*
+     * Initialize the JGroup class, called in create_vm.
+     */
+    private static void initializeJGroupClass() {
+        try {
+            // NOTE: below sequence should not be changed!
+            rootGroup = new JGroup(null, ROOT_GROUP_PATH);
+            jdkGroup = new JGroup(null, JDK_GROUP_PATH);
+            jvmGroup = new JGroup(null, JVM_GROUP_PATH);
+
+            // check before attach
+            checkRootGroupPath();
+
+            jvmGroup().attach();
+        } catch (Throwable t) {
+            System.err.println("Failed to initialize JGroup");
+            t.printStackTrace();
+            // do not throw any exception during VM initialization, but just exit
+            System.exit(128);
+        }
+    }
+
+    // Check if mandatory root group paths exist
+    private static void checkRootGroupPath() {
+        List<String> controllers = new ArrayList<>(4);
+        if (TenantGlobals.isCpuThrottlingEnabled()) {
+            controllers.add("cpu");
+            controllers.add("cpuset");
+        }
+        if (TenantGlobals.isCpuAccountingEnabled()) {
+            controllers.add("cpuacct");
+        }
+        for (String ctrl : controllers) {
+            String r = rootPathOf(ctrl);
+            if (r == null || !Files.exists(Paths.get(r, ROOT_GROUP_PATH))) {
+                throw new IllegalArgumentException("Bad ROOT group path: " + r + File.separator + ROOT_GROUP_PATH);
+            }
+        }
+    }
+
+    private static Path configPathOf(JGroup group, String config) {
+        String rootPath = null;
+        try {
+            rootPath = rootPathOf(config.split("\\.")[0]);
+        } catch (IllegalArgumentException e) {
+            // using 'cpu' group root as default
+            rootPath = rootPathOf("cpu");
+        }
+        return Paths.get(rootPath, group.groupPath(), config);
+    }
+
+    /*
+     * Destory the JGroup class, called in destroy_vm.
+     */
+    private static void destroyJGroupClass() {
+        jvmGroup.destory();
+        jvmGroup = null;
+    }
+
+    // --- debugging support ---
+    private static boolean debugJGroup;
+    private static final String DEBUG_JGROUP_PROP = "com.alibaba.tenant.debugJGroup";
+
+    // property name of user specified ROOT cgroup path, relative to each controller's ROOT path
+    private static final String PROP_ROOT_GROUP = "com.alibaba.tenant.jgroup.rootGroup";
+
+    // property name of user specified JDK group path, relative to ROOT_GROUP_PATH
+    private static final String PROP_JDK_GROUP = "com.alibaba.tenant.jgroup.jdkGroup";
+
+    // default value of property PROP_JDK_GROUP
+    private static final String DEFAULT_JDK_GROUP_NAME = "ajdk_multi_tenant";
+
+    // default value of ROOT_GROUP_PATH
+    private static final String DEFAULT_ROOT_GROUP_PATH = "/";
+
+    static {
+        nd = new NativeDispatcher();
+
+        String prop = System.getProperty(PROP_ROOT_GROUP);
+        if (prop == null) {
+            ROOT_GROUP_PATH = DEFAULT_ROOT_GROUP_PATH;
+        } else {
+            ROOT_GROUP_PATH = prop;
+        }
+
+        prop = System.getProperty(PROP_JDK_GROUP);
+        if (prop == null) {
+            JDK_GROUP_PATH = ROOT_GROUP_PATH + File.separator + DEFAULT_JDK_GROUP_NAME;
+        } else {
+            JDK_GROUP_PATH = ROOT_GROUP_PATH + File.separator + prop;
+        }
+
+        JVM_GROUP_PATH = JDK_GROUP_PATH + File.separator + nd.getProcessId();
+
+        debugJGroup = Boolean.parseBoolean(
+                AccessController.doPrivileged(new GetPropertyAction(DEBUG_JGROUP_PROP)));
+    }
+}

src/share/classes/com/alibaba/tenant/JGroupConstraint.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import com.alibaba.rcm.Constraint;
+import com.alibaba.rcm.ResourceType;
+
+/**
+ * Added extra method {@code sync} to flush configuration to cgroup filesystem
+ */
+abstract class JGroupConstraint extends Constraint {
+
+    JGroupConstraint(ResourceType type, long[] values) {
+        super(type, values);
+    }
+
+    /**
+     * Flush the constraint configuration of this constraint to underlying cgroup impl
+     * @param jgroup
+     */
+    abstract void sync(JGroup jgroup);
+}

src/share/classes/com/alibaba/tenant/JGroupInitializer.sh

+#
+# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Alibaba designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+#!/usr/bin/env bash
+
+#
+# To initialize global JGroup (Java wrapper of Linux cgroup) configuration:
+# 1, mount required cgroup controllers
+# 2, create ROOT cgroup 'ajdk_multi_tenant' for JGroup
+# 3, save cgroup configurations to /etc/cgconfig.conf
+#
+
+# globals
+JG_USER=""
+JG_GROUP=""
+JG_ROOT=""
+NOF_CPUS="0" # at least have 1 cpus (which is "0-0" for cpuset.cpus)
+IS_IN_DOCKER_CONTAINER="" # if the script is executed inside docker container, "TRUE" if yes, "" otherwise
+
+# check if all prerequisitions are ready
+check_prerequisitions() {
+    # needed commandline tools
+    REQUIRED_TOOLS=("mountpoint" "mount" "mkdir" "cut" "df" "sed" "getopt" "wc" "awk")
+    for REQ in ${REQUIRED_TOOLS[*]}; do
+        which $REQ > /dev/null 2>&1
+        if [ $? -ne 0 ]; then
+            echo "Cannot find prerequisition : $REQ, please install!"
+            exit 1
+        fi
+    done
+    # init globals
+    NOF_CPUS=$(grep processor /proc/cpuinfo | wc -l)
+    # check if we are in DOCKER
+    if [ -z "$IS_IN_DOCKER_CONTAINER" ]; then
+        if [ ! -z "$(cat /proc/1/cgroup | awk -F/ '{print $2}')" ]; then
+            IS_IN_DOCKER_CONTAINER="TRUE"
+        fi
+    fi
+}
+
+#
+# reference https://github.com/jpetazzo/dind/
+#
+cgroup_init() {
+    CGROUP=""
+    # loop over /proc/mounts, and search for mounted cgroup device
+    while read ln; do
+        # searching for cgroup record, like:
+        # tmpfs /sys/fs/cgroup tmpfs rw,nosuid,nodev,noexec,mode=755 0 0
+        DEV=$(echo $ln | awk '{print $3}')
+        if [ $DEV = "tmpfs" ]; then
+            if [ -n "$(echo $ln | grep cgroup)" ]; then
+                CGROUP="$(echo $ln | awk '{print $2}')"
+            fi
+        fi
+    done < /proc/mounts
+
+    # if did not found cgroup mount point, init cgroup mountpoints
+    if [ -z "$CGROUP" ]; then
+        # use '/cgroup' as DEFAULT mountpoint
+        CGROUP="/cgroup"
+        # use /sys/fs/cgroup for AliOS7
+        uname -r | grep -q alios7 && CGROUP='/sys/fs/cgroup'
+
+        echo "CGroup file system not mounted, trying to mount it to $CGROUP"
+        [ -d "$CGROUP" ] || mkdir $CGROUP
+        mountpoint -q $CGROUP || {
+            echo "Mounting cgroup file system"
+            mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
+                echo "Could not make a tmpfs mount. Did you use --privileged?"
+                exit 1
+            }
+        }
+
+        # Mount the cgroup hierarchies exactly as they are in the parent system.
+        for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+        do
+            [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+            mountpoint -q $CGROUP/$SUBSYS ||
+                    mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+            echo $SUBSYS | grep -q ^name= && {
+              NAME=$(echo $SUBSYS | sed s/^name=//)
+              [ -d $CGROUP/$NAME ] || ln -s $SUBSYS $CGROUP/$NAME
+            }
+
+            [ $SUBSYS = cpuacct,cpu ] && [ ! -d $CGROUP/cpu,cpuacct ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+        done
+    fi
+
+    # check if CGROUP file system is writable
+    # In kernel 3.10.0-327.13.1.el7.x86_64, `ls` and `-w` returns different results for $CGROUP;
+    # but the controller file inside $CGROUP is still writable anyway.
+    if [ ! -w $CGROUP ] && [ ! -w $CGROUP/cpu/cpu.shares ]; then
+        echo "CGroup file system mounted at $CGROUP is not writable, please check CGroup config!"
+        if [ "$IS_IN_DOCKER_CONTAINER" = "TRUE" ]; then
+            echo "It looks like to be inside a docker container, did you forget to add '--privileged' option to 'docker run'?"
+        fi
+        exit 1
+    fi
+
+    [ -d $CGROUP/cpu ] || ln -s cpu,cpuacct $CGROUP/cpu
+    [ -d $CGROUP/cpuacct ] || ln -s cpu,cpuacct $CGROUP/cpuacct
+
+    # create JDK sub-directories for each controller
+    CONTROLLERS=("cpu" "cpuacct" "cpuset")
+    for CTRL in ${CONTROLLERS[*]}; do
+        if [ ! -z "$JG_ROOT" ] && [ ! -d "$CGROUP/$CTRL/$JG_ROOT" ]; then
+            echo "User specified ROOT directory: $CGROUP/$CTRL/$JG_ROOT does not exist"
+            exit 1
+        fi
+        CTRL_DIR="$CGROUP/$CTRL/$JG_ROOT/ajdk_multi_tenant"
+        # create directories if needed
+        if [ ! -d "$CTRL_DIR" ]; then
+            mkdir -p "$CTRL_DIR"
+            if [ $? -ne 0 ]; then
+                echo "Failed to create JDK group: $CTRL_DIR"
+                exit 1
+            fi
+        fi
+        # setting up owners
+        chown -R $JG_USER:$JG_GROUP $CTRL_DIR
+    done
+
+    # initialize necessary controllers
+    cp $CGROUP/cpu/$JG_ROOT/cpu.shares $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.shares
+    cp $CGROUP/cpu/$JG_ROOT/cpu.cfs_period_us $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.cfs_period_us
+    #
+    # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu.html
+    # Setting the value in cpu.cfs_quota_us to -1 indicates that the cgroup does not adhere to any CPU time restrictions.
+    # This is also the default value for every cgroup
+    #
+    # No limit on JDK group
+    echo "-1" > $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.cfs_quota_us
+
+    #
+    # cpuset.cpus and cpuset.mems may be empty, which is normal behavior according to AliOS team,
+    # but in that case tasks/threads cannot be attached to that group.
+    # Here just check and forcefully initialize it with a default value;
+    # see http://aone.alibaba-inc.com/code/D87428 for more details
+    #
+    # 1) Check if parent group (controller root) has a valid config, initialize parent if needed;
+    # 2) Copy parent group's config to /ajdk_multi_tenant
+    #
+    if [ ! -z "$JG_ROOT" ]; then
+        ROOT_CPUSET_CPUS="$CGROUP/cpuset/$JG_ROOT/cpuset.cpus"
+    else
+        ROOT_CPUSET_CPUS="$CGROUP/cpuset/cpuset.cpus"
+    fi
+    if [ -z `cat $ROOT_CPUSET_CPUS` ]; then
+        echo "0-$NOF_CPUS" > $ROOT_CPUSET_CPUS
+        if [ 0 != $? ]; then
+            echo "Cannot init ROOT group's config $ROOT_CPUSET_CPUS"
+            exit 1
+        fi
+    fi
+    cp $ROOT_CPUSET_CPUS $CGROUP/cpuset/$JG_ROOT/ajdk_multi_tenant/cpuset.cpus
+
+    if [ ! -z "$JG_ROOT" ]; then
+        ROOT_CPUSET_MEMS="$CGROUP/cpuset/$JG_ROOT/cpuset.mems"
+    else
+        ROOT_CPUSET_MEMS="$CGROUP/cpuset/cpuset.mems"
+    fi
+    if [ -z `cat $ROOT_CPUSET_MEMS` ]; then
+        echo "0" > $ROOT_CPUSET_MEMS
+        if [ 0 != $? ]; then
+            echo "Cannot init ROOT group's config $ROOT_CPUSET_MEMS"
+            exit 1
+        fi
+    fi
+    cp $ROOT_CPUSET_MEMS $CGROUP/cpuset/$JG_ROOT/ajdk_multi_tenant/cpuset.mems
+
+    # save current CGroup configuration to /etc/cgconfig.conf
+    CONFIG_FILE='/etc/cgconfig.conf'
+    which cgsnapshot
+    if [ $? -eq 0 ]; then
+        cgsnapshot -s -f $CONFIG_FILE
+        if [ $? -ne 0 ]; then
+            echo "Failed to save cgroup configuration to $CONFIG_FILE"
+            exit 1
+        fi
+    fi
+}
+
+# show help information
+show_help() {
+    echo "Usage: jgroup [-u username] [-g usergroup] [-r root group path] [-h]"
+    echo "Initialize the jgroup configuration for multi-tenant JVM (which requires root permission)"
+    echo "e.g. jgroup -u admin -g admin"
+    echo "e.g. jgroup -u admin -g admin -r /system.slice/dockr-abcdefghijklmnopq123/"
+}
+
+# parse arguments
+parse_args() {
+    while getopts "u:g:r:h" arg; do
+        case "$arg"
+        in
+            u)
+                JG_USER="$OPTARG"
+                ;;
+            g)
+                JG_GROUP="$OPTARG"
+                ;;
+            r)
+                JG_ROOT="$OPTARG"
+                ;;
+            h|?)
+                show_help
+                exit 1
+                ;;
+        esac
+    done
+
+    # check result of parsing
+    if [ -z "$JG_GROUP" ] || [ -z "$JG_USER" ]; then
+        echo "Bad parameters: user = $JG_USER, group = $JG_GROUP"
+        show_help
+        exit 1
+    fi
+
+    # check if $JG_USER and $JG_GROUP exists in /etc
+    if [ -z "$(cat /etc/passwd | awk -F: '{print $1}' | grep $JG_USER)" ]; then
+        echo "User name $JG_USER does not exist in /etc/passwd!"
+        exit 1
+    fi
+
+    if [ -z "$(cat /etc/group | awk -F: '{print $1}' | grep $JG_GROUP)" ]; then
+        echo "Group name $JG_GROUP does not exist in /etc/group!"
+        exit 1
+    fi
+}
+
+# entrypoint
+parse_args $*
+check_prerequisitions
+cgroup_init
+
+# done
+exit 0

src/share/classes/com/alibaba/tenant/JGroupMain.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import sun.security.action.GetPropertyAction;
+import javax.tools.ToolProvider;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.AccessController;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Launcher of the jgroup configuration tools
+ * usually invoked during JDK installation.
+ * <p>
+ * NOTE: Privileged user permission (e.g. sudo, su) is needed!
+ */
+public class JGroupMain {
+
+    /*
+     * name of JGroup's initializer resource stored in jdk/tools.jar.
+     * The content of the resource file will be loaded from jdk/tools.jar and writen out to a temp shell script file,
+     * finally the temp shell script will be executed.
+     */
+    private static final String INITIALIZER_NAME = "com/alibaba/tenant/JGroupInitializer.sh";
+
+    /*
+     * Wait {@code TIMEOUT_MILLIS} for child process to end
+     */
+    private static final long TIMEOUT_MILLIS = 60_000;
+
+    private static final String SCRIPT_PREFIX = "jgroupInit_";
+
+    public static void main(String[] args) {
+        try {
+            new JGroupMain().doInitilization(args);
+        } catch (Exception e) {
+            throw (e instanceof RuntimeException ? (RuntimeException)e : new RuntimeException(e));
+        }
+    }
+
+    // Check if current process has permission to init jgroup
+    private void checkPermission() {
+        // check platform
+        String osName = AccessController.doPrivileged(new GetPropertyAction("os.name"));
+        if (!"Linux".equalsIgnoreCase(osName)) {
+            throw new RuntimeException("Only supports Linux platform");
+        }
+
+        // check permission of writing system files
+        File file = new File("/etc");
+        if (!file.canWrite()) {
+            throw new SecurityException("Permission denied! need WRITE permission on system files");
+        }
+    }
+
+    // do initialization work!
+    private void doInitilization(String[] args)
+            throws TimeoutException, IOException {
+        // pre-check
+        checkPermission();
+
+        // create a temporary shell script to do primary initialization work
+        String scriptAbsPath = generateInitScript();
+
+        // execute the script file to initialize jgroup in a child process
+        List<String> arguments = new ArrayList<>(1 + (args == null ? 0 : args.length));
+        arguments.add(scriptAbsPath);
+        for (String arg : args) {
+            arguments.add(arg);
+        }
+        ProcessBuilder pb = new ProcessBuilder(arguments);
+        pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+        pb.redirectError(ProcessBuilder.Redirect.INHERIT);
+        Process process = pb.start();
+        if (process == null) {
+            throw new RuntimeException("Failed to launch initializer process!");
+        }
+
+        // wait with timeout for child process to terminate
+        try {
+            process.waitFor();
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+        if (process.isAlive()) {
+            process.destroyForcibly();
+            throw new TimeoutException("ERROR: Initializer process does not finish in " + TIMEOUT_MILLIS + "ms!");
+        }
+
+        int retValue = process.exitValue();
+        if (retValue != 0) {
+            throw new IllegalStateException("Bad exit value from initialization process: " + retValue);
+        }
+    }
+
+    private String generateInitScript() throws IOException {
+        // load script content
+        ClassLoader loader = ToolProvider.getSystemToolClassLoader(); // JGroupInitializer.sh is in jdk/lib/tools.jar
+        InputStream stream = loader.getResourceAsStream(INITIALIZER_NAME);
+
+        if (stream == null) {
+            throw new IOException("Cannot load " + INITIALIZER_NAME + " using system classloader");
+        }
+
+        File script = File.createTempFile(SCRIPT_PREFIX, ".sh");
+        script.setExecutable(true);
+        script.setWritable(true);
+        script.deleteOnExit();
+
+        // copy content of initializer resource to a temp shell script file
+        OutputStream fos = new FileOutputStream(script);
+        int buf = 0;
+        while ((buf = stream.read()) != -1) {
+            fos.write(buf);
+        }
+        fos.close();
+
+        return script.getAbsolutePath();
+    }
+}

src/share/classes/com/alibaba/tenant/NativeDispatcher.java

@@ -29,6 +29,47 @@ import java.security.AccessController;
  */
 class NativeDispatcher {
 
+    // ------------- CGroup constants and flags -----------
+    // names of cgroup controllers
+    static final String CG_CPU              = "cpu";
+    static final String CG_CPU_SHARES       = "cpu.shares";
+    static final String CG_CPU_CFS_QUOTA    = "cpu.cfs_quota_us";
+    static final String CG_CPU_CFS_PERIOD   = "cpu.cfs_period_us";
+    static final String CG_CPUSET           = "cpuset";
+    static final String CG_CPUSET_CPUS      = "cpuset.cpus";
+    static final String CG_CPUSET_MEMS      = "cpuset.mems";
+    static final String CG_CPUACCT          = "cpuacct";
+    static final String CG_TASKS            = "tasks";
+
+    // flags to indicate if target cgroup controllers has been enabled
+    // will be initialized by native code
+    static boolean IS_CPU_SHARES_ENABLED  = false;
+    static boolean IS_CPU_CFS_ENABLED     = false;
+    static boolean IS_CPUSET_ENABLED      = false;
+    static boolean IS_CPUACCT_ENABLED     = false;
+
+    // mountpoint of each controller, will be initialized by native code
+    static String CG_MP_CPU       = null;
+    static String CG_MP_CPUSET    = null;
+    static String CG_MP_CPUACCT   = null;
+
+    static final boolean CGROUP_INIT_SUCCESS;
+
+    // retrieve process ID of current JVM process
+    long getProcessId() {
+        String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
+        return Long.parseLong(processName.split("@")[0]);
+    }
+
+    // --------- CGroup wrappers --------
+    private static native int init0();
+
+    // create new cgroup path
+    native int createCgroup(String groupPath);
+
+    // move current thread to cgroup at 'groupPath'
+    native int moveToCgroup(String groupPath);
+
     // Attach the current thread to given {@code tenant}
     native void attach(TenantContainer tenant);
 
@@ -59,5 +100,21 @@ class NativeDispatcher {
                 }
             });
         registerNatives0();
+
+        boolean initSuccess = false;
+        if (TenantGlobals.isCpuThrottlingEnabled() || TenantGlobals.isCpuAccountingEnabled()) {
+            AccessController.doPrivileged(
+                new java.security.PrivilegedAction<Void>() {
+                    public Void run() {
+                        System.loadLibrary("jgroup");
+                        return null;
+                    }
+                });
+            initSuccess = (0 == init0());
+            if (!initSuccess) {
+                throw new RuntimeException("JGroup native dispatcher initialization failed!");
+            }
+        }
+        CGROUP_INIT_SUCCESS = initSuccess;
     }
 }

src/share/classes/com/alibaba/tenant/TenantConfiguration.java

@@ -21,16 +21,23 @@
 
 package com.alibaba.tenant;
 
+import com.alibaba.rcm.Constraint;
+import com.alibaba.rcm.ResourceType;
 import sun.misc.SharedSecrets;
 import sun.misc.VM;
 import sun.security.action.GetPropertyAction;
 import java.security.AccessController;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Collection;
 import com.alibaba.rcm.ResourceType;
 import com.alibaba.rcm.Constraint;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+import java.util.stream.Stream;
 import static com.alibaba.rcm.ResourceType.*;
+import static com.alibaba.tenant.TenantResourceType.*;
 
 /**
  *
@@ -49,6 +56,22 @@ public class TenantConfiguration {
     public TenantConfiguration() {
     }
 
+    /**
+     * @param cpuShare
+     * @param maxHeapBytes
+     */
+    public TenantConfiguration(int cpuShare, long maxHeapBytes) {
+        limitCpuShares(cpuShare);
+        limitHeap(maxHeapBytes);
+    }
+
+    /**
+     * @param maxHeapBytes
+     */
+    public TenantConfiguration(long maxHeapBytes) {
+        limitHeap(maxHeapBytes);
+    }
+
     /**
      * Build TenantConfiguration with given constraints
      * @param constraints
@@ -61,15 +84,68 @@ public class TenantConfiguration {
         }
     }
 
-    /*
-     * @return all resource limits specified by this configuration
+    /**
+     * Use cgroup's cpu.cfs_* controller to limit cpu time of
+     * new {@code TenantContainer} object created from this configuration
+     * @param period    corresponding to cpu.cfs_period
+     * @param quota     corresponding to cpu.cfs_quota
+     * @return current {@code TenantConfiguration}
      */
-    Collection<Constraint> getAllConstraints() {
-        return constraints.values();
+    public TenantConfiguration limitCpuCfs(int period, int quota) {
+        if (!TenantGlobals.isCpuThrottlingEnabled()) {
+            throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled");
+        }
+        constraints.put(CPU_CFS, CPU_CFS.newConstraint(period, quota));
+        return this;
     }
 
-    void setConstraint(Constraint constraint) {
-        constraints.put(constraint.getResourceType(), constraint);
+    /**
+     * Use cgroup's cpu.shares controller to limit cpu shares of
+     * new {@code TenantContainer} object created from this configuration
+     * @param share relative weight of cpu shares
+     * @return current {@code TenantConfiguration}
+     */
+    public TenantConfiguration limitCpuShares(int share) {
+        if (!TenantGlobals.isCpuThrottlingEnabled()) {
+            // use warning messages instead of exception here to keep backward compatibility
+            System.err.println("WARNING: -XX:+TenantCpuThrottling is disabled!");
+            //throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled");
+        }
+        constraints.put(CPU_SHARES, CPU_SHARES.newConstraint(share));
+        return this;
+    }
+
+    /**
+     * Use cgroup's cpu.cpuset controller to limit cpu cores allowed to be used
+     * by new {@code TenantContainer} object created from this configuration
+     * @param cpuSets string of cpuset description, such as "1,2,3", "0-7,11"
+     * @return current {@code TenantConfiguration}
+     */
+    public TenantConfiguration limitCpuSet(String cpuSets) {
+        if (!TenantGlobals.isCpuThrottlingEnabled()) {
+            throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled");
+        }
+        constraints.put(CPUSET_CPUS, CPUSET_CPUS.newConstraint(parseCpuSetString(cpuSets)));
+        return this;
+    }
+
+    private long[] parseCpuSetString(String cpuSets) {
+        if (cpuSets == null || cpuSets.isEmpty()) {
+            return null;
+        }
+        return Stream.of(cpuSets.split(","))
+                .flatMapToLong(s -> {
+                    // for core "1-4", expand to 1,2,3,4
+                    if (s.contains("-")) {
+                        List<Long> ranges = Stream.of(s.split("-"))
+                                .map(Long::parseLong)
+                                .collect(Collectors.toList());
+                        return LongStream.range(ranges.get(0), ranges.get(1) + 1);
+                    } else {
+                        return LongStream.of(Long.parseLong(s));
+                    }
+                })
+                .toArray();
     }
 
     /**
@@ -78,10 +154,26 @@ public class TenantConfiguration {
      * @return current {@code TenantConfiguration}
      */
     public TenantConfiguration limitHeap(long maxJavaHeapBytes) {
+        if (!TenantGlobals.isHeapThrottlingEnabled()) {
+            // use warning messages instead of exception here to keep backward compatibility
+            System.err.println("WARNING: -XX:+TenantHeapThrottling not enabled!");
+            //throw new UnsupportedOperationException("-XX:+TenantHeapThrottling is not enabled");
+        }
         constraints.put(HEAP_RETAINED, HEAP_RETAINED.newConstraint(maxJavaHeapBytes));
         return this;
     }
 
+    /*
+     * @return all resource limits specified by this configuration
+     */
+    Collection<Constraint> getAllConstraints() {
+        return constraints.values();
+    }
+
+    void setConstraint(Constraint constraint) {
+        constraints.put(constraint.getResourceType(), constraint);
+    }
+
     /**
      * @return the max amount of heap the tenant is allowed to consume.
      */
@@ -91,4 +183,31 @@ public class TenantConfiguration {
         }
         return Runtime.getRuntime().maxMemory();
     }
+
+    /*
+     * Corresponding to combination of Linux cgroup's cpu.cfs_period_us and cpu.cfs_quota_us
+     * @return the max percent of cpu the tenant is allowed to consume, -1 means unlimited
+     */
+    public int getMaxCpuPercent() {
+        if (constraints.containsKey(CPU_CFS)) {
+            Constraint c = constraints.get(CPU_CFS);
+            int period = (int) c.getValues()[0];
+            int quota = (int) c.getValues()[1];
+            if (period > 0 && quota > 0) {
+                return (int) (((float) quota / (float) period) * 100);
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * Corresponding to Linux cgroup's cpu.shares
+     * @return the weight, impact the ratio of cpu among all tenants.
+     */
+    public int getCpuShares() {
+        if (constraints.containsKey(CPU_SHARES)) {
+            return (int) constraints.get(CPU_SHARES).getValues()[0];
+        }
+        return 0;
+    }
 }

src/share/classes/com/alibaba/tenant/TenantContainer.java

@@ -262,6 +262,7 @@ public class TenantContainer {
             nd.destroyTenantAllocationContext(allocationContext);
         }
 
+        resourceContainer.destroyImpl();
         // clear references
         spawnedThreads.clear();
         attachedThreads.clear();

src/share/classes/com/alibaba/tenant/TenantResourceContainer.java

@@ -49,10 +49,17 @@ class TenantResourceContainer extends AbstractResourceContainer {
                 Constraint c = translate(constraints.remove(ResourceType.CPU_PERCENT));
                 constraints.put(c.getResourceType(), c);
             }
+            jgroup = new JGroup(this);
         }
     }
 
     private static Constraint translate(Constraint constraint) {
+        if (constraint.getResourceType() == ResourceType.CPU_PERCENT) {
+            // translate CPU_PERCENT to cfs_quota_us/cfs_period_us
+            long period = Long.parseLong(JGroup.jvmGroup().getValue(NativeDispatcher.CG_CPU_CFS_PERIOD));
+            long quota = (period * constraint.getValues()[0] / 100L);
+            return TenantResourceType.CPU_CFS.newConstraint(period, quota);
+        }
         return constraint;
     }
 
@@ -69,22 +76,37 @@ class TenantResourceContainer extends AbstractResourceContainer {
      */
     private TenantContainer tenant;
 
+    /*
+     * tenant jgroup
+     */
+    private volatile JGroup jgroup;
+
     TenantResourceContainer getParent() {
         return parent;
     }
 
+    JGroup getJGroup() {
+        return jgroup;
+    }
+
     TenantContainer getTenant() {
         return this.tenant;
     }
 
     @Override
     protected void attach() {
+        if (jgroup != null) {
+            jgroup.attach();
+        }
         super.attach();
     }
 
     @Override
     protected void detach() {
         super.detach();
+        if (jgroup != null) {
+            jgroup.detach();
+        }
     }
 
 
@@ -106,6 +128,13 @@ class TenantResourceContainer extends AbstractResourceContainer {
     @Override
     public void updateConstraint(Constraint constraint) {
         Constraint c = translate(constraint);
+        if (c.getResourceType() instanceof TenantResourceType) {
+            TenantResourceType type = (TenantResourceType)c.getResourceType();
+            if (type.isJGroupResource()
+                    && jgroup != null) {
+                ((JGroupConstraint)c).sync(jgroup);
+            }
+        }
         constraints.put(c.getResourceType(), c);
     }
 
@@ -120,12 +149,19 @@ class TenantResourceContainer extends AbstractResourceContainer {
     }
 
     void destroyImpl() {
+        if (jgroup != null) {
+            jgroup.destory();
+            jgroup = null;
+        }
         parent = null;
         tenant = null;
     }
 
     // exposed to TenantContainer implementation
     long getProcessCpuTime() {
+        if ( jgroup != null) {
+            return jgroup.getCpuTime();
+        }
         return 0;
     }
 }

src/share/classes/com/alibaba/tenant/TenantResourceType.java

@@ -22,12 +22,152 @@
 package com.alibaba.tenant;
 
 import com.alibaba.rcm.ResourceType;
+import com.alibaba.rcm.Constraint;
+import java.util.stream.Collectors;
+import java.util.Arrays;
+import static com.alibaba.tenant.NativeDispatcher.*;
 
 /*
  * Type of resource that can be throttled when MultiTenant feature enabled
  */
 class TenantResourceType extends ResourceType {
+
+    /**
+     * Throttle CPU resource with cgroup controller: cpu.shares.
+     * Expecting one constraint value which represents cpu.shares value.
+     */
+    static final ResourceType CPU_SHARES = new TenantResourceType("CPU_SHARE",true) {
+        @Override
+        protected void validate(long... values) throws IllegalArgumentException {
+            if (values == null || values.length != 1
+                    || values[0] <= 0) {
+                throw new IllegalArgumentException("Bad CPU_SHARE constraint:" + values[0]);
+            }
+        }
+
+        @Override
+        public Constraint newConstraint(long ... values) {
+            validate(values);
+            return new JGroupConstraint(this, values) {
+                @Override
+                void sync(JGroup jgroup) {
+                    if (IS_CPU_SHARES_ENABLED) {
+                        jgroup.setValue(CG_CPU_SHARES, Integer.toString((int) getValues()[0]));
+                    }
+                }
+            };
+        }
+    };
+
+    /**
+     * Throttle CPU resource with cgroup controller: cpuset.cpus.
+     * Expecting N constraint value which represent cores described in cpuset.cpus.
+     * e.g. {@code TenantResourceType.CPUSET_CPUS.newConstraint(LongStream.range(0, 16).toArray()) }
+     */
+    static final ResourceType CPUSET_CPUS = new TenantResourceType("CPUSET_CPUS", true) {
+        @Override
+        protected void validate(long... values) throws IllegalArgumentException {
+            if (values == null || values.length > Runtime.getRuntime().availableProcessors()) {
+                throw new IllegalArgumentException("Invalid number of cpuset.cpus:"
+                        + (values == null ? 0 : values.length));
+            }
+            StringBuilder invalidCores = new StringBuilder();
+            for (long v : values) {
+                if (v < 0 || v >= Runtime.getRuntime().availableProcessors()) {
+                    invalidCores.append(v).append(",");
+                }
+            }
+            String cores = invalidCores.toString();
+            if (!cores.isEmpty()) {
+                throw new IllegalArgumentException("Invalid cpuset cores: " + cores);
+            }
+        }
+
+        @Override
+        public Constraint newConstraint(long... values) {
+            validate(values);
+            return new JGroupConstraint(this, values) {
+                @Override
+                void sync(JGroup jgroup) {
+                    if (IS_CPUSET_ENABLED) {
+                        String cores = Arrays.stream(getValues())
+                                .mapToObj(l -> Long.toString(l))
+                                .collect(Collectors.joining(","));
+                        jgroup.setValue(CG_CPUSET_CPUS, cores);
+                    }
+                }
+            };
+        }
+    };
+
+    /**
+     * Throttle CPU resource with cgroup controller: cpu.cfs_quota_us/cpu.cfs_period_us.
+     * Expecting two constraints:
+     * [0] corresponding to cpu.cfs_period_us
+     * [1] corresponding to cpu.cfs_quota_us
+     */
+    static final ResourceType CPU_CFS = new TenantResourceType("CPU_CFS", true) {
+        @Override
+        protected void validate(long... values) throws IllegalArgumentException {
+            if (values == null || values.length != 2) {
+                throw new IllegalArgumentException("CPU_CFS requires constraints period and quota at the same time");
+            }
+            long period = values[0];
+            long quota = values[1];
+            // according to https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
+            // cpu.cfs_period_us should be in range [1 ms, 1 sec]
+            if (period < 1_000 || period > 1_000_000) {
+                throw new IllegalArgumentException("Invalid cpu_cfs.period value:" + period
+                        + ", expected range [1_000, 1_000_000");
+            }
+            if (quota < 1_000 && quota != -1) {
+                throw new IllegalArgumentException("Invalid cpu_cfs.quota value:" + quota
+                        + ", should >= 1_000 or be unlimited (-1)");
+            }
+        }
+
+        @Override
+        public Constraint newConstraint(long... values) {
+            validate(values);
+            return new JGroupConstraint(this, values) {
+                @Override
+                void sync(JGroup jgroup) {
+                    if (IS_CPU_CFS_ENABLED) {
+                        jgroup.setValue(CG_CPU_CFS_PERIOD, Long.toString(values[0]));
+                        jgroup.setValue(CG_CPU_CFS_QUOTA, Long.toString(values[1]));
+                    }
+                }
+            };
+        }
+    };
+
+    /**
+     * Throttle total number of opened socket file descriptors.
+     * Expecting one constraint value which is the maximum number of socket FDs.
+     */
+    static final ResourceType SOCKET = new TenantResourceType("SOCKET",false) {
+        @Override
+        protected void validate(long... values) throws IllegalArgumentException {
+            if (values == null || values.length != 1
+                    || values[0] <= 0) {
+                throw new IllegalArgumentException("Bad CPU_SHARE constraint:" + values[0]);
+            }
+        }
+    };
+
     private TenantResourceType(String name, boolean isJGroup) {
         super(name);
+        this.isJGroupResource = isJGroup;
+    }
+
+    // if this type of resource is controlled by JGroup
+    private boolean isJGroupResource;
+
+    /**
+     * Check if this type of resource is controlled by cgroup
+     * @return
+     */
+    boolean isJGroupResource() {
+        return this.isJGroupResource;
     }
 }

test/multi-tenant/TestGetAllTenantIDs.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/*
+ * @test
+ * @summary Test data structure integrity while access TenantContainer.tenantContainerMap concurrently
+ * @library /lib/testlibrary
+ * @build TestGetAllTenantIDs
+ * @run main/othervm/timeout=120 -XX:+MultiTenant -Xmx1g  -Xms1g TestGetAllTenantIDs
+ */
+
+import com.alibaba.tenant.TenantConfiguration;
+import com.alibaba.tenant.TenantContainer;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import static jdk.testlibrary.Asserts.fail;
+
+public class TestGetAllTenantIDs {
+    public static void main(String[] args) {
+        int parallel = Runtime.getRuntime().availableProcessors();
+        TenantConfiguration config = new TenantConfiguration();
+        List<TenantContainer> tenants = new LinkedList<>();
+        IntStream.range(0, parallel)
+                .forEach(i -> tenants.add(TenantContainer.create(config)));
+
+        CountDownLatch startTesting = new CountDownLatch(1);
+        AtomicBoolean terminated = new AtomicBoolean(false);
+        Thread[] getterThreads = new Thread[parallel];
+        Thread[] setterThreads = new Thread[parallel];
+        Thread[][] allThreads = { getterThreads, setterThreads };
+        IntStream.range(0, parallel)
+                .forEach(i-> {
+                    getterThreads[i] = new Thread(()->{
+                        try { startTesting.await(); } catch(InterruptedException e) { fail(); }
+                        int len = 0;
+                        while (!terminated.get()) {
+                            len = TenantContainer.getAllTenantIds().size();
+                        }
+                    });
+                    getterThreads[i].start();
+                    setterThreads[i] = new Thread(()->{
+                        try { startTesting.await(); } catch(InterruptedException e) { fail(); }
+                        while (!terminated.get()) {
+                            TenantContainer tenant = TenantContainer.create(config);
+                        }
+                    });
+                    setterThreads[i].start();
+                });
+        startTesting.countDown();
+
+        try {
+            Thread.sleep(15 * 1000);
+        } catch (InterruptedException e) {
+            fail();
+        }
+
+        terminated.set(true);
+
+        for (Thread[] threads : allThreads) {
+            Stream.of(threads)
+                    .forEach(t -> {
+                        try {
+                            t.join();
+                        } catch (Exception e) {
+                            fail();
+                        }
+                    });
+        }
+    }
+}
\ No newline at end of file

test/multi-tenant/TestJGroupDebugMode.sh

+#
+# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Alibaba designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+#!/usr/bin/env bash
+
+#
+# @test TestJGroupDebugMode.sh
+# @summary test debugging mode of JGroup native implementation
+# @run shell/timeout=300 TestJGroupDebugMode.sh
+#
+
+set -x
+
+if [ "${TESTSRC}" = "" ]
+then
+    TESTSRC=${PWD}
+    echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+FS=/
+JAVA=${TESTJAVA}${FS}bin${FS}java
+JAVAC=${TESTJAVA}${FS}bin${FS}javac
+TEST_CLASS="Test"
+TEST_OPTS="-XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+UseG1GC"
+
+# generate and compile Java snippet for testing
+cat >> ${TEST_CLASS}.java << EOF
+import com.alibaba.tenant.*;
+public class ${TEST_CLASS} {
+    public static void main(String[] args) throws TenantException {
+        TenantConfiguration config = new TenantConfiguration()
+            .limitCpuShares(1024);
+        TenantContainer tenant = TenantContainer.create(config);
+        tenant.run(()-> {
+            // empty!
+        });
+    }
+}
+EOF
+
+${JAVAC} -cp . ${TEST_CLASS}.java
+if [ $? != 0 ]; then
+    echo "Failed to compile ${TEST_CLASS}.java"
+    exit 1
+fi
+
+# Test envrionment variable debugging options
+
+unset JGROUP_DEBUG
+
+# disable debugging
+export JGROUP_DEBUG=""
+if [ ! -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then
+    echo "Failed in non-debug mode"
+    exit 1
+fi
+
+# enable debugging
+export JGROUP_DEBUG="TRue"
+if [ -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then
+    echo "Failed in debug mode"
+    exit 1
+fi
+
+export JGROUP_DEBUG="trUe"
+if [ -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then
+    echo "Failed in debug mode"
+    exit 1
+fi
+
+# Test Java property debugging options
+if [ ! -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers:')" ]; then
+    echo "Failed in non-debug mode"
+    exit 1
+fi
+
+if [ ! -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=false -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then
+    echo "Failed in debug mode"
+    exit 1
+fi
+
+# enable debugging
+if [ -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=tRue -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then
+    echo "Failed in debug mode"
+    exit 1
+fi
+
+if [ -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=trUE -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then
+    echo "Failed in debug mode"
+    exit 1
+fi

test/multi-tenant/TestNoJGroupInit.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+import java.lang.reflect.Field;
+
+/* @test
+ * @summary test scenario where JGroup will not be initialized
+ * @compile TestNoJGroupInit.java
+ * @run main/othervm/timeout=300 -XX:+MultiTenant TestNoJGroupInit
+ */
+
+ public class TestNoJGroupInit {
+     public static void main(String[] args) throws Exception {
+         Class jgroupClazz = Class.forName("com.alibaba.tenant.NativeDispatcher");
+         Field f = jgroupClazz.getDeclaredField("CGROUP_INIT_SUCCESS");
+         f.setAccessible(true);
+         Boolean b = (Boolean)f.get(null);
+         if (b) {
+             throw new RuntimeException("Should not be initialized!");
+         }
+     }
+ }
\ No newline at end of file

test/multi-tenant/test/com/alibaba/tenant/TestCpuCfsThrottling.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import jdk.testlibrary.TestUtils;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import static jdk.testlibrary.Asserts.*;
+
+/*
+ * @test
+ * @summary test.com.alibaba.tenant.TestCpuCfsThrottling
+ * @library /lib/testlibrary
+ * @run main/othervm/bootclasspath -Xint -XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+TenantCpuAccounting
+ *                                 -Xmx200m -Xms200m com.alibaba.tenant.TestCpuCfsThrottling
+ */
+
+public class TestCpuCfsThrottling {
+
+    private void testCpuCfsQuotas() {
+        // limit whole JVM process to be running on one CPU core
+        Path jvmCpusetCpusPath = Paths.get(
+                JGroup.rootPathOf("cpuset"),
+                JGroup.JVM_GROUP_PATH,
+                "cpuset.cpus");
+        try {
+            Files.write(jvmCpusetCpusPath, "0".getBytes() /* limit jvm process to CPU core-#0 */);
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail();
+        }
+
+        // create two tenants and start two threads running on same core, and fight for CPU resources
+        int cfsPeriods[] = {1_000_000, 800_000};
+        int cfsQuotas[] = {1_000_000, 300_000};
+                TenantConfiguration configs[] = {
+                        new TenantConfiguration().limitCpuCfs(cfsPeriods[0], cfsQuotas[0]).limitCpuSet("0"),
+                        new TenantConfiguration().limitCpuCfs(cfsPeriods[1], cfsQuotas[1]).limitCpuSet("0")
+        };
+        TenantContainer tenants[] = new TenantContainer[configs.length];
+        for (int i = 0; i < configs.length; ++i) {
+            tenants[i] = TenantContainer.create(configs[i]);
+        }
+        long counters[] = new long[configs.length];
+
+        // verify CGroup configurations
+        for (int i = 0; i < configs.length; ++i) {
+            int actual = Integer.parseInt(getConfig(tenants[i], "cpu.cfs_period_us"));
+            assertEquals(cfsPeriods[i], actual);
+
+            actual = Integer.parseInt(getConfig(tenants[i], "cpu.cfs_quota_us"));
+            assertEquals(cfsQuotas[i], actual);
+        }
+
+        // Start two counter threads in different TenantContainers
+        runSerializedCounters(tenants, counters, 10_000);
+
+        // check results
+        assertGreaterThan(counters[0], 0L);
+        assertGreaterThan(counters[1], 0L);
+        assertGreaterThan(counters[0], counters[1]);
+        assertGreaterThan(counters[0], counters[1] * 2);
+    }
+
+    private void testAdjustCpuCfsQuotas() {
+        // limit whole JVM process to be running on one CPU core
+        Path jvmCpusetCpusPath = Paths.get(
+                JGroup.rootPathOf("cpuset"),
+                JGroup.JVM_GROUP_PATH,
+                "cpuset.cpus");
+        try {
+            Files.write(jvmCpusetCpusPath, "0".getBytes() /* limit jvm process to CPU core-#0 */);
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail();
+        }
+
+        // create two tenants and start two threads running on same core, and fight for CPU resources
+        TenantConfiguration configs[] = {
+                new TenantConfiguration().limitCpuCfs(1_000_000, 800_000).limitCpuSet("0"),
+                new TenantConfiguration().limitCpuCfs(1_000_000, 300_000).limitCpuSet("0")
+        };
+        TenantContainer tenants[] = new TenantContainer[configs.length];
+        for (int i = 0; i < configs.length; ++i) {
+            tenants[i] = TenantContainer.create(configs[i]);
+        }
+        long counters[] = new long[configs.length];
+
+        // Start two counter threads in different TenantContainers
+        runSerializedCounters(tenants, counters, 10_000);
+
+        // check results
+        assertGreaterThan(counters[0], 0L);
+        assertGreaterThan(counters[1], 0L);
+        assertGreaterThan(counters[0], counters[1]);
+        assertGreaterThan(counters[0], counters[1] * 2);
+
+        // cfs limitation adjustment needs time to take effect, here we sleep for a while!
+        try {
+            Thread.sleep(10_000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            fail();
+        }
+
+        // ======== round 2 testing, after modify resouce limitations ==========
+        tenants[0].update(configs[0].limitCpuCfs(1_000_000, 300_000));
+        tenants[1].update(configs[1].limitCpuCfs(1_000_000, 800_000));
+        counters[0] = 0L;
+        counters[1] = 0L;
+        runSerializedCounters(tenants, counters, 10_000);
+        // check results
+        assertGreaterThan(counters[0], 0L);
+        assertGreaterThan(counters[1], 0L);
+        assertGreaterThan(counters[1], counters[0]);
+        assertGreaterThan(counters[1], counters[0] * 2);
+    }
+
+    // run several counters in concurrent threads
+    private static void runConcurrentCounters(TenantContainer[] tenants,
+                                    long[] counters,
+                                    long milliLimit) {
+        if (tenants == null || counters == null) {
+            throw new IllegalArgumentException("Bad args");
+        }
+        // Start two counter threads in different TenantContainers
+        CountDownLatch startCounting = new CountDownLatch(1);
+        AtomicBoolean stopCounting = new AtomicBoolean(false);
+        Thread threadRefs[] = new Thread[2];
+        for (int i = 0; i < tenants.length; ++i) {
+            TenantContainer tenant = tenants[i];
+            int idx = i;
+            try {
+                // start a single non-root tenant thread to execute counter
+                tenant.run(()-> {
+                    threadRefs[idx] = new Thread(()->{
+                        try {
+                            startCounting.await();
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                            fail();
+                        }
+                        while (!stopCounting.get()) {
+                            ++counters[idx];
+                        }
+                    });
+                    threadRefs[idx].start();
+                });
+            } catch (TenantException e) {
+                e.printStackTrace();
+                fail();
+            }
+        }
+
+        // ROOT tenant thread will serve as monitor & controller
+        startCounting.countDown();
+        try {
+            // the absolute exec time limits for two tenant threads are same
+            Thread.sleep(milliLimit);
+            stopCounting.set(true);
+            for (Thread t : threadRefs) {
+                t.join();
+            }
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    // execute an action after certain milli seconds, clock start ticking after notified by startLatch
+    private static void runTimedTask(long afterMillis, CountDownLatch startLatch, Runnable action) {
+        Thread timer = new Thread(() -> {
+            if (startLatch != null) {
+                try {
+                    startLatch.await();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+
+            long start = System.currentTimeMillis();
+            long passed = 0;
+            while (passed < afterMillis) {
+                try {
+                    Thread.sleep(afterMillis - passed);
+                } catch (InterruptedException e) {
+                    //ignore
+                } finally {
+                    passed = System.currentTimeMillis() - start;
+                }
+            }
+            // performan action after specific time
+            action.run();
+        });
+        timer.start();
+    }
+
+    private static void runSerializedCounters(TenantContainer[] tenants,
+                                    long[] counters,
+                                    long milliLimit) {
+        if (tenants == null || counters == null) {
+            throw new IllegalArgumentException("Bad args");
+        }
+
+        AtomicBoolean stopCounting = new AtomicBoolean(false);
+
+        for (int i = 0; i < tenants.length; ++i) {
+            int idx = i;
+            stopCounting.set(false);
+            CountDownLatch startCounting = new CountDownLatch(1);
+            TenantContainer tenant = tenants[i];
+            runTimedTask(milliLimit, startCounting, ()-> {
+                stopCounting.set(true);
+            });
+            try {
+                // start a single non-root tenant thread to execute counter
+                tenant.run(() -> {
+                    startCounting.countDown();
+                    while (!stopCounting.get()) {
+                        ++counters[idx];
+                    }
+                });
+            } catch (TenantException e) {
+                e.printStackTrace();
+                fail();
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        TestUtils.runWithPrefix("test",
+                TestCpuCfsThrottling.class,
+                new TestCpuCfsThrottling());
+    }
+
+    //------ utililties
+    private String getConfig(TenantContainer tenant, String configName) {
+        String controllerName = configName.split("\\.")[0];
+        Path configPath = Paths.get(JGroup.rootPathOf(controllerName),
+                JGroup.JVM_GROUP_PATH,
+                "t" + tenant.getTenantId(),
+                configName);
+        try {
+            return new String(Files.readAllBytes(configPath)).trim();
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail();
+        }
+        return null;
+    }
+}

test/multi-tenant/test/com/alibaba/tenant/TestHierachicalTenants.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import jdk.testlibrary.TestUtils;
+import java.util.concurrent.CountDownLatch;
+import static jdk.testlibrary.Asserts.*;
+
+/*
+ * @test
+ * @summary Test hierarchical tenants support
+ * @library /lib/testlibrary
+ * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuThrottling -Xmx200m -Xms200m
+ *                                 com.alibaba.tenant.TestHierachicalTenants
+ */
+
+public class TestHierachicalTenants {
+
+    private void testParentLimit() {
+        TenantConfiguration config = new TenantConfiguration()
+                .limitCpuSet("0")  // forcefully bind to cpu-0
+                .limitCpuCfs(200000, 100000);
+        TenantContainer parent = TenantContainer.create(config);
+
+        long timeLimitMillis = 5_000;
+        long[] counts = new long[5];
+        try {
+            // counts[counts.length - 1] is the baseline number
+            parent.run(()->{
+                // the primary counter loop-1
+                long startTime = System.currentTimeMillis();
+                while (System.currentTimeMillis() - startTime < timeLimitMillis) {
+                    ++counts[counts.length - 1];
+                }
+            });
+
+            // create two children tenant containers, the overall limit should not exceed parent limit
+            CountDownLatch start = new CountDownLatch(1);
+            TenantContainer[] childrenTenants = new TenantContainer[counts.length - 1];
+            Thread[] threads = new Thread[childrenTenants.length];
+
+            for (int i = 0; i < childrenTenants.length; ++i) {
+                final int idx = i;
+                parent.run(()->{
+                    childrenTenants[idx] = TenantContainer.create(config);
+                });
+                childrenTenants[i].run(()->{
+                    threads[idx] = new Thread(()->{
+                        try {
+                            start.await();
+                        } catch (InterruptedException e) {
+                            e.printStackTrace();
+                            fail();
+                        }
+                        // below counter loop must be identical to parent's primary counter loop-1
+                        long startTime = System.currentTimeMillis();
+                        while (System.currentTimeMillis() - startTime < timeLimitMillis) {
+                            ++counts[idx];
+                        }
+                    });
+                    threads[idx].start();
+                });
+            }
+
+            // trigger children tester thread
+            start.countDown();
+
+            // join all children
+            for (Thread t : threads) {
+                try {
+                    t.join();
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                    fail();
+                }
+            }
+
+            System.out.println("counts:");
+            for (long cnt : counts) { System.out.println(cnt); }
+
+            // check result
+            long sum = 0;
+            double acceptableDiffPercent = 0.2;
+            for (int i = 0; i < counts.length - 1; ++i) {
+                assertLessThan(counts[i], counts[counts.length - 1]);
+                sum += counts[i];
+            }
+
+            assertLessThan(Math.abs(sum - (double)counts[counts.length - 1]), counts[counts.length - 1] * acceptableDiffPercent);
+
+        } catch (TenantException e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    private void testParentBasic() {
+        TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024);
+        TenantContainer parent = TenantContainer.create(config);
+        assertNull(getParent(parent));
+        TenantContainer children[] = new TenantContainer[1];
+        try {
+            parent.run(()->{
+                children[0] = TenantContainer.create(config);
+            });
+        } catch (TenantException e) {
+            e.printStackTrace();
+            fail();
+        }
+        assertTrue(getParent(children[0]) == parent);
+        children[0].destroy();
+        assertNull(getParent(children[0]));
+        assertNull(getParent(parent));
+        parent.destroy();
+    }
+
+    private void testNullParent() {
+        TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024);
+        TenantContainer tenant = TenantContainer.create(config);
+        assertNull(getParent(tenant));
+        tenant.destroy();
+    }
+
+    public static void main(String[] args) {
+        TestUtils.runWithPrefix("test",
+                TestHierachicalTenants.class,
+                new TestHierachicalTenants());
+    }
+
+    // -------------- utilities ---------------
+    private static TenantContainer getParent(TenantContainer config) {
+        if (config != null) {
+            TenantResourceContainer parentRes = config.resourceContainer.getParent();
+            if (parentRes != null) {
+                return parentRes.getTenant();
+            }
+        }
+        return null;
+    }
+}

test/multi-tenant/test/com/alibaba/tenant/TestJGroup.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+package com.alibaba.tenant;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/*
+ * @test
+ * @summary Test JGroup
+ * @library /lib/testlibrary
+ * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuAccounting -XX:+TenantCpuThrottling -XX:+UseG1GC -Xmx200m -Xms200m -Dcom.alibaba.tenant.DebugJGroup=true com.alibaba.tenant.TestJGroup
+ */
+
+class JGroupWorker implements Runnable{
+    private long id;
+    public long count = 0;
+    public JGroup group;
+
+    JGroupWorker(long id){
+        this.id = id;
+    }
+    public void run(){
+        TenantContainer tenant = TenantContainer.create(
+                new TenantConfiguration()
+                        .limitCpuShares((int)id * 512 + 512)
+                        .limitCpuSet("0")); // limit all tasks to one CPU to forcefully create contention
+        group = new JGroup(tenant.resourceContainer);
+        group.attach();
+        long msPre = System.currentTimeMillis();
+        while(System.currentTimeMillis() - msPre < (20000 - 1000 * id)){
+            count++;
+        }
+    }
+}
+
+class JGroupsWorker implements Runnable{
+    private JGroup t1;
+    private JGroup t2;
+    private JGroup t3;
+    public long count1 = 0;
+    public long count2 = 0;
+    public long count3 = 0;
+
+    JGroupsWorker(JGroup t1, JGroup t2, JGroup t3){
+        this.t1 = t1;
+        this.t2 = t2;
+        this.t3 = t3;
+    }
+    public void run(){
+        t1.attach();
+        long msPre = System.currentTimeMillis();
+        while(System.currentTimeMillis() - msPre < 5000){
+            count1++;
+        }
+        t1.detach();
+
+        t2.attach();
+        msPre = System.currentTimeMillis();
+        while(System.currentTimeMillis() - msPre < 4000){
+            count2++;
+        }
+        t2.detach();
+
+        t3.attach();
+        msPre = System.currentTimeMillis();
+        while(System.currentTimeMillis() - msPre < 3000){
+            count3++;
+        }
+        t3.detach();
+    }
+}
+
+
+public class TestJGroup {
+
+    private static void setUp() {
+        JGroup.jvmGroup().setValue("cpuset.cpus", "0");
+    }
+
+    static void mySleep(int t) {
+        try {
+            Thread.sleep(t);
+        } catch (InterruptedException e) {
+            System.out.println("Interreupted...");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        setUp();
+
+        long usage1 = 0;
+        long usage2 = 0;
+        long usage3 = 0;
+        JGroupWorker jw1 = new JGroupWorker(0);
+        JGroupWorker jw2 = new JGroupWorker(1);
+        JGroupWorker jw3 = new JGroupWorker(2);
+        Thread t1 = new Thread(jw1);
+        Thread t2 = new Thread(jw2);
+        Thread t3 = new Thread(jw3);
+        t1.start();
+        t2.start();
+        t3.start();
+
+        mySleep(1000);
+
+        JGroupsWorker js = new JGroupsWorker(jw1.group,
+                                       jw2.group,
+                                       jw3.group);
+        Thread t4 =new Thread(js);
+        t4.start();
+        try {
+            t4.join();
+        } catch (InterruptedException e) {
+            System.out.println("Interreupted...");
+        }
+
+        try {
+            t1.join();
+        } catch (InterruptedException e) {
+            System.out.println("Interreupted...");
+        }
+        usage1 = jw1.group.getCpuTime();
+        jw1.group.destory();
+
+        if(usage1 <= 0 ) {
+            throw new Exception("Invalid cpu usage, usage1: "+ usage1);
+        }
+        try {
+            t2.join();
+        } catch (InterruptedException e) {
+            System.out.println("Interreupted...");
+        }
+        usage2 = jw2.group.getCpuTime();
+        jw2.group.destory();
+
+        if(usage2 <= 0 ) {
+            throw new Exception("Invalid cpu usage, usage2: "+ usage2);
+        }
+        try {
+            t3.join();
+        } catch (InterruptedException e) {
+            System.out.println("Interreupted...");
+        }
+        usage3 = jw3.group.getCpuTime();
+        jw3.group.destory();
+
+        if(usage3 <= 0 ) {
+            throw new Exception("Invalid cpu usage, usage3: "+ usage3);
+        }
+
+        if ((js.count3 < js.count2) || (js.count2 < js.count1)) {
+            throw new Exception("com.alibaba.tenant.JGroupsWorker test failed!"
+                                + " count1 = " + js.count1
+                                + " count2 = " + js.count2
+                                + " count3 = " + js.count3);
+        }
+
+        if ((jw3.count < jw2.count) || (jw2.count < jw1.count)) {
+            throw new Exception("Three com.alibaba.tenant.JGroupWorker test failed!"
+                                + " jw1 count = " + jw1.count
+                                + " jw2 count = " + jw2.count
+                                + " jw3 count = " + jw3.count);
+        }
+
+        if ((usage3 < usage2) || (usage2 < usage1)) {
+            throw new Exception("Three com.alibaba.tenant.JGroupWorker test failed!"
+                                + " jw1 usage = " + usage1
+                                + " jw2 usage = " + usage2
+                                + " jw3 usage = " + usage3);
+        }
+    }
+}

test/multi-tenant/test/com/alibaba/tenant/TestJGroupInit.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+package com.alibaba.tenant;
+
+import jdk.testlibrary.TestUtils;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.StringTokenizer;
+import static jdk.testlibrary.Asserts.*;
+
+/*
+ * @test
+ * @summary Test initialization code of JGroup
+ * @library /lib/testlibrary
+ * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+UseG1GC
+ *                                 -Dcom.alibaba.tenant.DebugJGroup=true com.alibaba.tenant.TestJGroupInit
+ */
+public class TestJGroupInit {
+
+    private void testJGroupHierarchy() {
+        TenantConfiguration config = new TenantConfiguration()
+                .limitCpuShares(1024)
+                .limitHeap(64 * 1024 * 1024)
+                .limitCpuCfs(1_000_000, 500_000);
+        TenantContainer tenant = TenantContainer.create(config);
+        try {
+            Field f = TenantResourceContainer.class.getDeclaredField("jgroup");
+            f.setAccessible(true);
+            JGroup jgroup = (JGroup)f.get((TenantResourceContainer)(tenant.getResourceContainer()));
+
+            assertNotNull(jgroup); // tenant group
+            assertNotNull(jgroup.parent());    // jvm group
+            assertNotNull(jgroup.parent().parent());  // jdk group
+            assertNotNull(jgroup.parent().parent().parent());    // root group
+            assertNull(jgroup.parent().parent().parent().parent());  // null
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void testRootGroupExists() {
+        Path rootGroupPath = getRootCGroupPathOf("cpu");
+        System.err.println("rootpath=" + rootGroupPath);
+        assertTrue(Files.exists(rootGroupPath) && Files.isDirectory(rootGroupPath),
+                rootGroupPath + " cannot be found or is not a directory");
+        assertTrue(Files.exists(rootGroupPath.resolve("cpu.shares")), "cpu.shares cannot be found");
+    }
+
+    private void testNonRootGroupExists() {
+        Path rootGroupPath = getRootCGroupPathOf("cpu");
+        System.err.println("rootpath=" + rootGroupPath);
+        assertTrue(Files.exists(rootGroupPath) && Files.isDirectory(rootGroupPath));
+
+        int cpuShare = 1024;
+        long heapLimit = 128 * 1024 * 1024;
+        int cpuCfsPeriod = 1_000_000;
+        int cpuCfsQuota = 500_000;
+
+        TenantConfiguration config = new TenantConfiguration()
+                .limitCpuShares(cpuShare)
+                .limitHeap(heapLimit)
+                .limitCpuCfs(cpuCfsPeriod, cpuCfsQuota);
+        TenantContainer tenant = TenantContainer.create(config);
+        Path tenantPath = rootGroupPath.resolve("t" + tenant.getTenantId());
+        assertTrue(Files.exists(tenantPath) && Files.isDirectory(tenantPath));
+
+        // verify CGroup configurations
+        {
+            int actual = Integer.parseInt(getConfig(tenant, "cpu.cfs_period_us"));
+            assertEquals(cpuCfsPeriod, actual);
+
+            actual = Integer.parseInt(getConfig(tenant, "cpu.cfs_quota_us"));
+            assertEquals(cpuCfsQuota, actual);
+
+            actual = Integer.parseInt(getConfig(tenant, "cpu.shares"));
+            assertEquals(cpuShare, actual);
+        }
+
+        assertTrue(Files.exists(tenantPath) && Files.isDirectory(tenantPath),
+                tenantPath + " cannot be found or is not a directory");
+        Path path = tenantPath.resolve("cpu.shares");
+        assertTrue(Files.exists(path), path + " cannot be found");
+
+        try {
+            tenant.run(()->{
+                System.out.println("Hello from simple tenant");
+            });
+        } catch (TenantException e) {
+            e.printStackTrace();
+            fail();
+        } finally {
+            tenant.destroy();
+        }
+
+        assertTrue(!Files.exists(tenantPath));
+        assertTrue(!Files.exists(tenantPath.resolve("cpu.shares")));
+    }
+
+    private void testGetSetNonExistConfigs() {
+        TenantConfiguration config = new TenantConfiguration()
+                .limitCpuShares(1024)
+                .limitHeap(128 * 1024 * 1024);
+        TenantContainer tenant = TenantContainer.create(config);
+        JGroup jgroup = tenant.resourceContainer.getJGroup();
+
+        // controller name does not have separator "."
+        try {
+            jgroup.setValue("NonExistConfigName", "456");
+        } catch (IllegalArgumentException e) {
+            // ignore
+        }
+
+        // controller name does have separator ".", and looks like normal
+        try {
+            jgroup.setValue("NonExist.controller", "456");
+            fail();
+        } catch (Throwable e) {
+            // ignore
+        }
+
+        // controller name does not have separator "."
+        try {
+            String val = jgroup.getValue("NonExistConfigfsdjkfksdj123rName");
+        } catch (IllegalArgumentException e) {
+            // ignore
+        }
+
+        // controller name does have separator ".", and looks like normal
+        try {
+            String val = jgroup.getValue("NonExistConf.ksdj123rName");
+            fail();
+        } catch (Throwable e) {
+        }
+
+        jgroup.setValue("cpu.shares", "1023");
+        assertEquals("1023", jgroup.getValue("cpu.shares"));
+    }
+
+    private void testGetSetJVMGroupControllers() {
+        assertNotNull(JGroup.jdkGroup());
+        assertNotNull(JGroup.jvmGroup());
+
+        TenantConfiguration config = new TenantConfiguration();
+        TenantContainer tenant = TenantContainer.create(config);
+        JGroup group = new JGroup(tenant.resourceContainer);
+        assertTrue(group.parent() == JGroup.jvmGroup());
+
+        JGroup jvmGroup = JGroup.jvmGroup();
+        jvmGroup.setValue("cpu.shares", "1023");
+        assertEquals(jvmGroup.getValue("cpu.shares"), "1023");
+        jvmGroup.setValue("cpu.shares", "1024");
+        assertEquals(jvmGroup.getValue("cpu.shares"), "1024");
+    }
+
+    public static void main(String[] args) {
+        TestUtils.runWithPrefix("test",
+                TestJGroupInit.class,
+                new TestJGroupInit());
+    }
+
+    //---------utilities
+    private static volatile Map<String, Path> rootCGroupPathMap;
+
+    private static Path getRootCGroupPathOf(String controllerName) {
+        if (rootCGroupPathMap == null) {
+            synchronized (TestJGroupInit.class) {
+                if (rootCGroupPathMap == null) {
+                    rootCGroupPathMap = new HashMap<>();
+                    String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
+                    assertNotNull(processName);
+                    long pid = Long.parseLong(processName.split("@")[0]);
+                    assertGreaterThan(pid, 0L);
+
+                    try {
+                        Files.readAllLines(Paths.get("/proc/mounts"))
+                                .stream().forEach(line -> {
+                            if (line.startsWith("cgroup ")) {
+                                StringTokenizer st = new StringTokenizer(line);
+                                st.nextToken();
+                                String tok = st.nextToken();
+                                assertNotNull(tok);
+                                Path curPath = Paths.get(tok, "ajdk_multi_tenant", "" + pid);
+                                if (line.contains("cpu,") || line.contains("cpu ")) {
+                                    rootCGroupPathMap.put("cpu", curPath);
+                                } else if (line.contains("cpuset")) {
+                                    rootCGroupPathMap.put("cpuset", curPath);
+                                } else if (line.contains("cpuacct")) {
+                                    rootCGroupPathMap.put("cpuacct", curPath);
+                                }
+                            }
+                        });
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                        fail();
+                    }
+                }
+            }
+        }
+        return rootCGroupPathMap.get(controllerName);
+    }
+
+    private String getConfig(TenantContainer tenant, String configName) {
+        String controllerName = configName.split("\\.")[0];
+        Path configPath = Paths.get(JGroup.rootPathOf(controllerName),
+                JGroup.JVM_GROUP_PATH,
+                "t" + tenant.getTenantId(),
+                configName);
+        try {
+            return new String(Files.readAllBytes(configPath)).trim();
+        } catch (IOException e) {
+            e.printStackTrace();
+            fail();
+        }
+        return null;
+    }
+}

test/multi-tenant/test/com/alibaba/tenant/TestTenantConfiguration.java

+/*
+ * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Alibaba designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+package com.alibaba.tenant;
+
+/* @test
+ * @summary test TenantConfiguration facilities
+ * @library /lib/testlibrary
+ * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+UseG1GC -XX:+TenantCpuThrottling
+ *                                 com.alibaba.tenant.TestTenantConfiguration
+ */
+
+import com.alibaba.rcm.Constraint;
+import com.alibaba.rcm.ResourceType;
+import jdk.testlibrary.TestUtils;
+import java.util.LinkedList;
+import java.util.List;
+import static jdk.testlibrary.Asserts.*;
+
+public class TestTenantConfiguration {
+
+    private void testBasic() {
+        try {
+            TenantConfiguration config = new TenantConfiguration()
+                    .limitCpuCfs(1_000_000, 500_000)
+                    .limitCpuShares(1024)
+                    .limitHeap(64 * 1024 * 1024);
+            assertNotNull(config);
+            TenantContainer tenant = TenantContainer.create(config);
+            assertNotNull(tenant);
+            tenant.destroy();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            fail();
+        }
+    }
+
+    private void testIllegalLimits() {
+        Runnable illegalActions[] = {
+                ()->{ new TenantConfiguration().limitCpuSet(""); },
+                ()->{ new TenantConfiguration().limitCpuSet(null); },
+                ()->{ new TenantConfiguration().limitCpuShares(-128); },
+                ()->{ new TenantConfiguration().limitHeap(-64 * 1024 * 1024); },
+                ()->{ new TenantConfiguration().limitCpuCfs(-123, 10_000_000); },
+                ()->{ new TenantConfiguration().limitCpuCfs(1_000_000, -2); },
+                ()->{ new TenantConfiguration().limitCpuCfs(999, -1); }, // lower bound of period
+                ()->{ new TenantConfiguration().limitCpuCfs(1_000_001, -1); }, // upper bound of period
+                ()->{ new TenantConfiguration().limitCpuCfs(10_000, 999); }, // lower bound of quota
+                ()->{ new TenantConfiguration(-1024 * 1024 * 1024); },
+                ()->{ new TenantConfiguration(-12, 64 * 1024 * 1024); },
+        };
+        for (Runnable action : illegalActions) {
+            try {
+                action.run();
+                fail("should throw IllegalArgumentException");
+            } catch (IllegalArgumentException e) {
+                // expected
+            } catch (Throwable e) {
+                e.printStackTrace();
+                fail("Should not throw any other exceptions");
+            }
+        }
+
+    }
+
+    private void testValidLimits() {
+        Runnable validActions[] = {
+                ()->{ new TenantConfiguration().limitCpuCfs(1_000_000, 2_000_000); },
+                ()->{ new TenantConfiguration().limitCpuCfs(100_000, 10_000); },
+                ()->{ new TenantConfiguration().limitCpuCfs(100_000, -1); }
+        };
+        for (Runnable action : validActions) {
+            try {
+                action.run();
+            } catch (Throwable e) {
+                e.printStackTrace();
+                fail("Should not throw any exceptions");
+            }
+        }
+    }
+
+    private void testGetSetLimits() {
+        TenantConfiguration config = new TenantConfiguration();
+        assertNotNull(config.getAllConstraints());
+        assertTrue(config.getAllConstraints().isEmpty());
+
+        long maxHeap = 64 * 1024 * 1024;
+
+        config.limitHeap(maxHeap);
+        List<Constraint> constraints = new LinkedList<>();
+        config.getAllConstraints().forEach(c -> constraints.add(c));
+
+        assertNotNull(config.getAllConstraints());
+        assertTrue(config.getAllConstraints().size() == 1);
+        Constraint constraint = constraints.get(0);
+        assertEquals(constraint.getResourceType(), ResourceType.HEAP_RETAINED);
+        assertTrue(constraint.getResourceType() instanceof ResourceType);
+        assertEquals(constraint.getValues()[0], maxHeap);
+        assertEquals(constraints.size(), 1);
+
+        config.limitCpuShares(1024);
+        constraints.clear();
+        config.getAllConstraints().forEach(c -> constraints.add(c));
+        assertTrue(constraints.stream().anyMatch(c -> c.getResourceType() == TenantResourceType.CPU_SHARES));
+        int nofCPUShares = (int) constraints.stream().filter(c -> c.getResourceType() == TenantResourceType.CPU_SHARES)
+                .peek(c -> assertEquals(c.getValues()[0], 1024L))
+                .count();
+        assertEquals(nofCPUShares, 1);
+    }
+
+    private void testUniqueConfigWhenCreatingContainer() {
+        TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024).limitHeap(64 * 1024 * 1024);
+        TenantContainer tenant = TenantContainer.create(config);
+        assertTrue(tenant.getConfiguration() == config);
+    }
+
+    public static void main(String[] args) {
+        TestUtils.runWithPrefix("test",
+                TestTenantConfiguration.class,
+                new TestTenantConfiguration());
+    }
+}

test/multi-tenant/test/com/alibaba/tenant/TestTenantContainer.java

@@ -199,6 +199,100 @@ public class TestTenantContainer {
         tenant.destroy();
     }
 
+        class TenantWorker implements Runnable{
+        public Tenant tenant;
+        public long cpuTime;
+
+        public long getCpuTime () {
+            ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+            return bean.isCurrentThreadCpuTimeSupported() ? bean
+                    .getCurrentThreadCpuTime() : 0L;
+        }
+
+        TenantWorker(Tenant t) {
+            tenant = t;
+        }
+
+        public void run(){
+            while(System.currentTimeMillis() - tenant.pre_ms < 3000){
+                tenant.nextCount();
+            }
+            cpuTime = getCpuTime();
+        }
+    }
+
+    class Tenant implements Runnable{
+        public AtomicLong count = new AtomicLong(0);
+        public long pre_ms = 0;
+        TenantWorker workers[];
+        int max_cpu;
+        Thread threads[];
+        long time = 0;
+
+        public long nextCount() {
+            return count.incrementAndGet();
+        }
+
+        public void run(){
+            max_cpu = Runtime.getRuntime().availableProcessors();
+            workers = new TenantWorker[max_cpu];
+            threads = new Thread[max_cpu];
+            pre_ms = System.currentTimeMillis();
+            for (int i = 0; i < max_cpu; i++) {
+                workers[i] = new TenantWorker(this);
+                threads[i] = new Thread(workers[i]);
+                threads[i].start();
+            }
+        }
+        public long getcpuTime(){
+            for (int i = 0; i < max_cpu; i++) {
+                try {
+                    threads[i].join();
+                    time += workers[i].cpuTime;
+                } catch (InterruptedException e) {
+                    System.out.println("Interreupted...");
+                }
+            }
+            return time;
+        }
+    }
+
+    @Test
+    public void testTenantCpuThrottling() throws TenantException {
+        if (!TenantGlobals.isCpuThrottlingEnabled()) {
+            return;
+        }
+
+        long time0 = 0;
+        long time1 = 0;
+        // put two tenants on same CPU core
+        TenantConfiguration tconfig0 = new TenantConfiguration().limitCpuShares(512).limitCpuSet("0");
+        TenantConfiguration tconfig1 = new TenantConfiguration().limitCpuShares(1024).limitCpuSet("0");
+        TenantContainer container0 = TenantContainer.create(tconfig0);
+        TenantContainer container1 = TenantContainer.create(tconfig1);
+        Tenant tenant0 = new Tenant();
+        Tenant tenant1 = new Tenant();
+
+        container0.run(tenant0);
+        container1.run(tenant1);
+
+        time1 = tenant1.getcpuTime();
+        time0 = tenant0.getcpuTime();
+
+        System.out.println("testTenantCpuThrottling tenant0 count: " + tenant0.count + " tenant0 CpuTime:" + time0
+                                + " tenant1 count: " + tenant1.count + " tenant1 CpuTime:" + time1);
+        double rate = 0.0;
+        if (TenantGlobals.isCpuThrottlingEnabled()) {
+            rate = (double)time1 / (time0 * 2);
+            if ((rate > 1.1) || (rate < 0.9)) {
+                fail();
+            }
+        } else {
+            rate = (double)time1 / time0;
+        }
+        System.out.println("CPU throttling enabled: "+ TenantGlobals.isCpuThrottlingEnabled()+ ", rate = " + rate);
+    }
+
     @Test
     public void testTenantInheritance() throws TenantException {
         TenantConfiguration tconfig = new TenantConfiguration();
@@ -339,6 +433,7 @@ public class TestTenantContainer {
         test.testGetAttachedThreads();
         test.testRun();
         test.testCurrentWithGC();
+        test.testTenantCpuThrottling();
         test.testTenantInheritance();
         test.testRunInRootTenant();
         test.testTenantGetAllocatedMemory();