From 5cd07a3f7916aaa64051d6e753a5a30bebb13849 Mon Sep 17 00:00:00 2001 From: Chuansheng Lu Date: Tue, 31 Mar 2020 18:55:18 +0800 Subject: [PATCH] [MultiTenant] Added CPU throttling feature Summary: ported cgroup-based cpu throttling facilities Test Plan: jdk/test/multi-tenant Reviewed-by: yuleil, superajun-wsj Issue: https://github.com/alibaba/dragonwell8/issues/84 --- make/CompileLaunchers.gmk | 5 + make/CompileNativeLibraries.gmk | 2 + make/CopyIntoClasses.gmk | 6 + make/CreateJars.gmk | 6 +- make/Images.gmk | 4 + make/lib/JGroupLibraries.gmk | 51 ++ make/mapfiles/libjgroup/mapfile-vers | 30 ++ .../native/jgroup/com/alibaba/tenant/jgroup.c | 400 +++++++++++++++ .../classes/com/alibaba/tenant/JGroup.java | 484 ++++++++++++++++++ .../com/alibaba/tenant/JGroupConstraint.java | 41 ++ .../com/alibaba/tenant/JGroupInitializer.sh | 257 ++++++++++ .../com/alibaba/tenant/JGroupMain.java | 145 ++++++ .../com/alibaba/tenant/NativeDispatcher.java | 57 +++ .../alibaba/tenant/TenantConfiguration.java | 131 ++++- .../com/alibaba/tenant/TenantContainer.java | 1 + .../tenant/TenantResourceContainer.java | 36 ++ .../alibaba/tenant/TenantResourceType.java | 140 +++++ test/multi-tenant/TestGetAllTenantIDs.java | 92 ++++ test/multi-tenant/TestJGroupDebugMode.sh | 109 ++++ test/multi-tenant/TestNoJGroupInit.java | 39 ++ .../alibaba/tenant/TestCpuCfsThrottling.java | 274 ++++++++++ .../tenant/TestHierachicalTenants.java | 159 ++++++ .../test/com/alibaba/tenant/TestJGroup.java | 197 +++++++ .../com/alibaba/tenant/TestJGroupInit.java | 240 +++++++++ .../tenant/TestTenantConfiguration.java | 139 +++++ .../alibaba/tenant/TestTenantContainer.java | 95 ++++ 26 files changed, 3132 insertions(+), 8 deletions(-) create mode 100644 make/lib/JGroupLibraries.gmk create mode 100644 make/mapfiles/libjgroup/mapfile-vers create mode 100644 src/linux/native/jgroup/com/alibaba/tenant/jgroup.c create mode 100644 src/share/classes/com/alibaba/tenant/JGroup.java create mode 100644 src/share/classes/com/alibaba/tenant/JGroupConstraint.java create mode 100644 src/share/classes/com/alibaba/tenant/JGroupInitializer.sh create mode 100644 src/share/classes/com/alibaba/tenant/JGroupMain.java create mode 100644 test/multi-tenant/TestGetAllTenantIDs.java create mode 100644 test/multi-tenant/TestJGroupDebugMode.sh create mode 100644 test/multi-tenant/TestNoJGroupInit.java create mode 100644 test/multi-tenant/test/com/alibaba/tenant/TestCpuCfsThrottling.java create mode 100644 test/multi-tenant/test/com/alibaba/tenant/TestHierachicalTenants.java create mode 100644 test/multi-tenant/test/com/alibaba/tenant/TestJGroup.java create mode 100644 test/multi-tenant/test/com/alibaba/tenant/TestJGroupInit.java create mode 100644 test/multi-tenant/test/com/alibaba/tenant/TestTenantConfiguration.java diff --git a/make/CompileLaunchers.gmk b/make/CompileLaunchers.gmk index 1e764088f..d85815199 100644 --- a/make/CompileLaunchers.gmk +++ b/make/CompileLaunchers.gmk @@ -432,6 +432,11 @@ ifeq ($(OPENJDK_TARGET_OS), windows) -DJAVA_ARGS='{ "-J-ms8m"$(COMMA) "sun.security.krb5.internal.tools.Ktab"$(COMMA) }')) endif +ifeq ($(OPENJDK_TARGET_OS), linux) + $(eval $(call SetupLauncher,jgroup, \ + -DJAVA_ARGS='{ "-J-ms8m"$(COMMA) "com.alibaba.tenant.JGroupMain"$(COMMA) }')) +endif + ########################################################################################## # The order of the object files on the link command line affects the size of the resulting # binary (at least on linux) which causes the size to differ between old and new build. diff --git a/make/CompileNativeLibraries.gmk b/make/CompileNativeLibraries.gmk index c23b958b0..b7b477d73 100644 --- a/make/CompileNativeLibraries.gmk +++ b/make/CompileNativeLibraries.gmk @@ -87,6 +87,8 @@ include lib/Awt2dLibraries.gmk include lib/SoundLibraries.gmk +include lib/JGroupLibraries.gmk + # Include the corresponding custom file, if present. -include $(CUSTOM_MAKE_DIR)/CompileNativeLibraries.gmk diff --git a/make/CopyIntoClasses.gmk b/make/CopyIntoClasses.gmk index e093e9a95..1f914d0e4 100644 --- a/make/CopyIntoClasses.gmk +++ b/make/CopyIntoClasses.gmk @@ -74,6 +74,12 @@ COPY_FILES += \ COPY_FILES += \ $(JDK_TOPDIR)/src/share/classes/sun/net/idn/uidna.spp +# Copy jgroup init script to compilation output dir +ifeq ($(OPENJDK_TARGET_OS), linux) + JGROUP_RESOURCE_DIR = $(JDK_TOPDIR)/src/share/classes/com/alibaba/tenant + COPY_FILES += $(wildcard $(JGROUP_RESOURCE_DIR)/JGroupInitializer.sh) +endif + # # Swing plaf resources # diff --git a/make/CreateJars.gmk b/make/CreateJars.gmk index 6d85be350..20476fd6a 100644 --- a/make/CreateJars.gmk +++ b/make/CreateJars.gmk @@ -213,7 +213,8 @@ RT_JAR_EXCLUDES += \ $(LOCALEDATA_INCLUDES) \ com/oracle/jrockit/jfr \ oracle/jrockit/jfr \ - jdk/jfr + jdk/jfr \ + com/alibaba/tenant/JGroupMain.class # Find all files in the classes dir to use as dependencies. This could be more fine granular. ALL_FILES_IN_CLASSES := $(call not-containing, _the., $(filter-out %javac_state, \ @@ -523,7 +524,8 @@ $(eval $(call SetupArchive,BUILD_TOOLS_JAR, , \ META-INF/services/com.sun.jdi.connect.spi.TransportService \ META-INF/services/com.sun.tools.attach.spi.AttachProvider \ META-INF/services/com.sun.tools.internal.ws.wscompile.Plugin \ - META-INF/services/com.sun.tools.internal.xjc.Plugin, \ + META-INF/services/com.sun.tools.internal.xjc.Plugin \ + com/alibaba/tenant/JGroupMain.class, \ JAR := $(IMAGES_OUTPUTDIR)/lib/tools.jar, \ SKIP_METAINF := true, \ CHECK_COMPRESS_JAR := true)) diff --git a/make/Images.gmk b/make/Images.gmk index 3354d8080..9ee3e19b6 100644 --- a/make/Images.gmk +++ b/make/Images.gmk @@ -128,6 +128,10 @@ ifeq ($(PROFILE), ) jhat$(EXE_SUFFIX) \ clhsdb$(EXE_SUFFIX) \ hsdb$(EXE_SUFFIX) + # jgroup is not included in JRE build + ifeq ($(OPENJDK_TARGET_OS), linux) + NOT_JRE_BIN_FILES += jgroup$(EXE_SUFFIX) + endif endif WINDOWS_JDK_BIN_FILES = \ diff --git a/make/lib/JGroupLibraries.gmk b/make/lib/JGroupLibraries.gmk new file mode 100644 index 000000000..ce078af7b --- /dev/null +++ b/make/lib/JGroupLibraries.gmk @@ -0,0 +1,51 @@ +# +# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Alibaba designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# + +LIBJGROUP_SRC_DIRS := $(JDK_TOPDIR)/src/linux/native/jgroup/com/alibaba/tenant + +LIBJGROUP_CFLAGS := $(foreach dir, $(LIBJGROUP_SRC_DIRS), -I$(dir)) + +LIBJGROUP_EXCLUDE_FILES := + +$(eval $(call SetupNativeCompilation,BUILD_LIBJGROUP, \ + LIBRARY := jgroup, \ + OUTPUT_DIR := $(INSTALL_LIBRARIES_HERE), \ + SRC := $(LIBJGROUP_SRC_DIRS), \ + EXCLUDE_FILES := $(LIBJGROUP_EXCLUDE_FILES), \ + LANG := C, \ + OPTIMIZATION := HIGH, \ + CFLAGS := $(CFLAGS_JDKLIB) \ + $(LIBJGROUP_CFLAGS), \ + MAPFILE := $(JDK_TOPDIR)/make/mapfiles/libjgroup/mapfile-vers, \ + LDFLAGS := $(LDFLAGS_JDKLIB) \ + $(call SET_SHARED_LIBRARY_ORIGIN), \ + LDFLAGS_SUFFIX_macosx := , \ + LDFLAGS_SUFFIX_solaris := , \ + LDFLAGS_SUFFIX_linux := -ljava, \ + LDFLAGS_SUFFIX_aix := ,\ + LDFLAGS_SUFFIX_windows := , \ + OBJECT_DIR := $(JDK_OUTPUTDIR)/objs/libjgroup, \ + DEBUG_SYMBOLS := $(DEBUG_ALL_BINARIES))) + +$(BUILD_LIBJGROUP): $(BUILD_LIBJAVA) + +BUILD_LIBRARIES += $(BUILD_LIBJGROUP) + diff --git a/make/mapfiles/libjgroup/mapfile-vers b/make/mapfiles/libjgroup/mapfile-vers new file mode 100644 index 000000000..1ee5164fa --- /dev/null +++ b/make/mapfiles/libjgroup/mapfile-vers @@ -0,0 +1,30 @@ +# +# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Alibaba designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# + +SUNWprivate_1.1 { + global: + Java_com_alibaba_tenant_NativeDispatcher_init0; + Java_com_alibaba_tenant_NativeDispatcher_createCgroup; + Java_com_alibaba_tenant_NativeDispatcher_moveToCgroup; + + local: + *; +}; diff --git a/src/linux/native/jgroup/com/alibaba/tenant/jgroup.c b/src/linux/native/jgroup/com/alibaba/tenant/jgroup.c new file mode 100644 index 000000000..94013ae9d --- /dev/null +++ b/src/linux/native/jgroup/com/alibaba/tenant/jgroup.c @@ -0,0 +1,400 @@ +// +// Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. Alibaba designates this +// particular file as subject to the "Classpath" exception as provided +// by Oracle in the LICENSE file that accompanied this code. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "jni.h" +#include "jni_util.h" +#include "com_alibaba_tenant_NativeDispatcher.h" + +// enum index of cgroup controllers +enum CGroupController { + CG_CPU, + CG_CPUSET, + CG_CPUACCT, + CG_MAX_SUPPORTED, /* supported by here */ + /* unsupported controllers */ + CG_BLKIO, + CG_DEVICES, + CG_FREEZER, + CG_MEMORY, /* supported by -XX:+TenantHeapThrottling */ + CG_NET_CLS, + CG_NET_PRIO, + CG_NS, + CG_PERF_EVENT +}; + +// supported controller names, must be access with index < CG_MAX_SUPPORTED +static const char* cg_controller_names[] = { + "cpu", + "cpuset", + "cpuacct", +}; + +// mountpoints of each cgroup controller, dynamically allocated in initialization, +// and never reclaimed +static const char* cg_mount_points[CG_MAX_SUPPORTED] = { NULL }; + +// Array to indicate if a controller is enabled +static unsigned char cg_controller_enabled[CG_MAX_SUPPORTED] = { 0 }; + +// set controller enabling status at startup +#define ENABLE_CONTROLLER(idx) \ + if (idx < CG_MAX_SUPPORTED) { \ + cg_controller_enabled[idx] = 1; \ + } +// returns zero if controller disabled; otherwise non-zero +#define IS_CONTROLLER_ENABLED(idx) ((idx < CG_MAX_SUPPORTED) ? cg_controller_enabled[idx] : 0) + +// To control if extra debugging msg should be printed +static jboolean jgroup_debug = JNI_FALSE; + +// Execute statement only in debugging mode +#define JGROUP_DEBUG(stmt) do { \ + if (JNI_TRUE == jgroup_debug) { \ + stmt; \ + } \ + } while(0) + +// Environment variable name to turn on debugging output of JGroup native code +#define JGROUP_DEBUG_ENV_KEY "JGROUP_DEBUG" + +// logging support +#define JGROUP_LOG(stream, format, ...) do { \ + if (jgroup_debug == JNI_TRUE) { \ + fprintf(stream, format, __VA_ARGS__); \ + } \ + } while (0) + +#define JGROUP_MSG(msg) JGROUP_LOG(stdout, "[JGROUP][INFO] %s\n", msg) +#define JGROUP_INFO(format, ...) JGROUP_LOG(stdout, "[JGROUP][INFO]" format "\n", __VA_ARGS__) +#define JGROUP_WARN(format, ...) JGROUP_LOG(stderr, "[JGROUP][WARN]" format "\n", __VA_ARGS__) +#define JGROUP_ERROR(format, ...) JGROUP_LOG(stderr, "[JGROUP][ERROR]" format "\n", __VA_ARGS__) + +#define SIG_STRING "Ljava/lang/String;" +#define MAX_PATH_LEN 2048 +#define CG_MAX_CTRL_NAME_LEN 128 + +//--------------------- Utility methods ---------------------- + +// Returns thread ID of current thread +static pid_t get_tid() { + return syscall(SYS_gettid); +} + +// returns JNI_TRUE if a 'path' is a file (not a directory); otherwise JNI_FALSE +static jboolean +is_file(const char* path) { + if (path != NULL) { + struct stat st; + memset(&st, 0, sizeof(struct stat)); + if (0 == stat(path, &st)) { + return S_ISREG(st.st_mode) ? JNI_TRUE : JNI_FALSE; + } + } + return JNI_FALSE; +} + +//--------------------- JNI impl ---------------------- + +// Detect if cgroup is enabled by reading /proc/cgroups +static int +detect_cgroup_enabled() { + char subsys_name[CG_MAX_CTRL_NAME_LEN]; + int res = 0, err = 0; + int hierarchy = 0, num_cgroups = 0, enabled = 0; + const char* proc_cgroups_path = "/proc/cgroups"; + int cg_index = 0; + FILE* proc_cgroups = fopen(proc_cgroups_path, "re"); + + if (proc_cgroups != NULL) { + while (!feof(proc_cgroups)) { + err = fscanf(proc_cgroups, "%s %d %d %d", subsys_name, + &hierarchy, &num_cgroups, &enabled); + if (err < 0) { + break; + } + + for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) { + if (!strcmp(subsys_name, cg_controller_names[cg_index]) && enabled) { + ENABLE_CONTROLLER(cg_index); + JGROUP_INFO("cgroup controller %s is enabled!", cg_controller_names[cg_index]); + break; + } + } + } + JGROUP_INFO("Opening %s to detect enabled controllers", proc_cgroups_path); + } else { + JGROUP_ERROR("Failed to open %s", proc_cgroups_path); + res = -1; + } + + return res; +} + +// Detect cgroup mountpoints +static int +detect_cgroup_mount_points() { + int res = 0; + struct mntent *ent = NULL; + char mntent_buffer[4 * MAX_PATH_LEN]; + int cg_index = 0; + FILE* proc_mounts = NULL; + struct mntent* temp_ent = NULL; + + temp_ent = (struct mntent *) malloc(sizeof(struct mntent)); + if (temp_ent == NULL) { + JGROUP_MSG("Failed to allocate temp struct mntent"); + res = -1; + } else { + proc_mounts = fopen("/proc/mounts", "re"); + if (proc_mounts != NULL) { + while ((ent = getmntent_r(proc_mounts, + temp_ent, + mntent_buffer, + sizeof(mntent_buffer))) != NULL) { + // skip non-cgroup mounts + if (strcmp(ent->mnt_type, "cgroup")) { + continue; + } + for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) { + if (NULL != hasmntopt(ent, cg_controller_names[cg_index])) { + cg_mount_points[cg_index] = strdup(ent->mnt_dir); + JGROUP_INFO("found mountpoint for %s at %s", + cg_controller_names[cg_index], + cg_mount_points[cg_index]); + } + } + } + } else { + JGROUP_MSG("failed to open /proc/mounts"); + res = -1; + } + + free(temp_ent); + + // double check + for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) { + if (IS_CONTROLLER_ENABLED(cg_index) && cg_mount_points[cg_index] == NULL) { + JGROUP_ERROR("Cannot determine mountpoint for controller %s", cg_controller_names[cg_index]); + res = -1; + } + } + } + + return res; +} + +// initialize static fields of class 'com.alibaba.tenant.NativeDispatcher' +// returns 0 on success; non-zero otherwise +static int +init_Java_globals(JNIEnv* env, jclass clazz) { + int res = 0; + char* mp_field_names[] = { "CG_MP_CPU", "CG_MP_CPUSET", "CG_MP_CPUACCT" }; + size_t idx = 0; + jfieldID fid = NULL; + + // reflect to Java level + // initialize mountpoints + for (idx = 0; idx < CG_MAX_SUPPORTED; ++idx) { + fid = (*env)->GetStaticFieldID(env, clazz, mp_field_names[idx], SIG_STRING); + jstring mp = (*env)->NewStringUTF(env, cg_mount_points[idx]); + if (mp != NULL) { + (*env)->SetStaticObjectField(env, clazz, fid, mp); + } + } + + // initialize flags of if a controller is enabled + if (IS_CONTROLLER_ENABLED(CG_CPU)) { + char buf[MAX_PATH_LEN]; + // cpu.shares + snprintf(buf, MAX_PATH_LEN, "%s/cpu.shares", cg_mount_points[CG_CPU]); + if (is_file(buf) == JNI_TRUE) { + fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPU_SHARES_ENABLED", "Z"); + (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE); + } + + // cpu.cfs_*_us + snprintf(buf, MAX_PATH_LEN, "%s/cpu.cfs_period_us", cg_mount_points[CG_CPU]); + if (is_file(buf) == JNI_TRUE) { + fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPU_CFS_ENABLED", "Z"); + (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE); + } + JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPU], cg_mount_points[CG_CPU]); + } + + // cpuset.* + if (IS_CONTROLLER_ENABLED(CG_CPUSET)) { + fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPUSET_ENABLED", "Z"); + (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE); + JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPUSET], cg_mount_points[CG_CPUSET]); + } + + // cpuacct.* + if (IS_CONTROLLER_ENABLED(CG_CPUACCT)) { + fid = (*env)->GetStaticFieldID(env, clazz, "IS_CPUACCT_ENABLED", "Z"); + (*env)->SetStaticBooleanField(env, clazz, fid, JNI_TRUE); + JGROUP_INFO("%s mountpoint=%s", cg_controller_names[CG_CPUACCT], cg_mount_points[CG_CPUACCT]); + } + + return res; +} + +// invoked once in static initializer +JNIEXPORT jint JNICALL +Java_com_alibaba_tenant_NativeDispatcher_init0(JNIEnv* env, jclass clazz) { + jint res = 0; + + // Detect and initialize native configuration + if (0 != (res = detect_cgroup_enabled())) { + JGROUP_MSG("Failed to detect cgroup status!"); + return res; + } + + if (0 != (res = detect_cgroup_mount_points())) { + JGROUP_MSG("Failed to detect cgroup mountpoints!"); + return res; + } + + // detect debugging options from environment variables + char* true_str = "TRUE"; // will be compared in uppercase + char* env_val = getenv(JGROUP_DEBUG_ENV_KEY); + if (env_val != NULL && strlen(true_str) == strlen(env_val)) { + size_t idx = 0; + jgroup_debug = JNI_TRUE; + for (; idx < strlen(true_str); ++idx) { + if (toupper(env_val[idx]) != true_str[idx]) { + jgroup_debug = JNI_FALSE; + break; + } + } + } + + JGROUP_INFO("jgroup debugging mode = %s", jgroup_debug == JNI_TRUE ? "true" : "false"); + + // initialize cgroup + if (0 != (res = init_Java_globals(env, clazz))) { + JGROUP_MSG("Failed to init Java globals!"); + } else { + JGROUP_MSG("cgroup initialized successfully"); + } + + return res; +} + +// +// Create the cgroup dir, and initialize some mandatory values +// @param group_path relative path of target CGROUP +JNIEXPORT jint JNICALL +Java_com_alibaba_tenant_NativeDispatcher_createCgroup(JNIEnv* env, + jobject ignored, + jstring group_path) { + int res = 0; + int cg_index = 0; + struct stat st; + if (group_path == NULL) { + JGROUP_MSG("null group path to create"); + res = -1; + } else { + const char* path_str = (*env)->GetStringUTFChars(env, group_path, NULL); + if (path_str != NULL || strlen(path_str) > 0) { + char path_buf[MAX_PATH_LEN]; + for (cg_index = 0; cg_index < CG_MAX_SUPPORTED; ++cg_index) { + snprintf(path_buf, MAX_PATH_LEN, "%s/%s", cg_mount_points[cg_index], path_str); + + // if exists, just reuse + if (stat(path_buf, &st) == 0 && S_ISDIR(st.st_mode)) { + JGROUP_INFO("Reuse existing directory %s", path_buf); + continue; + } + + res = mkdir(path_buf, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); // modes are copied from libcg + if (res == 0) { + JGROUP_INFO("created cgroup dir %s", path_buf); + } else { + JGROUP_ERROR("failed to create dir %s (%s)", path_buf, strerror(errno)); + break; + } + } + (*env)->ReleaseStringUTFChars(env, group_path, path_str); + } else { + JGROUP_MSG("Empty cgroup name"); + res = -1; + } + } + return res; +} + +JNIEXPORT jint JNICALL +Java_com_alibaba_tenant_NativeDispatcher_moveToCgroup(JNIEnv* env, + jobject ignored, + jstring group_path) { + + int res = 0; + if (group_path == NULL) { + JGROUP_MSG("null group path to create"); + res = -1; + } else { + const char* path_str = (*env)->GetStringUTFChars(env, group_path, NULL); + if (path_str != NULL && strlen(path_str) > 0) { + char path_buf[MAX_PATH_LEN]; + snprintf(path_buf, MAX_PATH_LEN, "%s/%s/tasks", cg_mount_points[CG_CPU], path_str); + + FILE* fp_tasks = fopen(path_buf, "we"); + if (fp_tasks != NULL) { + + JGROUP_INFO("Open file %s (we)", path_buf); + + pid_t tid = get_tid(); + int sz = fprintf(fp_tasks, "%d", (int)tid); + fclose(fp_tasks); + + if (sz > 0) { + JGROUP_INFO("PID %d has been writen to %s", (int)tid, path_buf); + } else { + JGROUP_ERROR("Failed to write to %s", path_buf); + } + + JGROUP_INFO("Close file %s (we)", path_buf); + } else { + JGROUP_ERROR("Failed to open %s (%s)", path_buf, strerror(errno)); + res = -1; + } + + (*env)->ReleaseStringUTFChars(env, group_path, path_str); + } else { + JGROUP_MSG("Empty cgroup name"); + res = -1; + } + } + return res; +} diff --git a/src/share/classes/com/alibaba/tenant/JGroup.java b/src/share/classes/com/alibaba/tenant/JGroup.java new file mode 100644 index 000000000..94d65d066 --- /dev/null +++ b/src/share/classes/com/alibaba/tenant/JGroup.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import com.alibaba.rcm.Constraint; +import sun.security.action.GetPropertyAction; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.AccessController; +import java.util.ArrayList; +import java.util.List; +import static com.alibaba.tenant.NativeDispatcher.*; + +/** + * JGroup is a java mirror of Linux control group + */ +class JGroup { + + // to dispatch calls to libcgroup and file system + private static NativeDispatcher nd; + + /* + * The cgroup hierarchy is: + * + * OS/Container ROOT group + * | + * |__ JDK Group + * | + * |__ JVM group of process 1 + * | |___ TenantContainer1's group + * | |___ TenantContainer2's group + * | |___ ... + * | + * |__ JVM group of process 2 + * | + * |__ ... + * + * The cgroup path identifying TenantContainer 't0' is in following format + * ///// + * + * e.g. + * /sys/fs/cgroup/cpuset,cpu,cpuacct/ajdk_multi_tenant/12345/t0 + * /sys/fs/cgroup/cpuset/system.slice/docker-0fdsnjk23njkfnkfnwe/ajdk_multi_tenant/12345/t0 + * + */ + + // OS/Container ROOT group, will be appended to each cgroup controller's root path + static final String ROOT_GROUP_PATH; + + // JDK group must be initialized before running any JVM process with CPU throttling/accounting enabled + static final String JDK_GROUP_PATH; + + // Current JVM process's cgroup path, parent of all non-ROOT TenantContainers' groups + static final String JVM_GROUP_PATH; + + // deprecated constants + static final int DEFAULT_WEIGHT = 1024; + + static final int DEFAULT_MAXCPU = 100; + + // unique top level groups + // rootGroup and jdkGroup are static, just to serve 'parent' API + private static JGroup rootGroup; + private static JGroup jdkGroup; + // jvm group is the top level active cgroup + private static JGroup jvmGroup; + + static JGroup jvmGroup() { + return jvmGroup; + } + + static JGroup jdkGroup() { + return jdkGroup; + } + + static JGroup rootGroup() { + return rootGroup; + } + + /* + * CGroup path of this cgroup + */ + private String groupPath; + + /* + * parent group + */ + private JGroup parentGroup; + + // corresponding TenantResourceContainer object + private TenantResourceContainer resourceContainer; + + // Create a JGroup and initialize necessary underlying cgroup configurations + JGroup(TenantResourceContainer container) { + if (container == null) { + throw new IllegalArgumentException("Must provide a non-null TenantContainer to start with"); + } + this.resourceContainer = container; + assert resourceContainer.getTenant() != null; + init(resourceContainer, getTenantGroupPath(resourceContainer)); + } + + // + // only directly called by initializeJGroupClass to initialize jvmGroup and jdkGroup + // + private JGroup(TenantResourceContainer container, String path) { + init(container, path); + } + + private void init(TenantResourceContainer container, String path) { + this.groupPath = path; + if (container == null) { + this.parentGroup = null; + if (JVM_GROUP_PATH.equals(path)) { + this.parentGroup = jdkGroup(); + initSystemGroup(this); + } else if (JDK_GROUP_PATH.equals(path)) { + this.parentGroup = rootGroup(); + // JDK group has already been initialized by JGroupMain + } else if (!ROOT_GROUP_PATH.equals(path)) { + System.err.println("Should not call this with path=" + path); + } + } else { + this.parentGroup = container.getParent() == null ? jvmGroup() : container.getParent().getJGroup(); + initUserGroup(this, container.getConstraints()); + } + } + + // + // To generate group path for non-ROOT tenant containers + // current pattern: + // "JVM_GROUP_PATH/t" + // + private String getTenantGroupPath(TenantResourceContainer container) { + TenantResourceContainer parent = container.getParent(); + return (parent == null ? JVM_GROUP_PATH : parent.getJGroup().groupPath) + + File.separator + "t" + container.getTenant().getTenantId(); + } + + // returns the relative cgroup path of this JGroup + private String groupPath() { + return groupPath; + } + + // returns parent JGroup of this one, null if current is the top most JGroup + JGroup parent() { + return parentGroup; + } + + /* + * Destory this jgroup + */ + void destory() { + Path parentTasks = configPathOf(parent(), CG_TASKS); + try { + // move all tasks of children group to parent! + Files.walk(Paths.get(rootPathOf(CG_CPU), groupPath()), FileVisitOption.FOLLOW_LINKS) + .filter(path -> path.endsWith(CG_TASKS)) + .forEach(path -> { + try { + for (String pidLine : Files.readAllLines(path)) { + Files.write(parentTasks, pidLine.getBytes()); + debug("destroyCgroup: move " + pidLine + " from " + path + + " to " + parentTasks); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + + // delete all directories + Files.walk(Paths.get(rootPathOf(CG_CPU), groupPath()), FileVisitOption.FOLLOW_LINKS) + .map(Path::toFile) + .filter(File::isDirectory) + .forEach(File::delete); + + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Initialize a cgroup with user specified limits after JVM bootstrapping, + // for non-ROOT tenant containers + private void initUserGroup(JGroup jgroup, Iterable constraints) { + try { + initCgroupCommon(jgroup); + + // set value with user specified values + // ignoring non-cgroup resources, like ResourceType.MEMORY + if (constraints != null) { + for (Constraint c : constraints) { + if (c.getResourceType() instanceof TenantResourceType) { + TenantResourceType type = (TenantResourceType)c.getResourceType(); + if (type.isJGroupResource()) { + ((JGroupConstraint)c).sync(jgroup); + } + } + } + } + } catch (RuntimeException e) { + throw e; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + // Initialize system cgroups, used by JGroup#initializeJGroupClass + // during JVM initialization to create the JDK and JVM group. + // NOTE: never throw exception here, will cause VM code to crash, very rude! + private void initSystemGroup(JGroup jgroup) { + try { + initCgroupCommon(jgroup); + } catch (Throwable t) { + t.printStackTrace(); + System.exit(127); + } + } + + // create and initialize underlying cgroups + private void initCgroupCommon(JGroup jgroup) { + if (jgroup == null || jgroup.groupPath() == null + || jgroup.groupPath().isEmpty()) { + throw new IllegalArgumentException("Bad argument to createCGroup()"); + } + + if (nd.createCgroup(jgroup.groupPath()) == 0) { + // init two mandatory fields + if (IS_CPUSET_ENABLED) { + copyValueFromParentGroup(jgroup, CG_CPUSET_CPUS); + copyValueFromParentGroup(jgroup, CG_CPUSET_MEMS); + } + debug("Created group with standard controllers"); + } else { + throw new RuntimeException("Failed to create cgroup at " + jgroup.groupPath()); + } + } + + // Copy config value from parent group + private void copyValueFromParentGroup(JGroup jgroup, String configName) { + String parentValue = getValue(jgroup.parent(), configName); + setValue(jgroup, configName, parentValue); + debug("Set group " + jgroup.groupPath()+ "'s config " + configName + + " with parent group's value: " + parentValue); + } + + + // Not a frequent operation, using Java implementation + private String getValue(JGroup group, String key) { + if (group == null || group.groupPath() == null || group.groupPath().isEmpty() + || key == null || key.isEmpty() || !key.contains(".")) { + return null; + } + String ctrlName = key.split("\\.")[0]; + Path configPath = Paths.get(rootPathOf(ctrlName), group.groupPath(), key); + if (Files.exists(configPath)) { + try { + String value = new String(Files.readAllBytes(configPath)).trim(); + debug(key + "=" + value); + return value; + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + debug("getValue: config path " + configPath + " does not exist"); + } + return null; + } + + // Not a frequent operation, using Java implementation + private void setValue(JGroup group, String key, String value) { + if (group == null || group.groupPath() == null || group.groupPath().isEmpty() + || key == null || key.isEmpty() + || value == null || value.isEmpty() + || !key.contains(".")) { + throw new IllegalArgumentException("Cannot set " + group.groupPath() + + File.separator + key + "=" + value); + } + String controllerName = key.split("\\.")[0]; + Path configPath = Paths.get(rootPathOf(controllerName), group.groupPath(), key); + + if (Files.exists(configPath)) { + try { + debug("Set value: " + configPath + "=" + value); + Files.write(configPath, value.getBytes()); + } catch (IOException e) { + throw new RuntimeException("Failed to set " + group.groupPath() + + File.separator + key + " = " + value, e); + } + } else { + debug("setValue: config path " + configPath + " does not exist"); + } + } + + // used by injected test case + static String rootPathOf(String controllerName) { + switch (controllerName) { + case CG_CPU : return CG_MP_CPU; + case CG_CPUSET : return CG_MP_CPUSET; + case CG_CPUACCT : return CG_MP_CPUACCT; + default: + throw new IllegalArgumentException("Unsupported controller : " + controllerName); + } + } + + private static void debug(String... messages) { + if (debugJGroup) { + System.out.print("[JGroupDispatcher] "); + for (String msg : messages) { + System.out.print(msg); + System.out.print(" "); + } + System.out.println(); + } + } + + /* + * Attach the current thread into this jgroup. + * @return 0 if successful + */ + void attach() { + if (nd.moveToCgroup(groupPath()) != 0) { + throw new IllegalStateException("Cannot attach to group " + this); + } else { + debug("Attached to cgroup: " + groupPath()); + } + } + + /** + * Detach the current thread from this jgroup + * @return 0 if successful + */ + void detach() { + jvmGroup().attach(); + } + + /** + * Get cpuacct usage of this group + * @return cpu usage in nano seconds + */ + long getCpuTime() { + try { + return Long.parseLong(getValue(this, "cpuacct.usage")); + } catch (Exception e) { + System.err.println("Exception from JGroup.getCpuTime()"); + e.printStackTrace(); + } + return -1; + } + + /* + * Set the value of a cgroup attribute + */ + synchronized void setValue(String key, String name) { + setValue(this, key, name); + } + + /* + * Get the value of a cgroup attribute + */ + synchronized String getValue(String key) { + return getValue(this, key); + } + + /* + * Initialize the JGroup class, called in create_vm. + */ + private static void initializeJGroupClass() { + try { + // NOTE: below sequence should not be changed! + rootGroup = new JGroup(null, ROOT_GROUP_PATH); + jdkGroup = new JGroup(null, JDK_GROUP_PATH); + jvmGroup = new JGroup(null, JVM_GROUP_PATH); + + // check before attach + checkRootGroupPath(); + + jvmGroup().attach(); + } catch (Throwable t) { + System.err.println("Failed to initialize JGroup"); + t.printStackTrace(); + // do not throw any exception during VM initialization, but just exit + System.exit(128); + } + } + + // Check if mandatory root group paths exist + private static void checkRootGroupPath() { + List controllers = new ArrayList<>(4); + if (TenantGlobals.isCpuThrottlingEnabled()) { + controllers.add("cpu"); + controllers.add("cpuset"); + } + if (TenantGlobals.isCpuAccountingEnabled()) { + controllers.add("cpuacct"); + } + for (String ctrl : controllers) { + String r = rootPathOf(ctrl); + if (r == null || !Files.exists(Paths.get(r, ROOT_GROUP_PATH))) { + throw new IllegalArgumentException("Bad ROOT group path: " + r + File.separator + ROOT_GROUP_PATH); + } + } + } + + private static Path configPathOf(JGroup group, String config) { + String rootPath = null; + try { + rootPath = rootPathOf(config.split("\\.")[0]); + } catch (IllegalArgumentException e) { + // using 'cpu' group root as default + rootPath = rootPathOf("cpu"); + } + return Paths.get(rootPath, group.groupPath(), config); + } + + /* + * Destory the JGroup class, called in destroy_vm. + */ + private static void destroyJGroupClass() { + jvmGroup.destory(); + jvmGroup = null; + } + + // --- debugging support --- + private static boolean debugJGroup; + private static final String DEBUG_JGROUP_PROP = "com.alibaba.tenant.debugJGroup"; + + // property name of user specified ROOT cgroup path, relative to each controller's ROOT path + private static final String PROP_ROOT_GROUP = "com.alibaba.tenant.jgroup.rootGroup"; + + // property name of user specified JDK group path, relative to ROOT_GROUP_PATH + private static final String PROP_JDK_GROUP = "com.alibaba.tenant.jgroup.jdkGroup"; + + // default value of property PROP_JDK_GROUP + private static final String DEFAULT_JDK_GROUP_NAME = "ajdk_multi_tenant"; + + // default value of ROOT_GROUP_PATH + private static final String DEFAULT_ROOT_GROUP_PATH = "/"; + + static { + nd = new NativeDispatcher(); + + String prop = System.getProperty(PROP_ROOT_GROUP); + if (prop == null) { + ROOT_GROUP_PATH = DEFAULT_ROOT_GROUP_PATH; + } else { + ROOT_GROUP_PATH = prop; + } + + prop = System.getProperty(PROP_JDK_GROUP); + if (prop == null) { + JDK_GROUP_PATH = ROOT_GROUP_PATH + File.separator + DEFAULT_JDK_GROUP_NAME; + } else { + JDK_GROUP_PATH = ROOT_GROUP_PATH + File.separator + prop; + } + + JVM_GROUP_PATH = JDK_GROUP_PATH + File.separator + nd.getProcessId(); + + debugJGroup = Boolean.parseBoolean( + AccessController.doPrivileged(new GetPropertyAction(DEBUG_JGROUP_PROP))); + } +} diff --git a/src/share/classes/com/alibaba/tenant/JGroupConstraint.java b/src/share/classes/com/alibaba/tenant/JGroupConstraint.java new file mode 100644 index 000000000..2dea3f578 --- /dev/null +++ b/src/share/classes/com/alibaba/tenant/JGroupConstraint.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import com.alibaba.rcm.Constraint; +import com.alibaba.rcm.ResourceType; + +/** + * Added extra method {@code sync} to flush configuration to cgroup filesystem + */ +abstract class JGroupConstraint extends Constraint { + + JGroupConstraint(ResourceType type, long[] values) { + super(type, values); + } + + /** + * Flush the constraint configuration of this constraint to underlying cgroup impl + * @param jgroup + */ + abstract void sync(JGroup jgroup); +} diff --git a/src/share/classes/com/alibaba/tenant/JGroupInitializer.sh b/src/share/classes/com/alibaba/tenant/JGroupInitializer.sh new file mode 100644 index 000000000..8ae1769ef --- /dev/null +++ b/src/share/classes/com/alibaba/tenant/JGroupInitializer.sh @@ -0,0 +1,257 @@ +# +# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Alibaba designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# + +#!/usr/bin/env bash + +# +# To initialize global JGroup (Java wrapper of Linux cgroup) configuration: +# 1, mount required cgroup controllers +# 2, create ROOT cgroup 'ajdk_multi_tenant' for JGroup +# 3, save cgroup configurations to /etc/cgconfig.conf +# + +# globals +JG_USER="" +JG_GROUP="" +JG_ROOT="" +NOF_CPUS="0" # at least have 1 cpus (which is "0-0" for cpuset.cpus) +IS_IN_DOCKER_CONTAINER="" # if the script is executed inside docker container, "TRUE" if yes, "" otherwise + +# check if all prerequisitions are ready +check_prerequisitions() { + # needed commandline tools + REQUIRED_TOOLS=("mountpoint" "mount" "mkdir" "cut" "df" "sed" "getopt" "wc" "awk") + for REQ in ${REQUIRED_TOOLS[*]}; do + which $REQ > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Cannot find prerequisition : $REQ, please install!" + exit 1 + fi + done + # init globals + NOF_CPUS=$(grep processor /proc/cpuinfo | wc -l) + # check if we are in DOCKER + if [ -z "$IS_IN_DOCKER_CONTAINER" ]; then + if [ ! -z "$(cat /proc/1/cgroup | awk -F/ '{print $2}')" ]; then + IS_IN_DOCKER_CONTAINER="TRUE" + fi + fi +} + +# +# reference https://github.com/jpetazzo/dind/ +# +cgroup_init() { + CGROUP="" + # loop over /proc/mounts, and search for mounted cgroup device + while read ln; do + # searching for cgroup record, like: + # tmpfs /sys/fs/cgroup tmpfs rw,nosuid,nodev,noexec,mode=755 0 0 + DEV=$(echo $ln | awk '{print $3}') + if [ $DEV = "tmpfs" ]; then + if [ -n "$(echo $ln | grep cgroup)" ]; then + CGROUP="$(echo $ln | awk '{print $2}')" + fi + fi + done < /proc/mounts + + # if did not found cgroup mount point, init cgroup mountpoints + if [ -z "$CGROUP" ]; then + # use '/cgroup' as DEFAULT mountpoint + CGROUP="/cgroup" + # use /sys/fs/cgroup for AliOS7 + uname -r | grep -q alios7 && CGROUP='/sys/fs/cgroup' + + echo "CGroup file system not mounted, trying to mount it to $CGROUP" + [ -d "$CGROUP" ] || mkdir $CGROUP + mountpoint -q $CGROUP || { + echo "Mounting cgroup file system" + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use --privileged?" + exit 1 + } + } + + # Mount the cgroup hierarchies exactly as they are in the parent system. + for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) + do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS + + echo $SUBSYS | grep -q ^name= && { + NAME=$(echo $SUBSYS | sed s/^name=//) + [ -d $CGROUP/$NAME ] || ln -s $SUBSYS $CGROUP/$NAME + } + + [ $SUBSYS = cpuacct,cpu ] && [ ! -d $CGROUP/cpu,cpuacct ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct + done + fi + + # check if CGROUP file system is writable + # In kernel 3.10.0-327.13.1.el7.x86_64, `ls` and `-w` returns different results for $CGROUP; + # but the controller file inside $CGROUP is still writable anyway. + if [ ! -w $CGROUP ] && [ ! -w $CGROUP/cpu/cpu.shares ]; then + echo "CGroup file system mounted at $CGROUP is not writable, please check CGroup config!" + if [ "$IS_IN_DOCKER_CONTAINER" = "TRUE" ]; then + echo "It looks like to be inside a docker container, did you forget to add '--privileged' option to 'docker run'?" + fi + exit 1 + fi + + [ -d $CGROUP/cpu ] || ln -s cpu,cpuacct $CGROUP/cpu + [ -d $CGROUP/cpuacct ] || ln -s cpu,cpuacct $CGROUP/cpuacct + + # create JDK sub-directories for each controller + CONTROLLERS=("cpu" "cpuacct" "cpuset") + for CTRL in ${CONTROLLERS[*]}; do + if [ ! -z "$JG_ROOT" ] && [ ! -d "$CGROUP/$CTRL/$JG_ROOT" ]; then + echo "User specified ROOT directory: $CGROUP/$CTRL/$JG_ROOT does not exist" + exit 1 + fi + CTRL_DIR="$CGROUP/$CTRL/$JG_ROOT/ajdk_multi_tenant" + # create directories if needed + if [ ! -d "$CTRL_DIR" ]; then + mkdir -p "$CTRL_DIR" + if [ $? -ne 0 ]; then + echo "Failed to create JDK group: $CTRL_DIR" + exit 1 + fi + fi + # setting up owners + chown -R $JG_USER:$JG_GROUP $CTRL_DIR + done + + # initialize necessary controllers + cp $CGROUP/cpu/$JG_ROOT/cpu.shares $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.shares + cp $CGROUP/cpu/$JG_ROOT/cpu.cfs_period_us $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.cfs_period_us + # + # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu.html + # Setting the value in cpu.cfs_quota_us to -1 indicates that the cgroup does not adhere to any CPU time restrictions. + # This is also the default value for every cgroup + # + # No limit on JDK group + echo "-1" > $CGROUP/cpu/$JG_ROOT/ajdk_multi_tenant/cpu.cfs_quota_us + + # + # cpuset.cpus and cpuset.mems may be empty, which is normal behavior according to AliOS team, + # but in that case tasks/threads cannot be attached to that group. + # Here just check and forcefully initialize it with a default value; + # see http://aone.alibaba-inc.com/code/D87428 for more details + # + # 1) Check if parent group (controller root) has a valid config, initialize parent if needed; + # 2) Copy parent group's config to /ajdk_multi_tenant + # + if [ ! -z "$JG_ROOT" ]; then + ROOT_CPUSET_CPUS="$CGROUP/cpuset/$JG_ROOT/cpuset.cpus" + else + ROOT_CPUSET_CPUS="$CGROUP/cpuset/cpuset.cpus" + fi + if [ -z `cat $ROOT_CPUSET_CPUS` ]; then + echo "0-$NOF_CPUS" > $ROOT_CPUSET_CPUS + if [ 0 != $? ]; then + echo "Cannot init ROOT group's config $ROOT_CPUSET_CPUS" + exit 1 + fi + fi + cp $ROOT_CPUSET_CPUS $CGROUP/cpuset/$JG_ROOT/ajdk_multi_tenant/cpuset.cpus + + if [ ! -z "$JG_ROOT" ]; then + ROOT_CPUSET_MEMS="$CGROUP/cpuset/$JG_ROOT/cpuset.mems" + else + ROOT_CPUSET_MEMS="$CGROUP/cpuset/cpuset.mems" + fi + if [ -z `cat $ROOT_CPUSET_MEMS` ]; then + echo "0" > $ROOT_CPUSET_MEMS + if [ 0 != $? ]; then + echo "Cannot init ROOT group's config $ROOT_CPUSET_MEMS" + exit 1 + fi + fi + cp $ROOT_CPUSET_MEMS $CGROUP/cpuset/$JG_ROOT/ajdk_multi_tenant/cpuset.mems + + # save current CGroup configuration to /etc/cgconfig.conf + CONFIG_FILE='/etc/cgconfig.conf' + which cgsnapshot + if [ $? -eq 0 ]; then + cgsnapshot -s -f $CONFIG_FILE + if [ $? -ne 0 ]; then + echo "Failed to save cgroup configuration to $CONFIG_FILE" + exit 1 + fi + fi +} + +# show help information +show_help() { + echo "Usage: jgroup [-u username] [-g usergroup] [-r root group path] [-h]" + echo "Initialize the jgroup configuration for multi-tenant JVM (which requires root permission)" + echo "e.g. jgroup -u admin -g admin" + echo "e.g. jgroup -u admin -g admin -r /system.slice/dockr-abcdefghijklmnopq123/" +} + +# parse arguments +parse_args() { + while getopts "u:g:r:h" arg; do + case "$arg" + in + u) + JG_USER="$OPTARG" + ;; + g) + JG_GROUP="$OPTARG" + ;; + r) + JG_ROOT="$OPTARG" + ;; + h|?) + show_help + exit 1 + ;; + esac + done + + # check result of parsing + if [ -z "$JG_GROUP" ] || [ -z "$JG_USER" ]; then + echo "Bad parameters: user = $JG_USER, group = $JG_GROUP" + show_help + exit 1 + fi + + # check if $JG_USER and $JG_GROUP exists in /etc + if [ -z "$(cat /etc/passwd | awk -F: '{print $1}' | grep $JG_USER)" ]; then + echo "User name $JG_USER does not exist in /etc/passwd!" + exit 1 + fi + + if [ -z "$(cat /etc/group | awk -F: '{print $1}' | grep $JG_GROUP)" ]; then + echo "Group name $JG_GROUP does not exist in /etc/group!" + exit 1 + fi +} + +# entrypoint +parse_args $* +check_prerequisitions +cgroup_init + +# done +exit 0 diff --git a/src/share/classes/com/alibaba/tenant/JGroupMain.java b/src/share/classes/com/alibaba/tenant/JGroupMain.java new file mode 100644 index 000000000..41b5b3708 --- /dev/null +++ b/src/share/classes/com/alibaba/tenant/JGroupMain.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import sun.security.action.GetPropertyAction; +import javax.tools.ToolProvider; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.AccessController; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeoutException; + +/** + * Launcher of the jgroup configuration tools + * usually invoked during JDK installation. + *

+ * NOTE: Privileged user permission (e.g. sudo, su) is needed! + */ +public class JGroupMain { + + /* + * name of JGroup's initializer resource stored in jdk/tools.jar. + * The content of the resource file will be loaded from jdk/tools.jar and writen out to a temp shell script file, + * finally the temp shell script will be executed. + */ + private static final String INITIALIZER_NAME = "com/alibaba/tenant/JGroupInitializer.sh"; + + /* + * Wait {@code TIMEOUT_MILLIS} for child process to end + */ + private static final long TIMEOUT_MILLIS = 60_000; + + private static final String SCRIPT_PREFIX = "jgroupInit_"; + + public static void main(String[] args) { + try { + new JGroupMain().doInitilization(args); + } catch (Exception e) { + throw (e instanceof RuntimeException ? (RuntimeException)e : new RuntimeException(e)); + } + } + + // Check if current process has permission to init jgroup + private void checkPermission() { + // check platform + String osName = AccessController.doPrivileged(new GetPropertyAction("os.name")); + if (!"Linux".equalsIgnoreCase(osName)) { + throw new RuntimeException("Only supports Linux platform"); + } + + // check permission of writing system files + File file = new File("/etc"); + if (!file.canWrite()) { + throw new SecurityException("Permission denied! need WRITE permission on system files"); + } + } + + // do initialization work! + private void doInitilization(String[] args) + throws TimeoutException, IOException { + // pre-check + checkPermission(); + + // create a temporary shell script to do primary initialization work + String scriptAbsPath = generateInitScript(); + + // execute the script file to initialize jgroup in a child process + List arguments = new ArrayList<>(1 + (args == null ? 0 : args.length)); + arguments.add(scriptAbsPath); + for (String arg : args) { + arguments.add(arg); + } + ProcessBuilder pb = new ProcessBuilder(arguments); + pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); + pb.redirectError(ProcessBuilder.Redirect.INHERIT); + Process process = pb.start(); + if (process == null) { + throw new RuntimeException("Failed to launch initializer process!"); + } + + // wait with timeout for child process to terminate + try { + process.waitFor(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (process.isAlive()) { + process.destroyForcibly(); + throw new TimeoutException("ERROR: Initializer process does not finish in " + TIMEOUT_MILLIS + "ms!"); + } + + int retValue = process.exitValue(); + if (retValue != 0) { + throw new IllegalStateException("Bad exit value from initialization process: " + retValue); + } + } + + private String generateInitScript() throws IOException { + // load script content + ClassLoader loader = ToolProvider.getSystemToolClassLoader(); // JGroupInitializer.sh is in jdk/lib/tools.jar + InputStream stream = loader.getResourceAsStream(INITIALIZER_NAME); + + if (stream == null) { + throw new IOException("Cannot load " + INITIALIZER_NAME + " using system classloader"); + } + + File script = File.createTempFile(SCRIPT_PREFIX, ".sh"); + script.setExecutable(true); + script.setWritable(true); + script.deleteOnExit(); + + // copy content of initializer resource to a temp shell script file + OutputStream fos = new FileOutputStream(script); + int buf = 0; + while ((buf = stream.read()) != -1) { + fos.write(buf); + } + fos.close(); + + return script.getAbsolutePath(); + } +} diff --git a/src/share/classes/com/alibaba/tenant/NativeDispatcher.java b/src/share/classes/com/alibaba/tenant/NativeDispatcher.java index 5fa50373e..30beb4a9d 100644 --- a/src/share/classes/com/alibaba/tenant/NativeDispatcher.java +++ b/src/share/classes/com/alibaba/tenant/NativeDispatcher.java @@ -29,6 +29,47 @@ import java.security.AccessController; */ class NativeDispatcher { + // ------------- CGroup constants and flags ----------- + // names of cgroup controllers + static final String CG_CPU = "cpu"; + static final String CG_CPU_SHARES = "cpu.shares"; + static final String CG_CPU_CFS_QUOTA = "cpu.cfs_quota_us"; + static final String CG_CPU_CFS_PERIOD = "cpu.cfs_period_us"; + static final String CG_CPUSET = "cpuset"; + static final String CG_CPUSET_CPUS = "cpuset.cpus"; + static final String CG_CPUSET_MEMS = "cpuset.mems"; + static final String CG_CPUACCT = "cpuacct"; + static final String CG_TASKS = "tasks"; + + // flags to indicate if target cgroup controllers has been enabled + // will be initialized by native code + static boolean IS_CPU_SHARES_ENABLED = false; + static boolean IS_CPU_CFS_ENABLED = false; + static boolean IS_CPUSET_ENABLED = false; + static boolean IS_CPUACCT_ENABLED = false; + + // mountpoint of each controller, will be initialized by native code + static String CG_MP_CPU = null; + static String CG_MP_CPUSET = null; + static String CG_MP_CPUACCT = null; + + static final boolean CGROUP_INIT_SUCCESS; + + // retrieve process ID of current JVM process + long getProcessId() { + String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + return Long.parseLong(processName.split("@")[0]); + } + + // --------- CGroup wrappers -------- + private static native int init0(); + + // create new cgroup path + native int createCgroup(String groupPath); + + // move current thread to cgroup at 'groupPath' + native int moveToCgroup(String groupPath); + // Attach the current thread to given {@code tenant} native void attach(TenantContainer tenant); @@ -59,5 +100,21 @@ class NativeDispatcher { } }); registerNatives0(); + + boolean initSuccess = false; + if (TenantGlobals.isCpuThrottlingEnabled() || TenantGlobals.isCpuAccountingEnabled()) { + AccessController.doPrivileged( + new java.security.PrivilegedAction() { + public Void run() { + System.loadLibrary("jgroup"); + return null; + } + }); + initSuccess = (0 == init0()); + if (!initSuccess) { + throw new RuntimeException("JGroup native dispatcher initialization failed!"); + } + } + CGROUP_INIT_SUCCESS = initSuccess; } } diff --git a/src/share/classes/com/alibaba/tenant/TenantConfiguration.java b/src/share/classes/com/alibaba/tenant/TenantConfiguration.java index 5992f1b89..e95d998f3 100644 --- a/src/share/classes/com/alibaba/tenant/TenantConfiguration.java +++ b/src/share/classes/com/alibaba/tenant/TenantConfiguration.java @@ -21,16 +21,23 @@ package com.alibaba.tenant; +import com.alibaba.rcm.Constraint; +import com.alibaba.rcm.ResourceType; import sun.misc.SharedSecrets; import sun.misc.VM; import sun.security.action.GetPropertyAction; import java.security.AccessController; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Collection; import com.alibaba.rcm.ResourceType; import com.alibaba.rcm.Constraint; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import java.util.stream.Stream; import static com.alibaba.rcm.ResourceType.*; +import static com.alibaba.tenant.TenantResourceType.*; /** * @@ -49,6 +56,22 @@ public class TenantConfiguration { public TenantConfiguration() { } + /** + * @param cpuShare + * @param maxHeapBytes + */ + public TenantConfiguration(int cpuShare, long maxHeapBytes) { + limitCpuShares(cpuShare); + limitHeap(maxHeapBytes); + } + + /** + * @param maxHeapBytes + */ + public TenantConfiguration(long maxHeapBytes) { + limitHeap(maxHeapBytes); + } + /** * Build TenantConfiguration with given constraints * @param constraints @@ -61,15 +84,68 @@ public class TenantConfiguration { } } - /* - * @return all resource limits specified by this configuration + /** + * Use cgroup's cpu.cfs_* controller to limit cpu time of + * new {@code TenantContainer} object created from this configuration + * @param period corresponding to cpu.cfs_period + * @param quota corresponding to cpu.cfs_quota + * @return current {@code TenantConfiguration} */ - Collection getAllConstraints() { - return constraints.values(); + public TenantConfiguration limitCpuCfs(int period, int quota) { + if (!TenantGlobals.isCpuThrottlingEnabled()) { + throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled"); + } + constraints.put(CPU_CFS, CPU_CFS.newConstraint(period, quota)); + return this; } - void setConstraint(Constraint constraint) { - constraints.put(constraint.getResourceType(), constraint); + /** + * Use cgroup's cpu.shares controller to limit cpu shares of + * new {@code TenantContainer} object created from this configuration + * @param share relative weight of cpu shares + * @return current {@code TenantConfiguration} + */ + public TenantConfiguration limitCpuShares(int share) { + if (!TenantGlobals.isCpuThrottlingEnabled()) { + // use warning messages instead of exception here to keep backward compatibility + System.err.println("WARNING: -XX:+TenantCpuThrottling is disabled!"); + //throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled"); + } + constraints.put(CPU_SHARES, CPU_SHARES.newConstraint(share)); + return this; + } + + /** + * Use cgroup's cpu.cpuset controller to limit cpu cores allowed to be used + * by new {@code TenantContainer} object created from this configuration + * @param cpuSets string of cpuset description, such as "1,2,3", "0-7,11" + * @return current {@code TenantConfiguration} + */ + public TenantConfiguration limitCpuSet(String cpuSets) { + if (!TenantGlobals.isCpuThrottlingEnabled()) { + throw new UnsupportedOperationException("-XX:+TenantCpuThrottling is not enabled"); + } + constraints.put(CPUSET_CPUS, CPUSET_CPUS.newConstraint(parseCpuSetString(cpuSets))); + return this; + } + + private long[] parseCpuSetString(String cpuSets) { + if (cpuSets == null || cpuSets.isEmpty()) { + return null; + } + return Stream.of(cpuSets.split(",")) + .flatMapToLong(s -> { + // for core "1-4", expand to 1,2,3,4 + if (s.contains("-")) { + List ranges = Stream.of(s.split("-")) + .map(Long::parseLong) + .collect(Collectors.toList()); + return LongStream.range(ranges.get(0), ranges.get(1) + 1); + } else { + return LongStream.of(Long.parseLong(s)); + } + }) + .toArray(); } /** @@ -78,10 +154,26 @@ public class TenantConfiguration { * @return current {@code TenantConfiguration} */ public TenantConfiguration limitHeap(long maxJavaHeapBytes) { + if (!TenantGlobals.isHeapThrottlingEnabled()) { + // use warning messages instead of exception here to keep backward compatibility + System.err.println("WARNING: -XX:+TenantHeapThrottling not enabled!"); + //throw new UnsupportedOperationException("-XX:+TenantHeapThrottling is not enabled"); + } constraints.put(HEAP_RETAINED, HEAP_RETAINED.newConstraint(maxJavaHeapBytes)); return this; } + /* + * @return all resource limits specified by this configuration + */ + Collection getAllConstraints() { + return constraints.values(); + } + + void setConstraint(Constraint constraint) { + constraints.put(constraint.getResourceType(), constraint); + } + /** * @return the max amount of heap the tenant is allowed to consume. */ @@ -91,4 +183,31 @@ public class TenantConfiguration { } return Runtime.getRuntime().maxMemory(); } + + /* + * Corresponding to combination of Linux cgroup's cpu.cfs_period_us and cpu.cfs_quota_us + * @return the max percent of cpu the tenant is allowed to consume, -1 means unlimited + */ + public int getMaxCpuPercent() { + if (constraints.containsKey(CPU_CFS)) { + Constraint c = constraints.get(CPU_CFS); + int period = (int) c.getValues()[0]; + int quota = (int) c.getValues()[1]; + if (period > 0 && quota > 0) { + return (int) (((float) quota / (float) period) * 100); + } + } + return -1; + } + + /** + * Corresponding to Linux cgroup's cpu.shares + * @return the weight, impact the ratio of cpu among all tenants. + */ + public int getCpuShares() { + if (constraints.containsKey(CPU_SHARES)) { + return (int) constraints.get(CPU_SHARES).getValues()[0]; + } + return 0; + } } diff --git a/src/share/classes/com/alibaba/tenant/TenantContainer.java b/src/share/classes/com/alibaba/tenant/TenantContainer.java index 4481b0f78..cc25dd189 100644 --- a/src/share/classes/com/alibaba/tenant/TenantContainer.java +++ b/src/share/classes/com/alibaba/tenant/TenantContainer.java @@ -262,6 +262,7 @@ public class TenantContainer { nd.destroyTenantAllocationContext(allocationContext); } + resourceContainer.destroyImpl(); // clear references spawnedThreads.clear(); attachedThreads.clear(); diff --git a/src/share/classes/com/alibaba/tenant/TenantResourceContainer.java b/src/share/classes/com/alibaba/tenant/TenantResourceContainer.java index 8e0d4e84b..7d822c96b 100644 --- a/src/share/classes/com/alibaba/tenant/TenantResourceContainer.java +++ b/src/share/classes/com/alibaba/tenant/TenantResourceContainer.java @@ -49,10 +49,17 @@ class TenantResourceContainer extends AbstractResourceContainer { Constraint c = translate(constraints.remove(ResourceType.CPU_PERCENT)); constraints.put(c.getResourceType(), c); } + jgroup = new JGroup(this); } } private static Constraint translate(Constraint constraint) { + if (constraint.getResourceType() == ResourceType.CPU_PERCENT) { + // translate CPU_PERCENT to cfs_quota_us/cfs_period_us + long period = Long.parseLong(JGroup.jvmGroup().getValue(NativeDispatcher.CG_CPU_CFS_PERIOD)); + long quota = (period * constraint.getValues()[0] / 100L); + return TenantResourceType.CPU_CFS.newConstraint(period, quota); + } return constraint; } @@ -69,22 +76,37 @@ class TenantResourceContainer extends AbstractResourceContainer { */ private TenantContainer tenant; + /* + * tenant jgroup + */ + private volatile JGroup jgroup; + TenantResourceContainer getParent() { return parent; } + JGroup getJGroup() { + return jgroup; + } + TenantContainer getTenant() { return this.tenant; } @Override protected void attach() { + if (jgroup != null) { + jgroup.attach(); + } super.attach(); } @Override protected void detach() { super.detach(); + if (jgroup != null) { + jgroup.detach(); + } } @@ -106,6 +128,13 @@ class TenantResourceContainer extends AbstractResourceContainer { @Override public void updateConstraint(Constraint constraint) { Constraint c = translate(constraint); + if (c.getResourceType() instanceof TenantResourceType) { + TenantResourceType type = (TenantResourceType)c.getResourceType(); + if (type.isJGroupResource() + && jgroup != null) { + ((JGroupConstraint)c).sync(jgroup); + } + } constraints.put(c.getResourceType(), c); } @@ -120,12 +149,19 @@ class TenantResourceContainer extends AbstractResourceContainer { } void destroyImpl() { + if (jgroup != null) { + jgroup.destory(); + jgroup = null; + } parent = null; tenant = null; } // exposed to TenantContainer implementation long getProcessCpuTime() { + if ( jgroup != null) { + return jgroup.getCpuTime(); + } return 0; } } diff --git a/src/share/classes/com/alibaba/tenant/TenantResourceType.java b/src/share/classes/com/alibaba/tenant/TenantResourceType.java index f1cff5fc0..06c2e9d89 100644 --- a/src/share/classes/com/alibaba/tenant/TenantResourceType.java +++ b/src/share/classes/com/alibaba/tenant/TenantResourceType.java @@ -22,12 +22,152 @@ package com.alibaba.tenant; import com.alibaba.rcm.ResourceType; +import com.alibaba.rcm.Constraint; +import java.util.stream.Collectors; +import java.util.Arrays; +import static com.alibaba.tenant.NativeDispatcher.*; /* * Type of resource that can be throttled when MultiTenant feature enabled */ class TenantResourceType extends ResourceType { + + /** + * Throttle CPU resource with cgroup controller: cpu.shares. + * Expecting one constraint value which represents cpu.shares value. + */ + static final ResourceType CPU_SHARES = new TenantResourceType("CPU_SHARE",true) { + @Override + protected void validate(long... values) throws IllegalArgumentException { + if (values == null || values.length != 1 + || values[0] <= 0) { + throw new IllegalArgumentException("Bad CPU_SHARE constraint:" + values[0]); + } + } + + @Override + public Constraint newConstraint(long ... values) { + validate(values); + return new JGroupConstraint(this, values) { + @Override + void sync(JGroup jgroup) { + if (IS_CPU_SHARES_ENABLED) { + jgroup.setValue(CG_CPU_SHARES, Integer.toString((int) getValues()[0])); + } + } + }; + } + }; + + /** + * Throttle CPU resource with cgroup controller: cpuset.cpus. + * Expecting N constraint value which represent cores described in cpuset.cpus. + * e.g. {@code TenantResourceType.CPUSET_CPUS.newConstraint(LongStream.range(0, 16).toArray()) } + */ + static final ResourceType CPUSET_CPUS = new TenantResourceType("CPUSET_CPUS", true) { + @Override + protected void validate(long... values) throws IllegalArgumentException { + if (values == null || values.length > Runtime.getRuntime().availableProcessors()) { + throw new IllegalArgumentException("Invalid number of cpuset.cpus:" + + (values == null ? 0 : values.length)); + } + StringBuilder invalidCores = new StringBuilder(); + for (long v : values) { + if (v < 0 || v >= Runtime.getRuntime().availableProcessors()) { + invalidCores.append(v).append(","); + } + } + String cores = invalidCores.toString(); + if (!cores.isEmpty()) { + throw new IllegalArgumentException("Invalid cpuset cores: " + cores); + } + } + + @Override + public Constraint newConstraint(long... values) { + validate(values); + return new JGroupConstraint(this, values) { + @Override + void sync(JGroup jgroup) { + if (IS_CPUSET_ENABLED) { + String cores = Arrays.stream(getValues()) + .mapToObj(l -> Long.toString(l)) + .collect(Collectors.joining(",")); + jgroup.setValue(CG_CPUSET_CPUS, cores); + } + } + }; + } + }; + + /** + * Throttle CPU resource with cgroup controller: cpu.cfs_quota_us/cpu.cfs_period_us. + * Expecting two constraints: + * [0] corresponding to cpu.cfs_period_us + * [1] corresponding to cpu.cfs_quota_us + */ + static final ResourceType CPU_CFS = new TenantResourceType("CPU_CFS", true) { + @Override + protected void validate(long... values) throws IllegalArgumentException { + if (values == null || values.length != 2) { + throw new IllegalArgumentException("CPU_CFS requires constraints period and quota at the same time"); + } + long period = values[0]; + long quota = values[1]; + // according to https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt + // cpu.cfs_period_us should be in range [1 ms, 1 sec] + if (period < 1_000 || period > 1_000_000) { + throw new IllegalArgumentException("Invalid cpu_cfs.period value:" + period + + ", expected range [1_000, 1_000_000"); + } + if (quota < 1_000 && quota != -1) { + throw new IllegalArgumentException("Invalid cpu_cfs.quota value:" + quota + + ", should >= 1_000 or be unlimited (-1)"); + } + } + + @Override + public Constraint newConstraint(long... values) { + validate(values); + return new JGroupConstraint(this, values) { + @Override + void sync(JGroup jgroup) { + if (IS_CPU_CFS_ENABLED) { + jgroup.setValue(CG_CPU_CFS_PERIOD, Long.toString(values[0])); + jgroup.setValue(CG_CPU_CFS_QUOTA, Long.toString(values[1])); + } + } + }; + } + }; + + /** + * Throttle total number of opened socket file descriptors. + * Expecting one constraint value which is the maximum number of socket FDs. + */ + static final ResourceType SOCKET = new TenantResourceType("SOCKET",false) { + @Override + protected void validate(long... values) throws IllegalArgumentException { + if (values == null || values.length != 1 + || values[0] <= 0) { + throw new IllegalArgumentException("Bad CPU_SHARE constraint:" + values[0]); + } + } + }; + private TenantResourceType(String name, boolean isJGroup) { super(name); + this.isJGroupResource = isJGroup; + } + + // if this type of resource is controlled by JGroup + private boolean isJGroupResource; + + /** + * Check if this type of resource is controlled by cgroup + * @return + */ + boolean isJGroupResource() { + return this.isJGroupResource; } } diff --git a/test/multi-tenant/TestGetAllTenantIDs.java b/test/multi-tenant/TestGetAllTenantIDs.java new file mode 100644 index 000000000..9c976e62d --- /dev/null +++ b/test/multi-tenant/TestGetAllTenantIDs.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* + * @test + * @summary Test data structure integrity while access TenantContainer.tenantContainerMap concurrently + * @library /lib/testlibrary + * @build TestGetAllTenantIDs + * @run main/othervm/timeout=120 -XX:+MultiTenant -Xmx1g -Xms1g TestGetAllTenantIDs + */ + +import com.alibaba.tenant.TenantConfiguration; +import com.alibaba.tenant.TenantContainer; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import static jdk.testlibrary.Asserts.fail; + +public class TestGetAllTenantIDs { + public static void main(String[] args) { + int parallel = Runtime.getRuntime().availableProcessors(); + TenantConfiguration config = new TenantConfiguration(); + List tenants = new LinkedList<>(); + IntStream.range(0, parallel) + .forEach(i -> tenants.add(TenantContainer.create(config))); + + CountDownLatch startTesting = new CountDownLatch(1); + AtomicBoolean terminated = new AtomicBoolean(false); + Thread[] getterThreads = new Thread[parallel]; + Thread[] setterThreads = new Thread[parallel]; + Thread[][] allThreads = { getterThreads, setterThreads }; + IntStream.range(0, parallel) + .forEach(i-> { + getterThreads[i] = new Thread(()->{ + try { startTesting.await(); } catch(InterruptedException e) { fail(); } + int len = 0; + while (!terminated.get()) { + len = TenantContainer.getAllTenantIds().size(); + } + }); + getterThreads[i].start(); + setterThreads[i] = new Thread(()->{ + try { startTesting.await(); } catch(InterruptedException e) { fail(); } + while (!terminated.get()) { + TenantContainer tenant = TenantContainer.create(config); + } + }); + setterThreads[i].start(); + }); + startTesting.countDown(); + + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException e) { + fail(); + } + + terminated.set(true); + + for (Thread[] threads : allThreads) { + Stream.of(threads) + .forEach(t -> { + try { + t.join(); + } catch (Exception e) { + fail(); + } + }); + } + } +} \ No newline at end of file diff --git a/test/multi-tenant/TestJGroupDebugMode.sh b/test/multi-tenant/TestJGroupDebugMode.sh new file mode 100644 index 000000000..763f2a20d --- /dev/null +++ b/test/multi-tenant/TestJGroupDebugMode.sh @@ -0,0 +1,109 @@ +# +# Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. Alibaba designates this +# particular file as subject to the "Classpath" exception as provided +# by Oracle in the LICENSE file that accompanied this code. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# + +#!/usr/bin/env bash + +# +# @test TestJGroupDebugMode.sh +# @summary test debugging mode of JGroup native implementation +# @run shell/timeout=300 TestJGroupDebugMode.sh +# + +set -x + +if [ "${TESTSRC}" = "" ] +then + TESTSRC=${PWD} + echo "TESTSRC not set. Using "${TESTSRC}" as default" +fi +echo "TESTSRC=${TESTSRC}" +FS=/ +JAVA=${TESTJAVA}${FS}bin${FS}java +JAVAC=${TESTJAVA}${FS}bin${FS}javac +TEST_CLASS="Test" +TEST_OPTS="-XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+UseG1GC" + +# generate and compile Java snippet for testing +cat >> ${TEST_CLASS}.java << EOF +import com.alibaba.tenant.*; +public class ${TEST_CLASS} { + public static void main(String[] args) throws TenantException { + TenantConfiguration config = new TenantConfiguration() + .limitCpuShares(1024); + TenantContainer tenant = TenantContainer.create(config); + tenant.run(()-> { + // empty! + }); + } +} +EOF + +${JAVAC} -cp . ${TEST_CLASS}.java +if [ $? != 0 ]; then + echo "Failed to compile ${TEST_CLASS}.java" + exit 1 +fi + +# Test envrionment variable debugging options + +unset JGROUP_DEBUG + +# disable debugging +export JGROUP_DEBUG="" +if [ ! -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then + echo "Failed in non-debug mode" + exit 1 +fi + +# enable debugging +export JGROUP_DEBUG="TRue" +if [ -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then + echo "Failed in debug mode" + exit 1 +fi + +export JGROUP_DEBUG="trUe" +if [ -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'cgroup initialized successfully')" ]; then + echo "Failed in debug mode" + exit 1 +fi + +# Test Java property debugging options +if [ ! -z "$(${JAVA} ${TEST_OPTS} -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers:')" ]; then + echo "Failed in non-debug mode" + exit 1 +fi + +if [ ! -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=false -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then + echo "Failed in debug mode" + exit 1 +fi + +# enable debugging +if [ -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=tRue -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then + echo "Failed in debug mode" + exit 1 +fi + +if [ -z "$(${JAVA} ${TEST_OPTS} -Dcom.alibaba.tenant.debugJGroup=trUE -cp ${PWD} ${TEST_CLASS} | grep 'Created group with standard controllers')" ]; then + echo "Failed in debug mode" + exit 1 +fi diff --git a/test/multi-tenant/TestNoJGroupInit.java b/test/multi-tenant/TestNoJGroupInit.java new file mode 100644 index 000000000..334b300dd --- /dev/null +++ b/test/multi-tenant/TestNoJGroupInit.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ +import java.lang.reflect.Field; + +/* @test + * @summary test scenario where JGroup will not be initialized + * @compile TestNoJGroupInit.java + * @run main/othervm/timeout=300 -XX:+MultiTenant TestNoJGroupInit + */ + + public class TestNoJGroupInit { + public static void main(String[] args) throws Exception { + Class jgroupClazz = Class.forName("com.alibaba.tenant.NativeDispatcher"); + Field f = jgroupClazz.getDeclaredField("CGROUP_INIT_SUCCESS"); + f.setAccessible(true); + Boolean b = (Boolean)f.get(null); + if (b) { + throw new RuntimeException("Should not be initialized!"); + } + } + } \ No newline at end of file diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestCpuCfsThrottling.java b/test/multi-tenant/test/com/alibaba/tenant/TestCpuCfsThrottling.java new file mode 100644 index 000000000..77bda0e75 --- /dev/null +++ b/test/multi-tenant/test/com/alibaba/tenant/TestCpuCfsThrottling.java @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import jdk.testlibrary.TestUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import static jdk.testlibrary.Asserts.*; + +/* + * @test + * @summary test.com.alibaba.tenant.TestCpuCfsThrottling + * @library /lib/testlibrary + * @run main/othervm/bootclasspath -Xint -XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+TenantCpuAccounting + * -Xmx200m -Xms200m com.alibaba.tenant.TestCpuCfsThrottling + */ + +public class TestCpuCfsThrottling { + + private void testCpuCfsQuotas() { + // limit whole JVM process to be running on one CPU core + Path jvmCpusetCpusPath = Paths.get( + JGroup.rootPathOf("cpuset"), + JGroup.JVM_GROUP_PATH, + "cpuset.cpus"); + try { + Files.write(jvmCpusetCpusPath, "0".getBytes() /* limit jvm process to CPU core-#0 */); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + + // create two tenants and start two threads running on same core, and fight for CPU resources + int cfsPeriods[] = {1_000_000, 800_000}; + int cfsQuotas[] = {1_000_000, 300_000}; + TenantConfiguration configs[] = { + new TenantConfiguration().limitCpuCfs(cfsPeriods[0], cfsQuotas[0]).limitCpuSet("0"), + new TenantConfiguration().limitCpuCfs(cfsPeriods[1], cfsQuotas[1]).limitCpuSet("0") + }; + TenantContainer tenants[] = new TenantContainer[configs.length]; + for (int i = 0; i < configs.length; ++i) { + tenants[i] = TenantContainer.create(configs[i]); + } + long counters[] = new long[configs.length]; + + // verify CGroup configurations + for (int i = 0; i < configs.length; ++i) { + int actual = Integer.parseInt(getConfig(tenants[i], "cpu.cfs_period_us")); + assertEquals(cfsPeriods[i], actual); + + actual = Integer.parseInt(getConfig(tenants[i], "cpu.cfs_quota_us")); + assertEquals(cfsQuotas[i], actual); + } + + // Start two counter threads in different TenantContainers + runSerializedCounters(tenants, counters, 10_000); + + // check results + assertGreaterThan(counters[0], 0L); + assertGreaterThan(counters[1], 0L); + assertGreaterThan(counters[0], counters[1]); + assertGreaterThan(counters[0], counters[1] * 2); + } + + private void testAdjustCpuCfsQuotas() { + // limit whole JVM process to be running on one CPU core + Path jvmCpusetCpusPath = Paths.get( + JGroup.rootPathOf("cpuset"), + JGroup.JVM_GROUP_PATH, + "cpuset.cpus"); + try { + Files.write(jvmCpusetCpusPath, "0".getBytes() /* limit jvm process to CPU core-#0 */); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + + // create two tenants and start two threads running on same core, and fight for CPU resources + TenantConfiguration configs[] = { + new TenantConfiguration().limitCpuCfs(1_000_000, 800_000).limitCpuSet("0"), + new TenantConfiguration().limitCpuCfs(1_000_000, 300_000).limitCpuSet("0") + }; + TenantContainer tenants[] = new TenantContainer[configs.length]; + for (int i = 0; i < configs.length; ++i) { + tenants[i] = TenantContainer.create(configs[i]); + } + long counters[] = new long[configs.length]; + + // Start two counter threads in different TenantContainers + runSerializedCounters(tenants, counters, 10_000); + + // check results + assertGreaterThan(counters[0], 0L); + assertGreaterThan(counters[1], 0L); + assertGreaterThan(counters[0], counters[1]); + assertGreaterThan(counters[0], counters[1] * 2); + + // cfs limitation adjustment needs time to take effect, here we sleep for a while! + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + + // ======== round 2 testing, after modify resouce limitations ========== + tenants[0].update(configs[0].limitCpuCfs(1_000_000, 300_000)); + tenants[1].update(configs[1].limitCpuCfs(1_000_000, 800_000)); + counters[0] = 0L; + counters[1] = 0L; + runSerializedCounters(tenants, counters, 10_000); + // check results + assertGreaterThan(counters[0], 0L); + assertGreaterThan(counters[1], 0L); + assertGreaterThan(counters[1], counters[0]); + assertGreaterThan(counters[1], counters[0] * 2); + } + + // run several counters in concurrent threads + private static void runConcurrentCounters(TenantContainer[] tenants, + long[] counters, + long milliLimit) { + if (tenants == null || counters == null) { + throw new IllegalArgumentException("Bad args"); + } + // Start two counter threads in different TenantContainers + CountDownLatch startCounting = new CountDownLatch(1); + AtomicBoolean stopCounting = new AtomicBoolean(false); + Thread threadRefs[] = new Thread[2]; + for (int i = 0; i < tenants.length; ++i) { + TenantContainer tenant = tenants[i]; + int idx = i; + try { + // start a single non-root tenant thread to execute counter + tenant.run(()-> { + threadRefs[idx] = new Thread(()->{ + try { + startCounting.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + while (!stopCounting.get()) { + ++counters[idx]; + } + }); + threadRefs[idx].start(); + }); + } catch (TenantException e) { + e.printStackTrace(); + fail(); + } + } + + // ROOT tenant thread will serve as monitor & controller + startCounting.countDown(); + try { + // the absolute exec time limits for two tenant threads are same + Thread.sleep(milliLimit); + stopCounting.set(true); + for (Thread t : threadRefs) { + t.join(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + } + + // execute an action after certain milli seconds, clock start ticking after notified by startLatch + private static void runTimedTask(long afterMillis, CountDownLatch startLatch, Runnable action) { + Thread timer = new Thread(() -> { + if (startLatch != null) { + try { + startLatch.await(); + } catch (InterruptedException e) { + // ignore + } + } + + long start = System.currentTimeMillis(); + long passed = 0; + while (passed < afterMillis) { + try { + Thread.sleep(afterMillis - passed); + } catch (InterruptedException e) { + //ignore + } finally { + passed = System.currentTimeMillis() - start; + } + } + // performan action after specific time + action.run(); + }); + timer.start(); + } + + private static void runSerializedCounters(TenantContainer[] tenants, + long[] counters, + long milliLimit) { + if (tenants == null || counters == null) { + throw new IllegalArgumentException("Bad args"); + } + + AtomicBoolean stopCounting = new AtomicBoolean(false); + + for (int i = 0; i < tenants.length; ++i) { + int idx = i; + stopCounting.set(false); + CountDownLatch startCounting = new CountDownLatch(1); + TenantContainer tenant = tenants[i]; + runTimedTask(milliLimit, startCounting, ()-> { + stopCounting.set(true); + }); + try { + // start a single non-root tenant thread to execute counter + tenant.run(() -> { + startCounting.countDown(); + while (!stopCounting.get()) { + ++counters[idx]; + } + }); + } catch (TenantException e) { + e.printStackTrace(); + fail(); + } + } + } + + public static void main(String[] args) { + TestUtils.runWithPrefix("test", + TestCpuCfsThrottling.class, + new TestCpuCfsThrottling()); + } + + //------ utililties + private String getConfig(TenantContainer tenant, String configName) { + String controllerName = configName.split("\\.")[0]; + Path configPath = Paths.get(JGroup.rootPathOf(controllerName), + JGroup.JVM_GROUP_PATH, + "t" + tenant.getTenantId(), + configName); + try { + return new String(Files.readAllBytes(configPath)).trim(); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + return null; + } +} diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestHierachicalTenants.java b/test/multi-tenant/test/com/alibaba/tenant/TestHierachicalTenants.java new file mode 100644 index 000000000..104a046b4 --- /dev/null +++ b/test/multi-tenant/test/com/alibaba/tenant/TestHierachicalTenants.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import jdk.testlibrary.TestUtils; +import java.util.concurrent.CountDownLatch; +import static jdk.testlibrary.Asserts.*; + +/* + * @test + * @summary Test hierarchical tenants support + * @library /lib/testlibrary + * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuThrottling -Xmx200m -Xms200m + * com.alibaba.tenant.TestHierachicalTenants + */ + +public class TestHierachicalTenants { + + private void testParentLimit() { + TenantConfiguration config = new TenantConfiguration() + .limitCpuSet("0") // forcefully bind to cpu-0 + .limitCpuCfs(200000, 100000); + TenantContainer parent = TenantContainer.create(config); + + long timeLimitMillis = 5_000; + long[] counts = new long[5]; + try { + // counts[counts.length - 1] is the baseline number + parent.run(()->{ + // the primary counter loop-1 + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeLimitMillis) { + ++counts[counts.length - 1]; + } + }); + + // create two children tenant containers, the overall limit should not exceed parent limit + CountDownLatch start = new CountDownLatch(1); + TenantContainer[] childrenTenants = new TenantContainer[counts.length - 1]; + Thread[] threads = new Thread[childrenTenants.length]; + + for (int i = 0; i < childrenTenants.length; ++i) { + final int idx = i; + parent.run(()->{ + childrenTenants[idx] = TenantContainer.create(config); + }); + childrenTenants[i].run(()->{ + threads[idx] = new Thread(()->{ + try { + start.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + // below counter loop must be identical to parent's primary counter loop-1 + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeLimitMillis) { + ++counts[idx]; + } + }); + threads[idx].start(); + }); + } + + // trigger children tester thread + start.countDown(); + + // join all children + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + fail(); + } + } + + System.out.println("counts:"); + for (long cnt : counts) { System.out.println(cnt); } + + // check result + long sum = 0; + double acceptableDiffPercent = 0.2; + for (int i = 0; i < counts.length - 1; ++i) { + assertLessThan(counts[i], counts[counts.length - 1]); + sum += counts[i]; + } + + assertLessThan(Math.abs(sum - (double)counts[counts.length - 1]), counts[counts.length - 1] * acceptableDiffPercent); + + } catch (TenantException e) { + e.printStackTrace(); + fail(); + } + } + + private void testParentBasic() { + TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024); + TenantContainer parent = TenantContainer.create(config); + assertNull(getParent(parent)); + TenantContainer children[] = new TenantContainer[1]; + try { + parent.run(()->{ + children[0] = TenantContainer.create(config); + }); + } catch (TenantException e) { + e.printStackTrace(); + fail(); + } + assertTrue(getParent(children[0]) == parent); + children[0].destroy(); + assertNull(getParent(children[0])); + assertNull(getParent(parent)); + parent.destroy(); + } + + private void testNullParent() { + TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024); + TenantContainer tenant = TenantContainer.create(config); + assertNull(getParent(tenant)); + tenant.destroy(); + } + + public static void main(String[] args) { + TestUtils.runWithPrefix("test", + TestHierachicalTenants.class, + new TestHierachicalTenants()); + } + + // -------------- utilities --------------- + private static TenantContainer getParent(TenantContainer config) { + if (config != null) { + TenantResourceContainer parentRes = config.resourceContainer.getParent(); + if (parentRes != null) { + return parentRes.getTenant(); + } + } + return null; + } +} diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestJGroup.java b/test/multi-tenant/test/com/alibaba/tenant/TestJGroup.java new file mode 100644 index 000000000..d153f85ea --- /dev/null +++ b/test/multi-tenant/test/com/alibaba/tenant/TestJGroup.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ +package com.alibaba.tenant; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.management.ManagementFactory; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import static java.nio.file.StandardOpenOption.WRITE; + +/* + * @test + * @summary Test JGroup + * @library /lib/testlibrary + * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuAccounting -XX:+TenantCpuThrottling -XX:+UseG1GC -Xmx200m -Xms200m -Dcom.alibaba.tenant.DebugJGroup=true com.alibaba.tenant.TestJGroup + */ + +class JGroupWorker implements Runnable{ + private long id; + public long count = 0; + public JGroup group; + + JGroupWorker(long id){ + this.id = id; + } + public void run(){ + TenantContainer tenant = TenantContainer.create( + new TenantConfiguration() + .limitCpuShares((int)id * 512 + 512) + .limitCpuSet("0")); // limit all tasks to one CPU to forcefully create contention + group = new JGroup(tenant.resourceContainer); + group.attach(); + long msPre = System.currentTimeMillis(); + while(System.currentTimeMillis() - msPre < (20000 - 1000 * id)){ + count++; + } + } +} + +class JGroupsWorker implements Runnable{ + private JGroup t1; + private JGroup t2; + private JGroup t3; + public long count1 = 0; + public long count2 = 0; + public long count3 = 0; + + JGroupsWorker(JGroup t1, JGroup t2, JGroup t3){ + this.t1 = t1; + this.t2 = t2; + this.t3 = t3; + } + public void run(){ + t1.attach(); + long msPre = System.currentTimeMillis(); + while(System.currentTimeMillis() - msPre < 5000){ + count1++; + } + t1.detach(); + + t2.attach(); + msPre = System.currentTimeMillis(); + while(System.currentTimeMillis() - msPre < 4000){ + count2++; + } + t2.detach(); + + t3.attach(); + msPre = System.currentTimeMillis(); + while(System.currentTimeMillis() - msPre < 3000){ + count3++; + } + t3.detach(); + } +} + + +public class TestJGroup { + + private static void setUp() { + JGroup.jvmGroup().setValue("cpuset.cpus", "0"); + } + + static void mySleep(int t) { + try { + Thread.sleep(t); + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + } + + public static void main(String[] args) throws Exception { + setUp(); + + long usage1 = 0; + long usage2 = 0; + long usage3 = 0; + JGroupWorker jw1 = new JGroupWorker(0); + JGroupWorker jw2 = new JGroupWorker(1); + JGroupWorker jw3 = new JGroupWorker(2); + Thread t1 = new Thread(jw1); + Thread t2 = new Thread(jw2); + Thread t3 = new Thread(jw3); + t1.start(); + t2.start(); + t3.start(); + + mySleep(1000); + + JGroupsWorker js = new JGroupsWorker(jw1.group, + jw2.group, + jw3.group); + Thread t4 =new Thread(js); + t4.start(); + try { + t4.join(); + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + + try { + t1.join(); + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + usage1 = jw1.group.getCpuTime(); + jw1.group.destory(); + + if(usage1 <= 0 ) { + throw new Exception("Invalid cpu usage, usage1: "+ usage1); + } + try { + t2.join(); + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + usage2 = jw2.group.getCpuTime(); + jw2.group.destory(); + + if(usage2 <= 0 ) { + throw new Exception("Invalid cpu usage, usage2: "+ usage2); + } + try { + t3.join(); + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + usage3 = jw3.group.getCpuTime(); + jw3.group.destory(); + + if(usage3 <= 0 ) { + throw new Exception("Invalid cpu usage, usage3: "+ usage3); + } + + if ((js.count3 < js.count2) || (js.count2 < js.count1)) { + throw new Exception("com.alibaba.tenant.JGroupsWorker test failed!" + + " count1 = " + js.count1 + + " count2 = " + js.count2 + + " count3 = " + js.count3); + } + + if ((jw3.count < jw2.count) || (jw2.count < jw1.count)) { + throw new Exception("Three com.alibaba.tenant.JGroupWorker test failed!" + + " jw1 count = " + jw1.count + + " jw2 count = " + jw2.count + + " jw3 count = " + jw3.count); + } + + if ((usage3 < usage2) || (usage2 < usage1)) { + throw new Exception("Three com.alibaba.tenant.JGroupWorker test failed!" + + " jw1 usage = " + usage1 + + " jw2 usage = " + usage2 + + " jw3 usage = " + usage3); + } + } +} diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestJGroupInit.java b/test/multi-tenant/test/com/alibaba/tenant/TestJGroupInit.java new file mode 100644 index 000000000..d9210791b --- /dev/null +++ b/test/multi-tenant/test/com/alibaba/tenant/TestJGroupInit.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +package com.alibaba.tenant; + +import jdk.testlibrary.TestUtils; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; +import static jdk.testlibrary.Asserts.*; + +/* + * @test + * @summary Test initialization code of JGroup + * @library /lib/testlibrary + * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+TenantCpuThrottling -XX:+UseG1GC + * -Dcom.alibaba.tenant.DebugJGroup=true com.alibaba.tenant.TestJGroupInit + */ +public class TestJGroupInit { + + private void testJGroupHierarchy() { + TenantConfiguration config = new TenantConfiguration() + .limitCpuShares(1024) + .limitHeap(64 * 1024 * 1024) + .limitCpuCfs(1_000_000, 500_000); + TenantContainer tenant = TenantContainer.create(config); + try { + Field f = TenantResourceContainer.class.getDeclaredField("jgroup"); + f.setAccessible(true); + JGroup jgroup = (JGroup)f.get((TenantResourceContainer)(tenant.getResourceContainer())); + + assertNotNull(jgroup); // tenant group + assertNotNull(jgroup.parent()); // jvm group + assertNotNull(jgroup.parent().parent()); // jdk group + assertNotNull(jgroup.parent().parent().parent()); // root group + assertNull(jgroup.parent().parent().parent().parent()); // null + } catch (NoSuchFieldException | IllegalAccessException e) { + e.printStackTrace(); + } + } + + private void testRootGroupExists() { + Path rootGroupPath = getRootCGroupPathOf("cpu"); + System.err.println("rootpath=" + rootGroupPath); + assertTrue(Files.exists(rootGroupPath) && Files.isDirectory(rootGroupPath), + rootGroupPath + " cannot be found or is not a directory"); + assertTrue(Files.exists(rootGroupPath.resolve("cpu.shares")), "cpu.shares cannot be found"); + } + + private void testNonRootGroupExists() { + Path rootGroupPath = getRootCGroupPathOf("cpu"); + System.err.println("rootpath=" + rootGroupPath); + assertTrue(Files.exists(rootGroupPath) && Files.isDirectory(rootGroupPath)); + + int cpuShare = 1024; + long heapLimit = 128 * 1024 * 1024; + int cpuCfsPeriod = 1_000_000; + int cpuCfsQuota = 500_000; + + TenantConfiguration config = new TenantConfiguration() + .limitCpuShares(cpuShare) + .limitHeap(heapLimit) + .limitCpuCfs(cpuCfsPeriod, cpuCfsQuota); + TenantContainer tenant = TenantContainer.create(config); + Path tenantPath = rootGroupPath.resolve("t" + tenant.getTenantId()); + assertTrue(Files.exists(tenantPath) && Files.isDirectory(tenantPath)); + + // verify CGroup configurations + { + int actual = Integer.parseInt(getConfig(tenant, "cpu.cfs_period_us")); + assertEquals(cpuCfsPeriod, actual); + + actual = Integer.parseInt(getConfig(tenant, "cpu.cfs_quota_us")); + assertEquals(cpuCfsQuota, actual); + + actual = Integer.parseInt(getConfig(tenant, "cpu.shares")); + assertEquals(cpuShare, actual); + } + + assertTrue(Files.exists(tenantPath) && Files.isDirectory(tenantPath), + tenantPath + " cannot be found or is not a directory"); + Path path = tenantPath.resolve("cpu.shares"); + assertTrue(Files.exists(path), path + " cannot be found"); + + try { + tenant.run(()->{ + System.out.println("Hello from simple tenant"); + }); + } catch (TenantException e) { + e.printStackTrace(); + fail(); + } finally { + tenant.destroy(); + } + + assertTrue(!Files.exists(tenantPath)); + assertTrue(!Files.exists(tenantPath.resolve("cpu.shares"))); + } + + private void testGetSetNonExistConfigs() { + TenantConfiguration config = new TenantConfiguration() + .limitCpuShares(1024) + .limitHeap(128 * 1024 * 1024); + TenantContainer tenant = TenantContainer.create(config); + JGroup jgroup = tenant.resourceContainer.getJGroup(); + + // controller name does not have separator "." + try { + jgroup.setValue("NonExistConfigName", "456"); + } catch (IllegalArgumentException e) { + // ignore + } + + // controller name does have separator ".", and looks like normal + try { + jgroup.setValue("NonExist.controller", "456"); + fail(); + } catch (Throwable e) { + // ignore + } + + // controller name does not have separator "." + try { + String val = jgroup.getValue("NonExistConfigfsdjkfksdj123rName"); + } catch (IllegalArgumentException e) { + // ignore + } + + // controller name does have separator ".", and looks like normal + try { + String val = jgroup.getValue("NonExistConf.ksdj123rName"); + fail(); + } catch (Throwable e) { + } + + jgroup.setValue("cpu.shares", "1023"); + assertEquals("1023", jgroup.getValue("cpu.shares")); + } + + private void testGetSetJVMGroupControllers() { + assertNotNull(JGroup.jdkGroup()); + assertNotNull(JGroup.jvmGroup()); + + TenantConfiguration config = new TenantConfiguration(); + TenantContainer tenant = TenantContainer.create(config); + JGroup group = new JGroup(tenant.resourceContainer); + assertTrue(group.parent() == JGroup.jvmGroup()); + + JGroup jvmGroup = JGroup.jvmGroup(); + jvmGroup.setValue("cpu.shares", "1023"); + assertEquals(jvmGroup.getValue("cpu.shares"), "1023"); + jvmGroup.setValue("cpu.shares", "1024"); + assertEquals(jvmGroup.getValue("cpu.shares"), "1024"); + } + + public static void main(String[] args) { + TestUtils.runWithPrefix("test", + TestJGroupInit.class, + new TestJGroupInit()); + } + + //---------utilities + private static volatile Map rootCGroupPathMap; + + private static Path getRootCGroupPathOf(String controllerName) { + if (rootCGroupPathMap == null) { + synchronized (TestJGroupInit.class) { + if (rootCGroupPathMap == null) { + rootCGroupPathMap = new HashMap<>(); + String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + assertNotNull(processName); + long pid = Long.parseLong(processName.split("@")[0]); + assertGreaterThan(pid, 0L); + + try { + Files.readAllLines(Paths.get("/proc/mounts")) + .stream().forEach(line -> { + if (line.startsWith("cgroup ")) { + StringTokenizer st = new StringTokenizer(line); + st.nextToken(); + String tok = st.nextToken(); + assertNotNull(tok); + Path curPath = Paths.get(tok, "ajdk_multi_tenant", "" + pid); + if (line.contains("cpu,") || line.contains("cpu ")) { + rootCGroupPathMap.put("cpu", curPath); + } else if (line.contains("cpuset")) { + rootCGroupPathMap.put("cpuset", curPath); + } else if (line.contains("cpuacct")) { + rootCGroupPathMap.put("cpuacct", curPath); + } + } + }); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + } + } + } + return rootCGroupPathMap.get(controllerName); + } + + private String getConfig(TenantContainer tenant, String configName) { + String controllerName = configName.split("\\.")[0]; + Path configPath = Paths.get(JGroup.rootPathOf(controllerName), + JGroup.JVM_GROUP_PATH, + "t" + tenant.getTenantId(), + configName); + try { + return new String(Files.readAllBytes(configPath)).trim(); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + return null; + } +} diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestTenantConfiguration.java b/test/multi-tenant/test/com/alibaba/tenant/TestTenantConfiguration.java new file mode 100644 index 000000000..668e53bdf --- /dev/null +++ b/test/multi-tenant/test/com/alibaba/tenant/TestTenantConfiguration.java @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020 Alibaba Group Holding Limited. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Alibaba designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ +package com.alibaba.tenant; + +/* @test + * @summary test TenantConfiguration facilities + * @library /lib/testlibrary + * @run main/othervm/bootclasspath -XX:+MultiTenant -XX:+UseG1GC -XX:+TenantCpuThrottling + * com.alibaba.tenant.TestTenantConfiguration + */ + +import com.alibaba.rcm.Constraint; +import com.alibaba.rcm.ResourceType; +import jdk.testlibrary.TestUtils; +import java.util.LinkedList; +import java.util.List; +import static jdk.testlibrary.Asserts.*; + +public class TestTenantConfiguration { + + private void testBasic() { + try { + TenantConfiguration config = new TenantConfiguration() + .limitCpuCfs(1_000_000, 500_000) + .limitCpuShares(1024) + .limitHeap(64 * 1024 * 1024); + assertNotNull(config); + TenantContainer tenant = TenantContainer.create(config); + assertNotNull(tenant); + tenant.destroy(); + } catch (Throwable e) { + e.printStackTrace(); + fail(); + } + } + + private void testIllegalLimits() { + Runnable illegalActions[] = { + ()->{ new TenantConfiguration().limitCpuSet(""); }, + ()->{ new TenantConfiguration().limitCpuSet(null); }, + ()->{ new TenantConfiguration().limitCpuShares(-128); }, + ()->{ new TenantConfiguration().limitHeap(-64 * 1024 * 1024); }, + ()->{ new TenantConfiguration().limitCpuCfs(-123, 10_000_000); }, + ()->{ new TenantConfiguration().limitCpuCfs(1_000_000, -2); }, + ()->{ new TenantConfiguration().limitCpuCfs(999, -1); }, // lower bound of period + ()->{ new TenantConfiguration().limitCpuCfs(1_000_001, -1); }, // upper bound of period + ()->{ new TenantConfiguration().limitCpuCfs(10_000, 999); }, // lower bound of quota + ()->{ new TenantConfiguration(-1024 * 1024 * 1024); }, + ()->{ new TenantConfiguration(-12, 64 * 1024 * 1024); }, + }; + for (Runnable action : illegalActions) { + try { + action.run(); + fail("should throw IllegalArgumentException"); + } catch (IllegalArgumentException e) { + // expected + } catch (Throwable e) { + e.printStackTrace(); + fail("Should not throw any other exceptions"); + } + } + + } + + private void testValidLimits() { + Runnable validActions[] = { + ()->{ new TenantConfiguration().limitCpuCfs(1_000_000, 2_000_000); }, + ()->{ new TenantConfiguration().limitCpuCfs(100_000, 10_000); }, + ()->{ new TenantConfiguration().limitCpuCfs(100_000, -1); } + }; + for (Runnable action : validActions) { + try { + action.run(); + } catch (Throwable e) { + e.printStackTrace(); + fail("Should not throw any exceptions"); + } + } + } + + private void testGetSetLimits() { + TenantConfiguration config = new TenantConfiguration(); + assertNotNull(config.getAllConstraints()); + assertTrue(config.getAllConstraints().isEmpty()); + + long maxHeap = 64 * 1024 * 1024; + + config.limitHeap(maxHeap); + List constraints = new LinkedList<>(); + config.getAllConstraints().forEach(c -> constraints.add(c)); + + assertNotNull(config.getAllConstraints()); + assertTrue(config.getAllConstraints().size() == 1); + Constraint constraint = constraints.get(0); + assertEquals(constraint.getResourceType(), ResourceType.HEAP_RETAINED); + assertTrue(constraint.getResourceType() instanceof ResourceType); + assertEquals(constraint.getValues()[0], maxHeap); + assertEquals(constraints.size(), 1); + + config.limitCpuShares(1024); + constraints.clear(); + config.getAllConstraints().forEach(c -> constraints.add(c)); + assertTrue(constraints.stream().anyMatch(c -> c.getResourceType() == TenantResourceType.CPU_SHARES)); + int nofCPUShares = (int) constraints.stream().filter(c -> c.getResourceType() == TenantResourceType.CPU_SHARES) + .peek(c -> assertEquals(c.getValues()[0], 1024L)) + .count(); + assertEquals(nofCPUShares, 1); + } + + private void testUniqueConfigWhenCreatingContainer() { + TenantConfiguration config = new TenantConfiguration().limitCpuShares(1024).limitHeap(64 * 1024 * 1024); + TenantContainer tenant = TenantContainer.create(config); + assertTrue(tenant.getConfiguration() == config); + } + + public static void main(String[] args) { + TestUtils.runWithPrefix("test", + TestTenantConfiguration.class, + new TestTenantConfiguration()); + } +} diff --git a/test/multi-tenant/test/com/alibaba/tenant/TestTenantContainer.java b/test/multi-tenant/test/com/alibaba/tenant/TestTenantContainer.java index 5952b5817..76439e281 100644 --- a/test/multi-tenant/test/com/alibaba/tenant/TestTenantContainer.java +++ b/test/multi-tenant/test/com/alibaba/tenant/TestTenantContainer.java @@ -199,6 +199,100 @@ public class TestTenantContainer { tenant.destroy(); } + class TenantWorker implements Runnable{ + public Tenant tenant; + public long cpuTime; + + public long getCpuTime () { + ThreadMXBean bean = ManagementFactory.getThreadMXBean(); + return bean.isCurrentThreadCpuTimeSupported() ? bean + .getCurrentThreadCpuTime() : 0L; + } + + TenantWorker(Tenant t) { + tenant = t; + } + + public void run(){ + while(System.currentTimeMillis() - tenant.pre_ms < 3000){ + tenant.nextCount(); + } + cpuTime = getCpuTime(); + } + } + + class Tenant implements Runnable{ + public AtomicLong count = new AtomicLong(0); + public long pre_ms = 0; + TenantWorker workers[]; + int max_cpu; + Thread threads[]; + long time = 0; + + public long nextCount() { + return count.incrementAndGet(); + } + + public void run(){ + max_cpu = Runtime.getRuntime().availableProcessors(); + workers = new TenantWorker[max_cpu]; + threads = new Thread[max_cpu]; + pre_ms = System.currentTimeMillis(); + for (int i = 0; i < max_cpu; i++) { + workers[i] = new TenantWorker(this); + threads[i] = new Thread(workers[i]); + threads[i].start(); + } + } + public long getcpuTime(){ + for (int i = 0; i < max_cpu; i++) { + try { + threads[i].join(); + time += workers[i].cpuTime; + } catch (InterruptedException e) { + System.out.println("Interreupted..."); + } + } + return time; + } + } + + @Test + public void testTenantCpuThrottling() throws TenantException { + if (!TenantGlobals.isCpuThrottlingEnabled()) { + return; + } + + long time0 = 0; + long time1 = 0; + // put two tenants on same CPU core + TenantConfiguration tconfig0 = new TenantConfiguration().limitCpuShares(512).limitCpuSet("0"); + TenantConfiguration tconfig1 = new TenantConfiguration().limitCpuShares(1024).limitCpuSet("0"); + TenantContainer container0 = TenantContainer.create(tconfig0); + TenantContainer container1 = TenantContainer.create(tconfig1); + Tenant tenant0 = new Tenant(); + Tenant tenant1 = new Tenant(); + + container0.run(tenant0); + container1.run(tenant1); + + time1 = tenant1.getcpuTime(); + time0 = tenant0.getcpuTime(); + + System.out.println("testTenantCpuThrottling tenant0 count: " + tenant0.count + " tenant0 CpuTime:" + time0 + + " tenant1 count: " + tenant1.count + " tenant1 CpuTime:" + time1); + double rate = 0.0; + if (TenantGlobals.isCpuThrottlingEnabled()) { + rate = (double)time1 / (time0 * 2); + if ((rate > 1.1) || (rate < 0.9)) { + fail(); + } + } else { + rate = (double)time1 / time0; + } + System.out.println("CPU throttling enabled: "+ TenantGlobals.isCpuThrottlingEnabled()+ ", rate = " + rate); + } + @Test public void testTenantInheritance() throws TenantException { TenantConfiguration tconfig = new TenantConfiguration(); @@ -339,6 +433,7 @@ public class TestTenantContainer { test.testGetAttachedThreads(); test.testRun(); test.testCurrentWithGC(); + test.testTenantCpuThrottling(); test.testTenantInheritance(); test.testRunInRootTenant(); test.testTenantGetAllocatedMemory(); -- GitLab