From a352d35b1cbed0abd230dde265c3af06516e4cf4 Mon Sep 17 00:00:00 2001 From: LiuHao Date: Mon, 10 Feb 2020 18:35:53 +0800 Subject: [PATCH] finish share ipc 1. --ipc=host: mount "/dev/shm" of host into container 2. --ipc=shareable: create shm and mount into container Signed-off-by: LiuHao --- iSulad.spec | 2 +- src/contrib/config/config.json | 12 - .../config/systemcontainer_config.json | 12 - .../schema/schema/container/config-v2.json | 3 + src/namespace.c | 2 +- src/namespace.h | 11 + src/services/execution/execute/execution.c | 20 ++ .../execution/execute/execution_create.c | 3 + .../execution/manager/container_unix.c | 42 +++ .../execution/manager/container_unix.h | 2 + src/services/execution/spec/specs_mount.c | 264 +++++++++++++++++- 11 files changed, 340 insertions(+), 33 deletions(-) diff --git a/iSulad.spec b/iSulad.spec index 699bad3..6a7be61 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 1.1.11 -%global _release 20200204.221506.git50cfadfa +%global _release 20200213.091554.git2cbd4087 %global is_systemd 1 %global debug_package %{nil} diff --git a/src/contrib/config/config.json b/src/contrib/config/config.json index 9a9ec87..462f4ec 100644 --- a/src/contrib/config/config.json +++ b/src/contrib/config/config.json @@ -96,18 +96,6 @@ "ro" ] }, - { - "destination": "/dev/shm", - "type": "tmpfs", - "source": "shm", - "options": [ - "nosuid", - "noexec", - "nodev", - "mode=1777", - "size=65536k" - ] - }, { "destination": "/sys/fs/cgroup", "type": "cgroup", diff --git a/src/contrib/config/systemcontainer_config.json b/src/contrib/config/systemcontainer_config.json index 3d2bd89..b469a88 100644 --- a/src/contrib/config/systemcontainer_config.json +++ b/src/contrib/config/systemcontainer_config.json @@ -96,18 +96,6 @@ "ro" ] }, - { - "destination": "/dev/shm", - "type": "tmpfs", - "source": "shm", - "options": [ - "nosuid", - "noexec", - "nodev", - "mode=1777", - "size=65536k" - ] - }, { "destination": "/sys/fs/cgroup", "type": "cgroup", diff --git a/src/json/schema/schema/container/config-v2.json b/src/json/schema/schema/container/config-v2.json index 173886b..623e4e5 100644 --- a/src/json/schema/schema/container/config-v2.json +++ b/src/json/schema/schema/container/config-v2.json @@ -40,6 +40,9 @@ "ResolvConfPath": { "type": "string" }, + "ShmPath": { + "type": "string" + }, "LogPath": { "type": "string" }, diff --git a/src/namespace.c b/src/namespace.c index 9f13b63..3374ae3 100644 --- a/src/namespace.c +++ b/src/namespace.c @@ -21,7 +21,7 @@ #include "containers_store.h" -static char *connected_container(const char *mode) +char *connected_container(const char *mode) { const char *p = mode != NULL ? (mode + strlen(SHARE_NAMESPACE_PREFIX)) : NULL; diff --git a/src/namespace.h b/src/namespace.h index 7a7dc40..c684982 100644 --- a/src/namespace.h +++ b/src/namespace.h @@ -21,6 +21,7 @@ #define SHARE_NAMESPACE_PREFIX "container:" #define SHARE_NAMESPACE_HOST "host" #define SHARE_NAMESPACE_NONE "none" +#define SHARE_NAMESPACE_SHAREABLE "shareable" #define SHARE_NAMESPACE_PID_HOST_PATH "/proc/1/ns/pid" #define SHARE_NAMESPACE_NET_HOST_PATH "/proc/1/ns/net" @@ -68,6 +69,16 @@ static inline bool is_container(const char *mode) return false; } +static inline bool is_shareable(const char *mode) +{ + if (mode != NULL && strcmp(mode, SHARE_NAMESPACE_SHAREABLE) == 0) { + return true; + } + return false; +} + +char *connected_container(const char *mode); + char *get_share_namespace_path(const char *type, const char *src_path); #endif diff --git a/src/services/execution/execute/execution.c b/src/services/execution/execute/execution.c index 40e4376..45afa5a 100644 --- a/src/services/execution/execute/execution.c +++ b/src/services/execution/execute/execution.c @@ -52,6 +52,7 @@ #include "specs_extend.h" #include "utils.h" #include "error.h" +#include "namespace.h" static int filter_by_label(const container_t *cont, const container_get_id_request *request) @@ -1631,6 +1632,23 @@ out: return ret; } +static void umouont_share_shm(container_t *cont) +{ + if (has_mount_for(cont, "/dev/shm")) { + return; + } + if (cont->hostconfig->ipc_mode == NULL || is_shareable(cont->hostconfig->ipc_mode)) { + if (cont->common_config == NULL || cont->common_config->shm_path == NULL) { + return; + } + + INFO("Umounting share shm: %s", cont->common_config->shm_path); + if (umount2(cont->common_config->shm_path, MNT_DETACH)) { + ERROR("Failed to umount the target: %s", cont->common_config->shm_path); + } + } +} + static int do_cleanup_container_resources(container_t *cont) { int ret = 0; @@ -1686,6 +1704,8 @@ static int do_cleanup_container_resources(container_t *cont) goto out; } + umouont_share_shm(cont); + umount_host_channel(cont->hostconfig->host_channel); if (do_runtime_rm_helper(id, runtime, rootpath) != 0) { diff --git a/src/services/execution/execute/execution_create.c b/src/services/execution/execute/execution_create.c index 9e12bdd..da437c8 100644 --- a/src/services/execution/execute/execution_create.c +++ b/src/services/execution/execute/execution_create.c @@ -771,6 +771,9 @@ int container_create_cb(const container_create_request *request, cc = ISULAD_ERR_INPUT; goto clean_container_root_dir; } + // update runtime of host config + free(host_spec->runtime); + host_spec->runtime = util_strdup_s(runtime); v2_spec = util_common_calloc_s(sizeof(container_config_v2_common_config)); if (v2_spec == NULL) { diff --git a/src/services/execution/manager/container_unix.c b/src/services/execution/manager/container_unix.c index 12035f8..1865362 100644 --- a/src/services/execution/manager/container_unix.c +++ b/src/services/execution/manager/container_unix.c @@ -1054,4 +1054,46 @@ out: return ret; } +/* + * @cont: check container + * @mpath: target mount path + * */ +bool has_mount_for(container_t *cont, const char *mpath) +{ + size_t i = 0; + char *work = NULL; + + if (cont == NULL || mpath == NULL) { + return false; + } + + if (cont->common_config == NULL) { + return false; + } + + if (cont->common_config->mount_points == NULL) { + return false; + } + + for (; i < cont->common_config->mount_points->len; i++) { + if (strcmp(cont->common_config->mount_points->keys[i], mpath) == 0) { + return true; + } + } + + if (cont->hostconfig == NULL) { + return false; + } + for (i = 0; i < cont->hostconfig->binds_len; i++) { + work = strrchr(cont->hostconfig->binds[i], ':'); + if (work == NULL) { + continue; + } + if (strcmp(work, mpath) == 0) { + return true; + } + } + + return false; +} diff --git a/src/services/execution/manager/container_unix.h b/src/services/execution/manager/container_unix.h index 1f7f950..c509373 100644 --- a/src/services/execution/manager/container_unix.h +++ b/src/services/execution/manager/container_unix.h @@ -108,6 +108,8 @@ int save_config_v2_json(const char *id, const char *rootpath, const char *v2conf int container_read_proc(uint32_t pid, container_pid_t *pid_info); +bool has_mount_for(container_t *cont, const char *mpath); + #if defined(__cplusplus) || defined(c_plusplus) } #endif diff --git a/src/services/execution/spec/specs_mount.c b/src/services/execution/spec/specs_mount.c index df99cb4..c95b7bb 100644 --- a/src/services/execution/spec/specs_mount.c +++ b/src/services/execution/spec/specs_mount.c @@ -16,10 +16,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -41,6 +43,7 @@ #include "parse_common.h" #include "specs_mount.h" #include "specs_extend.h" +#include "containers_store.h" static int get_devices(const char *dir, char ***devices, size_t *device_len, int recursive_depth); @@ -1918,14 +1921,18 @@ out_free: return ret; } -static int change_dev_shm_size(oci_runtime_spec *oci_spec, int64_t shm_size) +static int change_dev_shm_size(oci_runtime_spec *oci_spec, host_config *host_spec) { size_t i = 0; size_t j = 0; char size_opt[MOUNT_PROPERTIES_SIZE] = { 0 }; char *tmp = NULL; - int nret = snprintf(size_opt, sizeof(size_opt), "size=%lld", (long long int)shm_size); + if (is_none(host_spec->ipc_mode)) { + return 0; + } + + int nret = snprintf(size_opt, sizeof(size_opt), "size=%lld", (long long int)host_spec->shm_size); if (nret < 0 || (size_t)nret >= sizeof(size_opt)) { ERROR("Out of memory"); return -1; @@ -2026,6 +2033,237 @@ out: return ret; } +static int chown_for_shm(const char *shm_path, const char *user_remap) +{ + unsigned int host_uid = 0; + unsigned int host_gid = 0; + unsigned int size = 0; + + if (shm_path == NULL) { + return 0; + } + + if (user_remap != NULL) { + if (util_parse_user_remap(user_remap, &host_uid, &host_gid, &size)) { + ERROR("Failed to split string '%s'.", user_remap); + return -1; + } + if (chown(shm_path, host_uid, host_gid) != 0) { + ERROR("Failed to chown host path '%s'.", shm_path); + return -1; + } + } + return 0; +} + +static char *get_prepare_share_shm_path(const char *truntime, const char *cid) +{ +#define SHM_MOUNT_FILE_NAME "/mounts/shm/" + char *c_root_path = NULL; + size_t slen = 0; + char *spath = NULL; + int nret = 0; + + if (truntime == NULL) { + truntime = "lcr"; + } + c_root_path = conf_get_routine_rootdir(truntime); + if (c_root_path == NULL) { + goto err_out; + } + + // c_root_path + "/" + cid + "/mounts/shm" + if (strlen(c_root_path) > (((PATH_MAX - strlen(cid)) - 1) - strlen(SHM_MOUNT_FILE_NAME)) - 1) { + ERROR("Too large path"); + goto err_out; + } + slen = strlen(c_root_path) + 1 + strlen(cid) + strlen(SHM_MOUNT_FILE_NAME) + 1; + spath = util_smart_calloc_s(sizeof(char), slen); + if (spath == NULL) { + ERROR("Out of memory"); + goto err_out; + } + + nret = sprintf(spath, "%s/%s/mounts/shm/", c_root_path, cid); + if (nret < 0) { + ERROR("Sprintf failed"); + goto err_out; + } + + return spath; +err_out: + free(spath); + free(c_root_path); + return NULL; +} + +static bool has_mount_shm(host_config *host_spec, container_config_v2_common_config *v2_spec) +{ + container_t *cont = NULL; + bool ret = false; + + cont = util_common_calloc_s(sizeof(container_t)); + if (cont == NULL) { + ERROR("Out of memory"); + goto out; + } + cont->common_config = v2_spec; + cont->hostconfig = host_spec; + + ret = has_mount_for(cont, "/dev/shm"); + + cont->common_config = NULL; + cont->hostconfig = NULL; +out: + free(cont); + return ret; +} + +static int prepare_share_shm(oci_runtime_spec *oci_spec, host_config *host_spec, + container_config_v2_common_config *v2_spec) +{ +#define MAX_PROPERTY_LEN 64 + char shmproperty[MAX_PROPERTY_LEN] = {0}; + int ret = -1; + int nret = 0; + bool has_mount = false; + char *spath = NULL; + + // has mount for /dev/shm + if (has_mount_shm(host_spec, v2_spec)) { + return 0; + } + + spath = get_prepare_share_shm_path(host_spec->runtime, v2_spec->id); + if (spath == NULL) { + goto out; + } + + nret = util_mkdir_p(spath, 0700); + if (nret != 0) { + ERROR("Build shm dir failed"); + goto out; + } + nret = sprintf(shmproperty, "mode=1777,size=%"PRId64, host_spec->shm_size); + if (nret < 0) { + ERROR("Sprintf failed"); + goto out; + } + + nret = mount("shm", spath, "tmpfs", MS_NOEXEC | MS_NODEV | MS_NOSUID, shmproperty); + if (nret < 0) { + ERROR("Mount %s failed: %s", spath, strerror(errno)); + goto out; + } + has_mount = true; + + nret = chown_for_shm(spath, host_spec->user_remap); + if (nret != 0) { + goto out; + } + + v2_spec->shm_path = spath; + spath = NULL; + ret = 0; +out: + if (ret != 0 && has_mount) { + (void)umount(spath); + } + free(spath); + return ret; +} + +static bool add_shm_mount(oci_runtime_spec *container, const char *shm_path) +{ + char **options = NULL; + size_t options_len = 3; + bool ret = false; + defs_mount *tmp_mounts = NULL; + + if (options_len > SIZE_MAX / sizeof(char *)) { + ERROR("Invalid option size"); + return ret; + } + options = util_common_calloc_s(options_len * sizeof(char *)); + if (options == NULL) { + ERROR("Out of memory"); + goto out_free; + } + options[0] = util_strdup_s("rbind"); + options[1] = util_strdup_s("rprivate"); + // default shm size is 64MB + options[2] = util_strdup_s("size=65536k"); + /* generate mount node */ + tmp_mounts = util_common_calloc_s(sizeof(defs_mount)); + if (tmp_mounts == NULL) { + ERROR("Malloc tmp_mounts memory failed"); + goto out_free; + } + + tmp_mounts->destination = util_strdup_s("/dev/shm"); + tmp_mounts->source = util_strdup_s(shm_path); + tmp_mounts->type = util_strdup_s("bind"); + tmp_mounts->options = options; + tmp_mounts->options_len = options_len; + options = NULL; + + /* expand mount array */ + if (!mounts_expand(container, 1)) { + goto out_free; + } + /* add a new mount node */ + container->mounts[container->mounts_len - 1] = tmp_mounts; + + ret = true; +out_free: + + if (!ret) { + util_free_array(options); + free_defs_mount(tmp_mounts); + } + return ret; +} + +#define SHM_MOUNT_POINT "/dev/shm" +static int setup_ipc_dirs(oci_runtime_spec *oci_spec, host_config *host_spec, + container_config_v2_common_config *v2_spec) +{ + int ret = 0; + container_t *cont = NULL; + char *tmp_cid = NULL; + char *right_path = NULL; + + // setup shareable dirs + if (host_spec->ipc_mode == NULL || is_shareable(host_spec->ipc_mode)) { + return prepare_share_shm(oci_spec, host_spec, v2_spec); + } + + if (is_container(host_spec->ipc_mode)) { + tmp_cid = connected_container(host_spec->ipc_mode); + cont = containers_store_get(tmp_cid); + if (cont == NULL) { + ERROR("Invalid share path: %s", host_spec->ipc_mode); + ret = -1; + goto out; + } + right_path = util_strdup_s(cont->common_config->shm_path); + container_unref(cont); + } else if (is_host(host_spec->ipc_mode)) { + if (!util_file_exists(SHM_MOUNT_POINT)) { + ERROR("/dev/shm is not mounted, but must be for --ipc=host"); + ret = -1; + goto out; + } + right_path = util_strdup_s(SHM_MOUNT_POINT); + } + + free(v2_spec->shm_path); + v2_spec->shm_path = right_path; +out: + free(tmp_cid); + return ret; +} + int merge_conf_mounts(oci_runtime_spec *oci_spec, host_config *host_spec, container_config_v2_common_config *v2_spec) { @@ -2062,12 +2300,24 @@ int merge_conf_mounts(oci_runtime_spec *oci_spec, host_config *host_spec, } } - if (host_spec->shm_size >= 0) { - if (host_spec->shm_size == 0) { - host_spec->shm_size = DEFAULT_SHM_SIZE; - } + if (host_spec->shm_size == 0) { + host_spec->shm_size = DEFAULT_SHM_SIZE; + } + + /* setup ipc dir */ + if (setup_ipc_dirs(oci_spec, host_spec, v2_spec) != 0) { + ret = -1; + goto out; + } + + /* add ipc mount */ + if (v2_spec->shm_path != NULL) { + // check whether duplication + add_shm_mount(oci_spec, v2_spec->shm_path); + } - ret = change_dev_shm_size(oci_spec, host_spec->shm_size); + if (host_spec->shm_size > 0) { + ret = change_dev_shm_size(oci_spec, host_spec); if (ret) { ERROR("Failed to set dev shm size"); goto out; -- GitLab