dev_cgroup_rule: add support device cgroup rule

Signed-off-by: Nlifeng68 <lifeng68@huawei.com>

src/client/connect/pack_config.c

@@ -1729,6 +1729,30 @@ out:
     return ret;
 }
 
+int generate_device_cgroup_rules(host_config **dstconfig, const isula_host_config_t *srcconfig)
+{
+    int ret = 0;
+    size_t i = 0;
+
+    if (srcconfig->device_cgroup_rules == NULL || srcconfig->device_cgroup_rules_len == 0) {
+        goto out;
+    }
+
+    (*dstconfig)->device_cgroup_rules = util_smart_calloc_s(sizeof(char *), srcconfig->device_cgroup_rules_len);
+    if ((*dstconfig)->device_cgroup_rules == NULL) {
+        ret = -1;
+        goto out;
+    }
+    for (i = 0; i < srcconfig->device_cgroup_rules_len; i++) {
+        (*dstconfig)->device_cgroup_rules[(*dstconfig)->device_cgroup_rules_len] =
+            util_strdup_s(srcconfig->device_cgroup_rules[i]);
+        (*dstconfig)->device_cgroup_rules_len++;
+    }
+
+out:
+    return ret;
+}
+
 int generate_groups(host_config **dstconfig, const isula_host_config_t *srcconfig)
 {
     int ret = 0;
@@ -1862,6 +1886,13 @@ static int pack_host_config_common(host_config *dstconfig, const isula_host_conf
     if (ret < 0) {
         goto out;
     }
+
+    /* device cgroup rules*/
+    ret = generate_device_cgroup_rules(&dstconfig, srcconfig);
+    if (ret < 0) {
+        goto out;
+    }
+
 out:
     return ret;
 }

src/client/libisula.c

@@ -351,6 +351,10 @@ void isula_host_config_free(isula_host_config_t *hostconfig)
     hostconfig->blkio_throttle_write_iops_device = NULL;
     hostconfig->blkio_throttle_write_iops_device_len = 0;
 
+    util_free_array_by_len(hostconfig->device_cgroup_rules, hostconfig->device_cgroup_rules_len);
+    hostconfig->device_cgroup_rules = NULL;
+    hostconfig->device_cgroup_rules_len = 0;
+
     container_cgroup_resources_free(hostconfig->cr);
     hostconfig->cr = NULL;
 

src/client/libisula.h

@@ -186,6 +186,9 @@ typedef struct isula_host_config {
     char **blkio_throttle_write_iops_device;
     size_t blkio_throttle_write_iops_device_len;
 
+    char **device_cgroup_rules;
+    size_t device_cgroup_rules_len;
+
     bool privileged;
     bool system_container;
     char **ns_change_files;

src/cmd/command_parser.c

@@ -583,3 +583,17 @@ out:
     free(dup);
     return ret;
 }
+
+int command_convert_device_cgroup_rules(command_option_t *option, const char *arg)
+{
+    if (option == NULL) {
+        return -1;
+    }
+
+    if (!util_valid_device_cgroup_rule(arg)) {
+        COMMAND_ERROR("Invalid value \"%s\" for flag --%s", arg, option->large);
+        return EINVALIDARGS;
+    }
+
+    return command_append_array(option, arg);
+}

src/cmd/command_parser.h

@@ -106,6 +106,8 @@ int command_convert_swappiness(command_option_t *option, const char *arg);
 
 int command_convert_nanocpus(command_option_t *option, const char *arg);
 
+int command_convert_device_cgroup_rules(command_option_t *option, const char *arg);
+
 #ifdef __cplusplus
 }
 #endif

src/cmd/isula/base/create.c

@@ -842,7 +842,7 @@ static void request_pack_host_dns(const struct client_arguments *args, isula_hos
     }
 }
 
-static void request_pack_host_ulimit(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_ulimit(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     /* ulimit options */
     if (args->custom_conf.ulimits != NULL) {
@@ -851,7 +851,8 @@ static void request_pack_host_ulimit(const struct client_arguments *args, isula_
     }
 }
 
-static void request_pack_host_weight_devices(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_weight_devices(const struct client_arguments *args,
+                                                    isula_host_config_t *hostconfig)
 {
     /* blkio weight devices */
     if (args->custom_conf.weight_devices != NULL) {
@@ -860,7 +861,8 @@ static void request_pack_host_weight_devices(const struct client_arguments *args
     }
 }
 
-static void request_pack_host_device_read_bps(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_device_read_bps(const struct client_arguments *args,
+                                                     isula_host_config_t *hostconfig)
 {
     if (args->custom_conf.blkio_throttle_read_bps_device != NULL) {
         hostconfig->blkio_throttle_read_bps_device_len =
@@ -869,7 +871,8 @@ static void request_pack_host_device_read_bps(const struct client_arguments *arg
     }
 }
 
-static void request_pack_host_device_write_bps(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_device_write_bps(const struct client_arguments *args,
+                                                      isula_host_config_t *hostconfig)
 {
     if (args->custom_conf.blkio_throttle_write_bps_device != NULL) {
         hostconfig->blkio_throttle_write_bps_device_len =
@@ -878,7 +881,8 @@ static void request_pack_host_device_write_bps(const struct client_arguments *ar
     }
 }
 
-static void request_pack_host_device_read_iops(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_device_read_iops(const struct client_arguments *args,
+                                                      isula_host_config_t *hostconfig)
 {
     if (args->custom_conf.blkio_throttle_read_iops_device != NULL) {
         hostconfig->blkio_throttle_read_iops_device_len =
@@ -887,7 +891,8 @@ static void request_pack_host_device_read_iops(const struct client_arguments *ar
     }
 }
 
-static void request_pack_host_device_write_iops(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_device_write_iops(const struct client_arguments *args,
+                                                       isula_host_config_t *hostconfig)
 {
     if (args->custom_conf.blkio_throttle_write_iops_device != NULL) {
         hostconfig->blkio_throttle_write_iops_device_len =
@@ -896,7 +901,16 @@ static void request_pack_host_device_write_iops(const struct client_arguments *a
     }
 }
 
-static void request_pack_host_blockio(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_device_cgroup_rules(const struct client_arguments *args,
+                                                         isula_host_config_t *hostconfig)
+{
+    if (args->custom_conf.device_cgroup_rules != NULL) {
+        hostconfig->device_cgroup_rules_len = util_array_len((const char **)(args->custom_conf.device_cgroup_rules));
+        hostconfig->device_cgroup_rules = args->custom_conf.device_cgroup_rules;
+    }
+}
+
+inline static void request_pack_host_blockio(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     request_pack_host_weight_devices(args, hostconfig);
     request_pack_host_device_read_bps(args, hostconfig);
@@ -905,7 +919,7 @@ static void request_pack_host_blockio(const struct client_arguments *args, isula
     request_pack_host_device_write_iops(args, hostconfig);
 }
 
-static void request_pack_host_devices(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_devices(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     /* devices */
     if (args->custom_conf.devices != NULL) {
@@ -914,7 +928,8 @@ static void request_pack_host_devices(const struct client_arguments *args, isula
     }
 }
 
-static void request_pack_host_hugepage_limits(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_hugepage_limits(const struct client_arguments *args,
+                                                     isula_host_config_t *hostconfig)
 {
     /* hugepage limits */
     if (args->custom_conf.hugepage_limits != NULL) {
@@ -923,7 +938,7 @@ static void request_pack_host_hugepage_limits(const struct client_arguments *arg
     }
 }
 
-static void request_pack_host_binds(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_binds(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     /* volumes to binds */
     if (args->custom_conf.volumes != NULL) {
@@ -932,7 +947,7 @@ static void request_pack_host_binds(const struct client_arguments *args, isula_h
     }
 }
 
-static void request_pack_host_hook_spec(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_hook_spec(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     /* hook-spec file */
     if (args->custom_conf.hook_spec != NULL) {
@@ -940,7 +955,8 @@ static void request_pack_host_hook_spec(const struct client_arguments *args, isu
     }
 }
 
-static void request_pack_host_restart_policy(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_restart_policy(const struct client_arguments *args,
+                                                    isula_host_config_t *hostconfig)
 {
     if (args->restart != NULL) {
         hostconfig->restart_policy = args->restart;
@@ -970,7 +986,7 @@ static void request_pack_host_namespaces(const struct client_arguments *args, is
     }
 }
 
-static void request_pack_host_security(const struct client_arguments *args, isula_host_config_t *hostconfig)
+inline static void request_pack_host_security(const struct client_arguments *args, isula_host_config_t *hostconfig)
 {
     /* security opt */
     if (args->custom_conf.security != NULL) {
@@ -1062,6 +1078,8 @@ static int request_pack_host_config(const struct client_arguments *args, isula_h
 
     request_pack_host_security(args, hostconfig);
 
+    request_pack_host_device_cgroup_rules(args, hostconfig);
+
     return ret;
 }
 

src/cmd/isula/base/create.h

@@ -459,7 +459,14 @@ extern "C" {
       "Limit CPU real-time runtime in microseconds.",                                                                                                     \
       command_convert_llong },                                                                                                                            \
     { CMD_OPT_TYPE_CALLBACK,   false, "cpus", 0, &((cmdargs).cr).nano_cpus, "Number of CPUs.",                                                            \
-      command_convert_nanocpus },
+      command_convert_nanocpus },                                                                                                                         \
+    { CMD_OPT_TYPE_CALLBACK,                                                                                                                              \
+      false,                                                                                                                                              \
+      "device-cgroup-rule",                                                                                                                               \
+      0,                                                                                                                                                  \
+      &(cmdargs).custom_conf.device_cgroup_rules,                                                                                                         \
+      "Add a rule to the cgroup allowed devices list.",                                                                                                   \
+      command_convert_device_cgroup_rules },
 
 #define CREATE_EXTEND_OPTIONS(cmdargs)        \
     { CMD_OPT_TYPE_BOOL,                      \

src/cmd/isula/client_arguments.c

@@ -250,6 +250,9 @@ void client_arguments_free(struct client_arguments *args)
 
     util_free_array(custom_conf->blkio_throttle_write_iops_device);
     custom_conf->blkio_throttle_write_iops_device = NULL;
+
+    util_free_array(custom_conf->device_cgroup_rules);
+    custom_conf->device_cgroup_rules = NULL;
 }
 
 /* print common help */

src/cmd/isula/client_arguments.h

@@ -203,6 +203,9 @@ struct custom_configs {
 
     /* device write iops */
     char **blkio_throttle_write_iops_device;
+
+    /* device cgroup rules */
+    char **device_cgroup_rules;
 };
 
 struct args_cgroup_resources {

src/daemon/modules/spec/specs_mount.c

@@ -1788,6 +1788,194 @@ out:
     return ret;
 }
 
+static int merge_conf_populate_device(oci_runtime_spec *oci_spec, host_config *host_spec)
+{
+    int ret = 0;
+
+    if (host_spec->privileged) {
+        INFO("Skipped \"--device\" due to conflict with \"--privileged\"");
+        return 0;
+    }
+
+    if (host_spec->devices != NULL && host_spec->devices_len != 0) {
+        /* privileged containers will populate all devices in host */
+        ret = merge_custom_devices(oci_spec, host_spec->devices, host_spec->devices_len);
+        if (ret != 0) {
+            ERROR("Failed to merge custom devices");
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+/* format example: b 7:* rmw */
+static int parse_device_cgroup_rule(defs_device_cgroup *spec_dev_cgroup, const char *rule)
+{
+    int ret = 0;
+    char **parts = NULL;
+    size_t parts_len = 0;
+    char **file_mode = NULL;
+    size_t file_mode_len = 0;
+
+    if (rule == NULL || spec_dev_cgroup == NULL) {
+        return -1;
+    }
+
+    parts = util_string_split(rule, ' ');
+    if (parts == NULL) {
+        ERROR("Failed to split rule %s", rule);
+        ret = -1;
+        goto free_out;
+    }
+
+    parts_len = util_array_len((const char **)parts);
+    if (parts_len != 3) {
+        ERROR("Invalid rule %s", rule);
+        ret = -1;
+        goto free_out;
+    }
+
+    /* fill spec cgroup dev */
+    spec_dev_cgroup->allow = true;
+    spec_dev_cgroup->access = util_strdup_s(parts[2]);
+    spec_dev_cgroup->type = util_strdup_s(parts[0]);
+
+    file_mode = util_string_split(parts[1], ':');
+    if (file_mode == NULL) {
+        ERROR("Invalid rule mode %s", parts[1]);
+        ret = -1;
+        goto free_out;
+    }
+
+    file_mode_len = util_array_len((const char **)file_mode);
+    if (file_mode_len != 2) {
+        ERROR("Invalid rule mode %s", parts[1]);
+        ret = -1;
+        goto free_out;
+    }
+
+    if (strcmp(file_mode[0], "*") == 0) {
+        spec_dev_cgroup->major = -1;
+    } else {
+        if (util_safe_llong(file_mode[0], (long long *)&spec_dev_cgroup->major) != 0) {
+            ERROR("Invalid rule mode %s", file_mode[0]);
+            ret = -1;
+            goto free_out;
+        }
+    }
+
+    if (strcmp(file_mode[1], "*") == 0) {
+        spec_dev_cgroup->minor = -1;
+    } else {
+        if (util_safe_llong(file_mode[1], (long long *)&spec_dev_cgroup->minor) != 0) {
+            ERROR("Invalid rule mode %s", file_mode[1]);
+            ret = -1;
+            goto free_out;
+        }
+    }
+
+free_out:
+    util_free_array(parts);
+    util_free_array(file_mode);
+
+    return ret;
+}
+
+static int make_device_cgroup_rule(defs_device_cgroup **out_spec_dev_cgroup, const char *dev_cgroup_rule)
+{
+    int ret = 0;
+    defs_device_cgroup *spec_dev_cgroup = NULL;
+
+    spec_dev_cgroup = util_common_calloc_s(sizeof(defs_device_cgroup));
+    if (spec_dev_cgroup == NULL) {
+        ERROR("Memory out");
+        ret = -1;
+        goto erro_out;
+    }
+
+    ret = parse_device_cgroup_rule(spec_dev_cgroup, dev_cgroup_rule);
+    if (ret != 0) {
+        ERROR("Failed to parse device cgroup rule %s", dev_cgroup_rule);
+        ret = -1;
+        goto erro_out;
+    }
+
+    *out_spec_dev_cgroup = spec_dev_cgroup;
+    goto out;
+
+erro_out:
+    free_defs_device_cgroup(spec_dev_cgroup);
+out:
+    return ret;
+}
+
+static int do_merge_device_cgroup_rules(oci_runtime_spec *oci_spec, const char **dev_cgroup_rules,
+                                        size_t dev_cgroup_rules_len)
+{
+    int ret = 0;
+    size_t new_size = 0, old_size = 0;
+    size_t i = 0;
+
+    ret = make_sure_oci_spec_linux_resources(oci_spec);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* malloc for cgroup->device */
+    defs_device_cgroup **spec_cgroup_dev = NULL;
+    if (dev_cgroup_rules_len > SIZE_MAX / sizeof(defs_device_cgroup *) - oci_spec->linux->resources->devices_len) {
+        ERROR("Too many cgroup devices to merge!");
+        ret = -1;
+        goto out;
+    }
+    new_size = (oci_spec->linux->resources->devices_len + dev_cgroup_rules_len) * sizeof(defs_device_cgroup *);
+    old_size = oci_spec->linux->resources->devices_len * sizeof(defs_device_cgroup *);
+    ret = mem_realloc((void **)&spec_cgroup_dev, new_size, oci_spec->linux->resources->devices, old_size);
+    if (ret != 0) {
+        ERROR("Out of memory");
+        ret = -1;
+        goto out;
+    }
+    oci_spec->linux->resources->devices = spec_cgroup_dev;
+
+    for (i = 0; i < dev_cgroup_rules_len; i++) {
+        ret = make_device_cgroup_rule(&oci_spec->linux->resources->devices[oci_spec->linux->resources->devices_len],
+                                      dev_cgroup_rules[i]);
+        if (ret != 0) {
+            ERROR("Failed to merge custom device");
+            ret = -1;
+            goto out;
+        }
+        oci_spec->linux->resources->devices_len++;
+    }
+out:
+    return ret;
+}
+
+static int merge_conf_device_cgroup_rule(oci_runtime_spec *oci_spec, host_config *host_spec)
+{
+    int ret = 0;
+
+    if (host_spec->privileged) {
+        INFO("Skipped \"--device-cgroup-rule\" due to conflict with \"--privileged\"");
+        return 0;
+    }
+
+    if (host_spec->device_cgroup_rules == NULL || host_spec->device_cgroup_rules_len == 0) {
+        return 0;
+    }
+
+    ret = do_merge_device_cgroup_rules(oci_spec, (const char **)host_spec->device_cgroup_rules,
+                                       host_spec->device_cgroup_rules_len);
+    if (ret != 0) {
+        ERROR("Failed to merge custom devices cgroup rules");
+        goto out;
+    }
+out:
+    return ret;
+}
+
 int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec)
 {
     int ret = 0;
@@ -1842,17 +2030,15 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec)
     }
 
     /* devices which will be populated into container */
-    if (host_spec->devices != NULL && host_spec->devices_len != 0) {
-        /* privileged containers will populate all devices in host */
-        if (!host_spec->privileged) {
-            ret = merge_custom_devices(oci_spec, host_spec->devices, host_spec->devices_len);
-            if (ret != 0) {
-                ERROR("Failed to merge custom devices");
-                goto out;
-            }
-        } else {
-            INFO("Skipped \"--device\" due to conflict with \"--privileged\"");
-        }
+    if (merge_conf_populate_device(oci_spec, host_spec)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* device cgroup rules which will be added into container */
+    if (merge_conf_device_cgroup_rule(oci_spec, host_spec)) {
+        ret = -1;
+        goto out;
     }
 
 out:

src/daemon/modules/spec/verify.c

@@ -1951,6 +1951,25 @@ out:
     return ret;
 }
 
+/* verify device cgroup rule */
+static int verify_host_config_device_cgroup_rules(const host_config *hostconfig)
+{
+    int ret = 0;
+    size_t i = 0;
+
+    for (i = 0; i < hostconfig->device_cgroup_rules_len; i++) {
+        if (!util_valid_device_cgroup_rule(hostconfig->device_cgroup_rules[i])) {
+            ERROR("Invalid device cgroup rule %s", hostconfig->device_cgroup_rules[i]);
+            isulad_set_error_message("Invalid device cgroup rule %s", hostconfig->device_cgroup_rules[i]);
+            ret = -1;
+            goto out;
+        }
+    }
+
+out:
+    return ret;
+}
+
 /* verify host config settings */
 int verify_host_config_settings(host_config *hostconfig, bool update)
 {
@@ -1977,6 +1996,11 @@ int verify_host_config_settings(host_config *hostconfig, bool update)
         goto out;
     }
 
+    ret = verify_host_config_device_cgroup_rules(hostconfig);
+    if (ret != 0) {
+        goto out;
+    }
+
 #ifdef ENABLE_OCI_IMAGE
     // storage options
     ret = verify_storage_opts(hostconfig);

src/utils/cutils/utils_verify.c

@@ -649,5 +649,16 @@ bool util_valid_positive_interger(const char *value)
         return false;
     }
 
+    return util_reg_match(patten, value) == 0;
+}
+
+bool util_valid_device_cgroup_rule(const char *value)
+{
+    const char *patten = "^([acb]) ([0-9]+|\\*):([0-9]+|\\*) ([rwm]{1,3})$";
+
+    if (value == NULL) {
+        return false;
+    }
+
     return util_reg_match(patten, value) == 0;
 }
\ No newline at end of file

src/utils/cutils/utils_verify.h

@@ -101,6 +101,8 @@ bool util_valid_exec_suffix(const char *suffix);
 
 bool util_valid_positive_interger(const char *value);
 
+bool util_valid_device_cgroup_rule(const char *value);
+
 #ifdef __cplusplus
 }
 #endif