dev_mapper: refact device mapper init

Signed-off-by: Nlifeng68 <lifeng68@huawei.com>

src/image/oci/storage/layer_store/graphdriver/devmapper/device_setup.h

@@ -23,14 +23,6 @@
 extern "C" {
 #endif
 
-// struct image_devmapper_direct_lvm_config {
-//     char *device;
-//     uint64_t thinp_percent;
-//     uint64_t thinp_meta_percent;
-//     uint64_t auto_extend_percent;
-//     uint64_t auto_extend_threshold;
-// };
-
 int validate_lvm_config(image_devmapper_direct_lvm_config *cfg);
 int check_dev_available(const char *dev);
 int check_dev_invg(const char *dev);

src/image/oci/storage/layer_store/graphdriver/devmapper/devices_constants.h

@@ -16,6 +16,11 @@
 #ifndef __DEVICES_CONSTANTS_
 #define __DEVICES_CONSTANTS_
 
+#include "map.h"
+#include "isula_libutils/image_devmapper_transaction.h"
+#include "isula_libutils/image_devmapper_deviceset_metadata.h"
+#include "isula_libutils/image_devmapper_direct_lvm_config.h"
+
 #define DEVICE_FILE_DIR "/var/lib/isulad/storage/devicemapper/metadata"
 #define DEVICE_SET_METAFILE "deviceset-metadata"
 #define TRANSACTION_METADATA "transaction-metadata"
@@ -32,4 +37,57 @@
 
 #define DEFAULT_DEVICE_SET_MODE 0700
 
+struct device_set {
+    char *root;
+    char *device_prefix;
+    uint64_t transaction_id;
+    int next_device_id; // deviceset-metadata
+    map_t *device_id_map;
+
+    // options
+    int64_t data_loop_back_size;
+    int64_t meta_data_loop_back_size;
+    uint64_t base_fs_size;
+    char *filesystem;
+    char *mount_options;
+    char **mkfs_args; // []string类型数组切片
+    size_t mkfs_args_len;
+    char *data_device;
+    char *data_loop_file;
+    char *metadata_device;
+    char *metadata_loop_file;
+    uint64_t thinp_block_size;
+    bool do_blk_discard;
+    char *thin_pool_device;
+
+    image_devmapper_transaction *metadata_trans;
+
+    bool override_udev_sync_check;
+    bool deferred_remove;
+    bool deferred_delete;
+    char *base_device_uuid;
+    char *base_device_filesystem;
+    uint nr_deleted_devices; // number of deleted devices
+    uint32_t min_free_space_percent;
+    char *xfs_nospace_retries; // max retries when xfs receives ENOSPC
+    int64_t udev_wait_timeout;
+
+    image_devmapper_direct_lvm_config *lvm_setup_config;
+    bool driver_deferred_removal_support;
+    bool enable_deferred_removal;
+    bool enable_deferred_deletion;
+    bool user_base_size;
+};
+
+typedef struct {
+    map_t *map; // map string image_devmapper_device_info*   key string will be strdup  value ptr will not
+    pthread_rwlock_t rwlock;
+} metadata_store_t;
+
+struct devmapper_conf {
+    pthread_rwlock_t devmapper_driver_rwlock;
+    struct device_set *devset;
+    metadata_store_t *meta_store;
+};
+
 #endif
\ No newline at end of file

src/image/oci/storage/layer_store/graphdriver/devmapper/deviceset.h

@@ -19,63 +19,17 @@
 #include "driver.h"
 #include "metadata_store.h"
 #include "device_setup.h"
-#include "map.h"
-#include "isula_libutils/image_devmapper_transaction.h"
-#include "isula_libutils/image_devmapper_deviceset_metadata.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-struct device_set {
-    char *root;
-    char *device_prefix;
-    uint64_t transaction_id;
-    int next_device_id; // deviceset-metadata
-    map_t *device_id_map;
-
-    // options
-    int64_t data_loop_back_size;
-    int64_t meta_data_loop_back_size;
-    uint64_t base_fs_size;
-    char *filesystem;
-    char *mount_options;
-    char **mkfs_args; // []string类型数组切片
-    size_t mkfs_args_len;
-    char *data_device;
-    char *data_loop_file;
-    char *metadata_device;
-    char *metadata_loop_file;
-    uint64_t thinp_block_size;
-    bool do_blk_discard;
-    char *thin_pool_device;
-
-    image_devmapper_transaction *metadata_trans;
-
-    bool override_udev_sync_check;
-    bool deferred_remove;
-    bool deferred_delete;
-    char *base_device_uuid;
-    char *base_device_filesystem;
-    uint nr_deleted_devices; // number of deleted devices
-    uint32_t min_free_space_percent;
-    char *xfs_nospace_retries; // max retries when xfs receives ENOSPC
-    int64_t udev_wait_timeout;
-
-    image_devmapper_direct_lvm_config *lvm_setup_config;
-};
-
 struct device_metadata {
     int device_id;
     uint64_t device_size;
     char *device_name;
 };
 
-struct devmapper_conf {
-    pthread_rwlock_t devmapper_driver_rwlock;
-    struct device_set *devset;
-};
-
 struct disk_usage {
     // Used bytes on the disk.
     uint64_t used;
@@ -110,14 +64,16 @@ int devmapper_conf_wrlock();
 int devmapper_conf_unlock();
 struct device_set *devmapper_driver_devices_get();
 
-int add_device(const char *hash, const char *base_hash, const json_map_string_string *storage_opts);
-int mount_device(const char *hash, const char *path, const struct driver_mount_opts *mount_opts);
-int unmount_device(const char *hash, const char *mount_path);
-bool has_device(const char *hash);
+int add_device(const char *hash, const char *base_hash, const struct graphdriver *driver,
+               const json_map_string_string *storage_opts);
+int mount_device(const char *hash, const char *path, const struct driver_mount_opts *mount_opts,
+                 const struct graphdriver *driver);
+int unmount_device(const char *hash, const char *mount_path, const struct graphdriver *driver);
+bool has_device(const char *hash, const struct graphdriver *driver);
 
-int delete_device(const char *hash, bool sync_delete);
+int delete_device(const char *hash, bool sync_delete, const struct graphdriver *driver);
 
-int export_device_metadata(struct device_metadata *dev_metadata, const char *hash);
+int export_device_metadata(struct device_metadata *dev_metadata, const char *hash, const struct graphdriver *driver);
 struct status *device_set_status();
 void free_devmapper_status(struct status *st);
 

src/image/oci/storage/layer_store/graphdriver/devmapper/driver_devmapper.c

@@ -39,9 +39,10 @@ int devmapper_init(struct graphdriver *driver, const char *drvier_home, const ch
     return device_init(driver, drvier_home, options, len);
 }
 
-static int do_create(const char *id, const char *parent, const struct driver_create_opts *create_opts)
+static int do_create(const char *id, const char *parent, const struct graphdriver *driver,
+                     const struct driver_create_opts *create_opts)
 {
-    return add_device(id, parent, create_opts->storage_opt);
+    return add_device(id, parent, driver, create_opts->storage_opt);
 }
 
 // devmapper_create_rw creates a layer that is writable for use as a container file system
@@ -52,7 +53,7 @@ int devmapper_create_rw(const char *id, const char *parent, const struct graphdr
         return -1;
     }
 
-    return do_create(id, parent, create_opts);
+    return do_create(id, parent, driver, create_opts);
 }
 
 // Create adds a device with a given id and the parent.
@@ -63,7 +64,7 @@ int devmapper_create_ro(const char *id, const char *parent, const struct graphdr
         return -1;
     }
 
-    return do_create(id, parent, create_opts);
+    return do_create(id, parent, driver, create_opts);
 }
 
 // Remove removes a device with a given id, unmounts the filesystem.
@@ -77,11 +78,11 @@ int devmapper_rm_layer(const char *id, const struct graphdriver *driver)
         return -1;
     }
 
-    if (!has_device(id)) {
+    if (!has_device(id, driver)) {
         return 0;
     }
 
-    ret = delete_device(id, false);
+    ret = delete_device(id, false, driver);
     if (ret != 0) {
         ERROR("failed to remove device %s", id);
         return ret;
@@ -161,7 +162,7 @@ char *devmapper_mount_layer(const char *id, const struct graphdriver *driver,
     }
 
     DEBUG("devmapper: start to mount container device");
-    ret = mount_device(id, mnt_point_dir, mount_opts);
+    ret = mount_device(id, mnt_point_dir, mount_opts, driver);
     if (ret != 0) {
         goto out;
     }
@@ -175,7 +176,7 @@ char *devmapper_mount_layer(const char *id, const struct graphdriver *driver,
     if (util_mkdir_p(rootfs, 0755) != 0 || !util_dir_exists(rootfs)) {
         ERROR("Unable to create devmapper rootfs directory %s.", rootfs);
         ret = -1;
-        if (unmount_device(id, mnt_point_dir) != 0) {
+        if (unmount_device(id, mnt_point_dir, driver) != 0) {
             DEBUG("devmapper: unmount %s failed", mnt_point_dir);
         }
         goto out;
@@ -187,7 +188,7 @@ char *devmapper_mount_layer(const char *id, const struct graphdriver *driver,
         // of later problems
         ret = write_file(id_file, id);
         if (ret != 0) {
-            if (unmount_device(id, mnt_point_dir) != 0) {
+            if (unmount_device(id, mnt_point_dir, driver) != 0) {
                 DEBUG("devmapper: unmount %s failed", mnt_point_dir);
             }
         }
@@ -228,7 +229,7 @@ int devmapper_umount_layer(const char *id, const struct graphdriver *driver)
         goto out;
     }
 
-    ret = unmount_device(id, mp);
+    ret = unmount_device(id, mp, driver);
     if (ret != 0) {
         DEBUG("devmapper: unmount %s failed", mp);
     }
@@ -255,7 +256,7 @@ static void free_driver_mount_opts(struct driver_mount_opts *opts)
 
 bool devmapper_layer_exists(const char *id, const struct graphdriver *driver)
 {
-    return has_device(id);
+    return has_device(id, driver);
 }
 
 int devmapper_apply_diff(const char *id, const struct graphdriver *driver, const struct io_read_wrapper *content,
@@ -318,7 +319,7 @@ int devmapper_get_layer_metadata(const char *id, const struct graphdriver *drive
         goto out;
     }
 
-    ret = export_device_metadata(&dev_metadata, id);
+    ret = export_device_metadata(&dev_metadata, id, driver);
     if (ret != 0) {
         ERROR("Failed to export device metadata of device %s", id);
         goto out;
@@ -475,7 +476,7 @@ int devmapper_get_driver_status(const struct graphdriver *driver, struct graphdr
         return -1;
     }
 
-    st = device_set_status();
+    st = device_set_status(driver);
     if (st == NULL) {
         ERROR("Failed to get device set status");
         ret = -1;

src/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.c

@@ -18,13 +18,6 @@
 #include "utils.h"
 #include "isula_libutils/log.h"
 
-typedef struct {
-    map_t *map; // map string image_devmapper_device_info*   key string will be strdup  value ptr will not
-    pthread_rwlock_t rwlock;
-} metadata_store_t;
-
-static metadata_store_t *g_metadata_store = NULL;
-
 /* metadata store map kvfree */
 static void metadata_store_map_kvfree(void *key, void *value)
 {
@@ -46,7 +39,7 @@ static void metadata_store_free(metadata_store_t *store)
 }
 
 /* metadata store new */
-static metadata_store_t *metadata_store_new(void)
+metadata_store_t *metadata_store_new(void)
 {
     int ret;
     metadata_store_t *store = NULL;
@@ -73,62 +66,53 @@ error_out:
     return NULL;
 }
 
-int metadata_store_init(void)
-{
-    g_metadata_store = metadata_store_new();
-    if (g_metadata_store == NULL) {
-        return -1;
-    }
-    return 0;
-}
-
-bool metadata_store_add(const char *hash, image_devmapper_device_info *device)
+bool metadata_store_add(const char *hash, image_devmapper_device_info *device, metadata_store_t *meta_store)
 {
     bool ret = false;
 
-    if (pthread_rwlock_wrlock(&g_metadata_store->rwlock)) {
+    if (pthread_rwlock_wrlock(&meta_store->rwlock)) {
         ERROR("devmapper: lock metadata store failed");
         return false;
     }
 
-    ret = map_replace(g_metadata_store->map, (void *)hash, (void *)device);
-    if (pthread_rwlock_unlock(&g_metadata_store->rwlock)) {
+    ret = map_replace(meta_store->map, (void *)hash, (void *)device);
+    if (pthread_rwlock_unlock(&meta_store->rwlock)) {
         ERROR("devmapper: unlock metadata store failed");
         return false;
     }
     return ret;
 }
 
-image_devmapper_device_info *metadata_store_get(const char *hash)
+image_devmapper_device_info *metadata_store_get(const char *hash, metadata_store_t *meta_store)
 {
     image_devmapper_device_info *device = NULL;
 
     if (hash == NULL) {
         return NULL;
     }
-    if (pthread_rwlock_rdlock(&g_metadata_store->rwlock) != 0) {
+    if (pthread_rwlock_rdlock(&meta_store->rwlock) != 0) {
         ERROR("devmapper:lock memory store failed");
         return device;
     }
-    device = map_search(g_metadata_store->map, (void *)hash);
-    if (pthread_rwlock_unlock(&g_metadata_store->rwlock) != 0) {
+    device = map_search(meta_store->map, (void *)hash);
+    if (pthread_rwlock_unlock(&meta_store->rwlock) != 0) {
         ERROR("devmapper:unlock memory store failed");
     }
 
     return device;
 }
 
-bool metadata_store_remove(const char *hash)
+bool metadata_store_remove(const char *hash, metadata_store_t *meta_store)
 {
     bool ret = false;
 
-    if (pthread_rwlock_wrlock(&g_metadata_store->rwlock) != 0) {
+    if (pthread_rwlock_wrlock(&meta_store->rwlock) != 0) {
         ERROR("devmapper:lock memory store failed");
         return false;
     }
 
-    ret = map_remove(g_metadata_store->map, (void *)hash);
-    if (pthread_rwlock_unlock(&g_metadata_store->rwlock) != 0) {
+    ret = map_remove(meta_store->map, (void *)hash);
+    if (pthread_rwlock_unlock(&meta_store->rwlock) != 0) {
         ERROR("devmapper:unlock memory store failed");
         return false;
     }
@@ -136,23 +120,23 @@ bool metadata_store_remove(const char *hash)
 }
 
 /* metadata store list hashes */
-char **metadata_store_list_hashes(void)
+char **metadata_store_list_hashes(metadata_store_t *meta_store)
 {
     bool ret = false;
     char **hashes_array = NULL;
     map_itor *itor = NULL;
 
-    if (pthread_rwlock_rdlock(&g_metadata_store->rwlock) != 0) {
+    if (pthread_rwlock_rdlock(&meta_store->rwlock) != 0) {
         ERROR("devmapper:lock memory store failed");
         return NULL;
     }
 
-    if (map_size(g_metadata_store->map) == 0) {
+    if (map_size(meta_store->map) == 0) {
         ret = true;
         goto unlock;
     }
 
-    itor = map_itor_new(g_metadata_store->map);
+    itor = map_itor_new(meta_store->map);
     if (itor == NULL) {
         ERROR("Out of memory");
         goto unlock;
@@ -167,7 +151,7 @@ char **metadata_store_list_hashes(void)
     }
     ret = true;
 unlock:
-    if (pthread_rwlock_unlock(&g_metadata_store->rwlock)) {
+    if (pthread_rwlock_unlock(&meta_store->rwlock)) {
         ERROR("unlock metadata store failed");
     }
     map_itor_free(itor);

src/image/oci/storage/layer_store/graphdriver/devmapper/metadata_store.h

@@ -15,22 +15,22 @@
 #ifndef __METADATA_STORE_H
 #define __METADATA_STORE_H
 
-#include "map.h"
+#include "devices_constants.h"
 #include "isula_libutils/image_devmapper_device_info.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int metadata_store_init(void);
+metadata_store_t *metadata_store_new(void);
 
-bool metadata_store_add(const char *hash, image_devmapper_device_info *device);
+bool metadata_store_add(const char *hash, image_devmapper_device_info *device, metadata_store_t *meta_store);
 
-image_devmapper_device_info *metadata_store_get(const char *hash);
+image_devmapper_device_info *metadata_store_get(const char *hash, metadata_store_t *meta_store);
 
-bool metadata_store_remove(const char *hash);
+bool metadata_store_remove(const char *hash, metadata_store_t *meta_store);
 
-char **metadata_store_list_hashes(void);
+char **metadata_store_list_hashes(metadata_store_t *meta_store);
 
 #ifdef __cplusplus
 }

src/image/oci/storage/layer_store/graphdriver/driver.h

@@ -22,6 +22,7 @@
 #include "isula_libutils/json_common.h"
 #include "console.h"
 #include "driver_overlay2_types.h"
+#include "devices_constants.h"
 #include "storage.h"
 #include "image.h"
 #include "isula_libutils/container_inspect.h"
@@ -87,6 +88,9 @@ struct graphdriver {
 
     // options for overlay2
     struct overlay_options *overlay_opts;
+
+    // options for device mapper
+    struct devmapper_conf *devmapper_opts;
 };
 
 int graphdriver_init(const struct storage_module_init_options *opts);