Merge tag 'for-5.15/dm-changes' of...

Merge tag 'for-5.15/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mike Snitzer:

 - Add DM infrastructure for IMA-based remote attestion. These changes
   are the basis for deploying DM-based storage in a "cloud" that must
   validate configurations end-users run to maintain trust. These DM
   changes allow supported DM targets' configurations to be measured via
   IMA. But the policy and enforcement (of which configurations are
   valid) is managed by something outside the kernel (e.g. Keylime).

 - Fix DM crypt scalability regression on systems with many cpus due to
   percpu_counter spinlock contention in crypt_page_alloc().

 - Use in_hardirq() instead of deprecated in_irq() in DM crypt.

 - Add event counters to DM writecache to allow users to further assess
   how the writecache is performing.

 - Various code cleanup in DM writecache's main IO mapping function.

* tag 'for-5.15/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm:
  dm crypt: use in_hardirq() instead of deprecated in_irq()
  dm ima: update dm documentation for ima measurement support
  dm ima: update dm target attributes for ima measurements
  dm ima: add a warning in dm_init if duplicate ima events are not measured
  dm ima: prefix ima event name related to device mapper with dm_
  dm ima: add version info to dm related events in ima log
  dm ima: prefix dm table hashes in ima log with hash algorithm
  dm crypt: Avoid percpu_counter spinlock contention in crypt_page_alloc()
  dm: add documentation for IMA measurement support
  dm: update target status functions to support IMA measurement
  dm ima: measure data on device rename
  dm ima: measure data on table clear
  dm ima: measure data on device remove
  dm ima: measure data on device resume
  dm ima: measure data on table load
  dm writecache: add event counters
  dm writecache: report invalid return from writecache_map helpers
  dm writecache: further writecache_map() cleanup
  dm writecache: factor out writecache_map_remap_origin()
  dm writecache: split up writecache_map() to improve code readability

Documentation/admin-guide/device-mapper/index.rst

@@ -13,6 +13,7 @@ Device Mapper
     dm-dust
     dm-ebs
     dm-flakey
+    dm-ima
     dm-init
     dm-integrity
     dm-io

Documentation/admin-guide/device-mapper/writecache.rst

@@ -78,13 +78,23 @@ Status:
 2. the number of blocks
 3. the number of free blocks
 4. the number of blocks under writeback
+5. the number of read requests
+6. the number of read requests that hit the cache
+7. the number of write requests
+8. the number of write requests that hit uncommitted block
+9. the number of write requests that hit committed block
+10. the number of write requests that bypass the cache
+11. the number of write requests that are allocated in the cache
+12. the number of write requests that are blocked on the freelist
+13. the number of flush requests
+14. the number of discard requests
 
 Messages:
 	flush
-		flush the cache device. The message returns successfully
+		Flush the cache device. The message returns successfully
 		if the cache device was flushed without an error
 	flush_on_suspend
-		flush the cache device on next suspend. Use this message
+		Flush the cache device on next suspend. Use this message
 		when you are going to remove the cache device. The proper
 		sequence for removing the cache device is:
 
@@ -98,3 +108,5 @@ Messages:
 		6. the cache device is now inactive and it can be deleted
 	cleaner
 		See above "cleaner" constructor documentation.
+	clear_stats
+		Clear the statistics that are reported on the status line

drivers/md/Makefile

@@ -96,6 +96,10 @@ ifeq ($(CONFIG_BLK_DEV_ZONED),y)
 dm-mod-objs			+= dm-zone.o
 endif
 
+ifeq ($(CONFIG_IMA),y)
+dm-mod-objs			+= dm-ima.o
+endif
+
 ifeq ($(CONFIG_DM_VERITY_FEC),y)
 dm-verity-objs			+= dm-verity-fec.o
 endif

drivers/md/dm-cache-target.c

@@ -3122,6 +3122,30 @@ static void cache_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" %s", cache->ctr_args[i]);
 		if (cache->nr_ctr_args)
 			DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
+		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		if (get_cache_mode(cache) == CM_FAIL)
+			DMEMIT(",metadata_mode=fail");
+		else if (get_cache_mode(cache) == CM_READ_ONLY)
+			DMEMIT(",metadata_mode=ro");
+		else
+			DMEMIT(",metadata_mode=rw");
+
+		format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
+		DMEMIT(",cache_metadata_device=%s", buf);
+		format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
+		DMEMIT(",cache_device=%s", buf);
+		format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
+		DMEMIT(",cache_origin_device=%s", buf);
+		DMEMIT(",writethrough=%c", writethrough_mode(cache) ? 'y' : 'n');
+		DMEMIT(",writeback=%c", writeback_mode(cache) ? 'y' : 'n');
+		DMEMIT(",passthrough=%c", passthrough_mode(cache) ? 'y' : 'n');
+		DMEMIT(",metadata2=%c", cache->features.metadata_version == 2 ? 'y' : 'n');
+		DMEMIT(",no_discard_passdown=%c", cache->features.discard_passdown ? 'n' : 'y');
+		DMEMIT(";");
+		break;
 	}
 
 	return;

drivers/md/dm-clone-target.c

@@ -1499,6 +1499,11 @@ static void clone_status(struct dm_target *ti, status_type_t type,
 
 		for (i = 0; i < clone->nr_ctr_args; i++)
 			DMEMIT(" %s", clone->ctr_args[i]);
+		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return;

drivers/md/dm-core.h

@@ -18,6 +18,7 @@
 #include <trace/events/block.h>
 
 #include "dm.h"
+#include "dm-ima.h"
 
 #define DM_RESERVED_MAX_IOS		1024
 
@@ -119,6 +120,10 @@ struct mapped_device {
 	unsigned int nr_zones;
 	unsigned int *zwp_offset;
 #endif
+
+#ifdef CONFIG_IMA
+	struct dm_ima_measurements ima;
+#endif
 };
 
 /*

drivers/md/dm-crypt.c

@@ -2223,11 +2223,11 @@ static void kcryptd_queue_crypt(struct dm_crypt_io *io)
 	if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
 	    (bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
 		/*
-		 * in_irq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
+		 * in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
 		 * irqs_disabled(): the kernel may run some IO completion from the idle thread, but
 		 * it is being executed with irqs disabled.
 		 */
-		if (in_irq() || irqs_disabled()) {
+		if (in_hardirq() || irqs_disabled()) {
 			tasklet_init(&io->tasklet, kcryptd_crypt_tasklet, (unsigned long)&io->work);
 			tasklet_schedule(&io->tasklet);
 			return;
@@ -2661,7 +2661,12 @@ static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
 	struct crypt_config *cc = pool_data;
 	struct page *page;
 
-	if (unlikely(percpu_counter_compare(&cc->n_allocated_pages, dm_crypt_pages_per_client) >= 0) &&
+	/*
+	 * Note, percpu_counter_read_positive() may over (and under) estimate
+	 * the current usage by at most (batch - 1) * num_online_cpus() pages,
+	 * but avoids potential spinlock contention of an exact result.
+	 */
+	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
 	    likely(gfp_mask & __GFP_NORETRY))
 		return NULL;
 
@@ -3485,7 +3490,34 @@ static void crypt_status(struct dm_target *ti, status_type_t type,
 			if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
 				DMEMIT(" iv_large_sectors");
 		}
+		break;
 
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');
+		DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');
+		DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?
+		       'y' : 'n');
+		DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?
+		       'y' : 'n');
+		DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?
+		       'y' : 'n');
+		DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
+		       'y' : 'n');
+
+		if (cc->on_disk_tag_size)
+			DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
+			       cc->on_disk_tag_size, cc->cipher_auth);
+		if (cc->sector_size != (1 << SECTOR_SHIFT))
+			DMEMIT(",sector_size=%d", cc->sector_size);
+		if (cc->cipher_string)
+			DMEMIT(",cipher_string=%s", cc->cipher_string);
+
+		DMEMIT(",key_size=%u", cc->key_size);
+		DMEMIT(",key_parts=%u", cc->key_parts);
+		DMEMIT(",key_extra_size=%u", cc->key_extra_size);
+		DMEMIT(",key_mac_size=%u", cc->key_mac_size);
+		DMEMIT(";");
 		break;
 	}
 }

drivers/md/dm-delay.c

@@ -326,6 +326,10 @@ static void delay_status(struct dm_target *ti, status_type_t type,
 			DMEMIT_DELAY_CLASS(&dc->flush);
 		}
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 

drivers/md/dm-dust.c

@@ -527,6 +527,10 @@ static void dust_status(struct dm_target *ti, status_type_t type,
 		DMEMIT("%s %llu %u", dd->dev->name,
 		       (unsigned long long)dd->start, dd->blksz);
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 

drivers/md/dm-ebs-target.c

@@ -401,6 +401,9 @@ static void ebs_status(struct dm_target *ti, status_type_t type,
 		snprintf(result, maxlen, ec->u_bs_set ? "%s %llu %u %u" : "%s %llu %u",
 			 ec->dev->name, (unsigned long long) ec->start, ec->e_bs, ec->u_bs);
 		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 

drivers/md/dm-era-target.c

@@ -1644,6 +1644,10 @@ static void era_status(struct dm_target *ti, status_type_t type,
 		format_dev_t(buf, era->origin_dev->bdev->bd_dev);
 		DMEMIT("%s %u", buf, era->sectors_per_block);
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return;

drivers/md/dm-flakey.c

@@ -440,6 +440,10 @@ static void flakey_status(struct dm_target *ti, status_type_t type,
 			       fc->corrupt_bio_value, fc->corrupt_bio_flags);
 
 		break;
+
+	case STATUSTYPE_IMA:
+		result[0] = '\0';
+		break;
 	}
 }
 

drivers/md/dm-ima.h

+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2021 Microsoft Corporation
+ *
+ * Author: Tushar Sugandhi <tusharsu@linux.microsoft.com>
+ *
+ * File: dm-ima.h
+ *       Header file for device mapper IMA measurements.
+ */
+
+#ifndef DM_IMA_H
+#define DM_IMA_H
+
+#define DM_IMA_MEASUREMENT_BUF_LEN	4096
+#define DM_IMA_DEVICE_BUF_LEN		1024
+#define DM_IMA_TARGET_METADATA_BUF_LEN	128
+#define DM_IMA_TARGET_DATA_BUF_LEN	2048
+#define DM_IMA_DEVICE_CAPACITY_BUF_LEN	128
+#define DM_IMA_TABLE_HASH_ALG		"sha256"
+
+#define __dm_ima_stringify(s) #s
+#define __dm_ima_str(s) __dm_ima_stringify(s)
+
+#define DM_IMA_VERSION_STR "dm_version="	\
+	__dm_ima_str(DM_VERSION_MAJOR) "."	\
+	__dm_ima_str(DM_VERSION_MINOR) "."	\
+	__dm_ima_str(DM_VERSION_PATCHLEVEL) ";"
+
+#ifdef CONFIG_IMA
+
+struct dm_ima_device_table_metadata {
+	/*
+	 * Contains data specific to the device which is common across
+	 * all the targets in the table (e.g. name, uuid, major, minor, etc).
+	 * The values are stored in comma separated list of key1=val1,key2=val2;
+	 * pairs delimited by a semicolon at the end of the list.
+	 */
+	char *device_metadata;
+	unsigned int device_metadata_len;
+	unsigned int num_targets;
+
+	/*
+	 * Contains the sha256 hashes of the IMA measurements of the target
+	 * attributes' key-value pairs from the active/inactive tables.
+	 */
+	char *hash;
+	unsigned int hash_len;
+};
+
+/*
+ * This structure contains device metadata, and table hash for
+ * active and inactive tables for ima measurements.
+ */
+struct dm_ima_measurements {
+	struct dm_ima_device_table_metadata active_table;
+	struct dm_ima_device_table_metadata inactive_table;
+	unsigned int dm_version_str_len;
+};
+
+void dm_ima_reset_data(struct mapped_device *md);
+void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags);
+void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap);
+void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all);
+void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map);
+void dm_ima_measure_on_device_rename(struct mapped_device *md);
+
+#else
+
+static inline void dm_ima_reset_data(struct mapped_device *md) {}
+static inline void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags) {}
+static inline void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap) {}
+static inline void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all) {}
+static inline void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map) {}
+static inline void dm_ima_measure_on_device_rename(struct mapped_device *md) {}
+
+#endif /* CONFIG_IMA */
+
+#endif /* DM_IMA_H */

drivers/md/dm-integrity.c

@@ -3306,6 +3306,30 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 		EMIT_ALG(journal_mac_alg, "journal_mac");
 		break;
 	}
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c",
+			ic->dev->name, ic->start, ic->tag_size, ic->mode);
+
+		if (ic->meta_dev)
+			DMEMIT(",meta_device=%s", ic->meta_dev->name);
+		if (ic->sectors_per_block != 1)
+			DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT);
+
+		DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
+		       'y' : 'n');
+		DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
+		DMEMIT(",fix_padding=%c",
+		       ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
+		DMEMIT(",fix_hmac=%c",
+		       ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n');
+		DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n');
+
+		DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS);
+		DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors);
+		DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors);
+		DMEMIT(";");
+		break;
 	}
 }
 

drivers/md/dm-ioctl.c

@@ -6,7 +6,7 @@
  */
 
 #include "dm-core.h"
-
+#include "dm-ima.h"
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/miscdevice.h>
@@ -20,6 +20,7 @@
 #include <linux/compat.h>
 
 #include <linux/uaccess.h>
+#include <linux/ima.h>
 
 #define DM_MSG_PREFIX "ioctl"
 #define DM_DRIVER_EMAIL "dm-devel@redhat.com"
@@ -347,6 +348,7 @@ static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool
 			dm_sync_table(md);
 			dm_table_destroy(t);
 		}
+		dm_ima_measure_on_device_remove(md, true);
 		dm_put(md);
 		if (likely(keep_open_devices))
 			dm_destroy(md);
@@ -483,6 +485,9 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 		param->flags |= DM_UEVENT_GENERATED_FLAG;
 
 	md = hc->md;
+
+	dm_ima_measure_on_device_rename(md);
+
 	up_write(&_hash_lock);
 	kfree(old_name);
 
@@ -981,6 +986,8 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si
 
 	param->flags &= ~DM_DEFERRED_REMOVE;
 
+	dm_ima_measure_on_device_remove(md, false);
+
 	if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr))
 		param->flags |= DM_UEVENT_GENERATED_FLAG;
 
@@ -1159,8 +1166,12 @@ static int do_resume(struct dm_ioctl *param)
 
 	if (dm_suspended_md(md)) {
 		r = dm_resume(md);
-		if (!r && !dm_kobject_uevent(md, KOBJ_CHANGE, param->event_nr))
-			param->flags |= DM_UEVENT_GENERATED_FLAG;
+		if (!r) {
+			dm_ima_measure_on_device_resume(md, new_map ? true : false);
+
+			if (!dm_kobject_uevent(md, KOBJ_CHANGE, param->event_nr))
+				param->flags |= DM_UEVENT_GENERATED_FLAG;
+		}
 	}
 
 	/*
@@ -1224,6 +1235,8 @@ static void retrieve_status(struct dm_table *table,
 
 	if (param->flags & DM_STATUS_TABLE_FLAG)
 		type = STATUSTYPE_TABLE;
+	else if (param->flags & DM_IMA_MEASUREMENT_FLAG)
+		type = STATUSTYPE_IMA;
 	else
 		type = STATUSTYPE_INFO;
 
@@ -1425,6 +1438,8 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si
 	if (r)
 		goto err_unlock_md_type;
 
+	dm_ima_measure_on_table_load(t, STATUSTYPE_IMA);
+
 	immutable_target_type = dm_get_immutable_target_type(md);
 	if (immutable_target_type &&
 	    (immutable_target_type != dm_table_get_immutable_target_type(t)) &&
@@ -1493,6 +1508,7 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 	struct hash_cell *hc;
 	struct mapped_device *md;
 	struct dm_table *old_map = NULL;
+	bool has_new_map = false;
 
 	down_write(&_hash_lock);
 
@@ -1506,6 +1522,7 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 	if (hc->new_map) {
 		old_map = hc->new_map;
 		hc->new_map = NULL;
+		has_new_map = true;
 	}
 
 	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
@@ -1517,6 +1534,7 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 		dm_sync_table(md);
 		dm_table_destroy(old_map);
 	}
+	dm_ima_measure_on_table_clear(md, has_new_map);
 	dm_put(md);
 
 	return 0;

drivers/md/dm-linear.c

@@ -106,6 +106,7 @@ static void linear_status(struct dm_target *ti, status_type_t type,
 			  unsigned status_flags, char *result, unsigned maxlen)
 {
 	struct linear_c *lc = (struct linear_c *) ti->private;
+	size_t sz = 0;
 
 	switch (type) {
 	case STATUSTYPE_INFO:
@@ -113,8 +114,13 @@ static void linear_status(struct dm_target *ti, status_type_t type,
 		break;
 
 	case STATUSTYPE_TABLE:
-		snprintf(result, maxlen, "%s %llu", lc->dev->name,
-				(unsigned long long)lc->start);
+		DMEMIT("%s %llu", lc->dev->name, (unsigned long long)lc->start);
+		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",device_name=%s,start=%llu;", lc->dev->name,
+		       (unsigned long long)lc->start);
 		break;
 	}
 }

drivers/md/dm-log-userspace-base.c

@@ -820,6 +820,9 @@ static int userspace_status(struct dm_dirty_log *log, status_type_t status_type,
 			DMEMIT("integrated_flush ");
 		DMEMIT("%s ", table_args);
 		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 	return (r) ? 0 : (int)sz;
 }

drivers/md/dm-log-writes.c

@@ -834,6 +834,10 @@ static void log_writes_status(struct dm_target *ti, status_type_t type,
 	case STATUSTYPE_TABLE:
 		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 

drivers/md/dm-log.c

@@ -793,6 +793,11 @@ static int core_status(struct dm_dirty_log *log, status_type_t status,
 		DMEMIT("%s %u %u ", log->type->name,
 		       lc->sync == DEFAULTSYNC ? 1 : 2, lc->region_size);
 		DMEMIT_SYNC;
+		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return sz;
@@ -817,6 +822,11 @@ static int disk_status(struct dm_dirty_log *log, status_type_t status,
 		       lc->sync == DEFAULTSYNC ? 2 : 3, lc->log_dev->name,
 		       lc->region_size);
 		DMEMIT_SYNC;
+		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return sz;

drivers/md/dm-mpath.c

@@ -1790,7 +1790,7 @@ static void multipath_resume(struct dm_target *ti)
 static void multipath_status(struct dm_target *ti, status_type_t type,
 			     unsigned status_flags, char *result, unsigned maxlen)
 {
-	int sz = 0;
+	int sz = 0, pg_counter, pgpath_counter;
 	unsigned long flags;
 	struct multipath *m = ti->private;
 	struct priority_group *pg;
@@ -1904,6 +1904,44 @@ static void multipath_status(struct dm_target *ti, status_type_t type,
 			}
 		}
 		break;
+
+	case STATUSTYPE_IMA:
+		sz = 0; /*reset the result pointer*/
+
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",nr_priority_groups=%u", m->nr_priority_groups);
+
+		pg_counter = 0;
+		list_for_each_entry(pg, &m->priority_groups, list) {
+			if (pg->bypassed)
+				state = 'D';	/* Disabled */
+			else if (pg == m->current_pg)
+				state = 'A';	/* Currently Active */
+			else
+				state = 'E';	/* Enabled */
+			DMEMIT(",pg_state_%d=%c", pg_counter, state);
+			DMEMIT(",nr_pgpaths_%d=%u", pg_counter, pg->nr_pgpaths);
+			DMEMIT(",path_selector_name_%d=%s", pg_counter, pg->ps.type->name);
+
+			pgpath_counter = 0;
+			list_for_each_entry(p, &pg->pgpaths, list) {
+				DMEMIT(",path_name_%d_%d=%s,is_active_%d_%d=%c,fail_count_%d_%d=%u",
+				       pg_counter, pgpath_counter, p->path.dev->name,
+				       pg_counter, pgpath_counter, p->is_active ? 'A' : 'F',
+				       pg_counter, pgpath_counter, p->fail_count);
+				if (pg->ps.type->status) {
+					DMEMIT(",path_selector_status_%d_%d=",
+					       pg_counter, pgpath_counter);
+					sz += pg->ps.type->status(&pg->ps, &p->path,
+								  type, result + sz,
+								  maxlen - sz);
+				}
+				pgpath_counter++;
+			}
+			pg_counter++;
+		}
+		DMEMIT(";");
+		break;
 	}
 
 	spin_unlock_irqrestore(&m->lock, flags);

drivers/md/dm-ps-historical-service-time.c

@@ -255,6 +255,9 @@ static int hst_status(struct path_selector *ps, struct dm_path *path,
 		case STATUSTYPE_TABLE:
 			DMEMIT("0 ");
 			break;
+		case STATUSTYPE_IMA:
+			*result = '\0';
+			break;
 		}
 	}
 

drivers/md/dm-ps-io-affinity.c

@@ -170,6 +170,9 @@ static int ioa_status(struct path_selector *ps, struct dm_path *path,
 		pi = path->pscontext;
 		DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));
 		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return sz;

drivers/md/dm-ps-queue-length.c

@@ -102,6 +102,9 @@ static int ql_status(struct path_selector *ps, struct dm_path *path,
 		case STATUSTYPE_TABLE:
 			DMEMIT("%u ", pi->repeat_count);
 			break;
+		case STATUSTYPE_IMA:
+			*result = '\0';
+			break;
 		}
 	}
 

drivers/md/dm-ps-round-robin.c

@@ -100,6 +100,10 @@ static int rr_status(struct path_selector *ps, struct dm_path *path,
 			pi = path->pscontext;
 			DMEMIT("%u ", pi->repeat_count);
 			break;
+
+		case STATUSTYPE_IMA:
+			*result = '\0';
+			break;
 		}
 	}
 

drivers/md/dm-ps-service-time.c

@@ -99,6 +99,9 @@ static int st_status(struct path_selector *ps, struct dm_path *path,
 			DMEMIT("%u %u ", pi->repeat_count,
 			       pi->relative_throughput);
 			break;
+		case STATUSTYPE_IMA:
+			result[0] = '\0';
+			break;
 		}
 	}
 

drivers/md/dm-raid.c

@@ -3671,6 +3671,45 @@ static void raid_status(struct dm_target *ti, status_type_t type,
 		for (i = 0; i < rs->raid_disks; i++)
 			DMEMIT(" %s %s", __get_dev_name(rs->dev[i].meta_dev),
 					 __get_dev_name(rs->dev[i].data_dev));
+		break;
+
+	case STATUSTYPE_IMA:
+		rt = get_raid_type_by_ll(mddev->new_level, mddev->new_layout);
+		if (!rt)
+			return;
+
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",raid_type=%s,raid_disks=%d", rt->name, mddev->raid_disks);
+
+		/* Access most recent mddev properties for status output */
+		smp_rmb();
+		recovery = rs->md.recovery;
+		state = decipher_sync_action(mddev, recovery);
+		DMEMIT(",raid_state=%s", sync_str(state));
+
+		for (i = 0; i < rs->raid_disks; i++) {
+			DMEMIT(",raid_device_%d_status=", i);
+			DMEMIT(__raid_dev_status(rs, &rs->dev[i].rdev));
+		}
+
+		if (rt_is_raid456(rt)) {
+			DMEMIT(",journal_dev_mode=");
+			switch (rs->journal_dev.mode) {
+			case R5C_JOURNAL_MODE_WRITE_THROUGH:
+				DMEMIT("%s",
+				       _raid456_journal_mode[R5C_JOURNAL_MODE_WRITE_THROUGH].param);
+				break;
+			case R5C_JOURNAL_MODE_WRITE_BACK:
+				DMEMIT("%s",
+				       _raid456_journal_mode[R5C_JOURNAL_MODE_WRITE_BACK].param);
+				break;
+			default:
+				DMEMIT("invalid");
+				break;
+			}
+		}
+		DMEMIT(";");
+		break;
 	}
 }
 

drivers/md/dm-raid1.c

@@ -1435,6 +1435,23 @@ static void mirror_status(struct dm_target *ti, status_type_t type,
 		}
 
 		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",nr_mirrors=%d", ms->nr_mirrors);
+		for (m = 0; m < ms->nr_mirrors; m++) {
+			DMEMIT(",mirror_device_%d=%s", m, ms->mirror[m].dev->name);
+			DMEMIT(",mirror_device_%d_status=%c",
+			       m, device_status_char(&(ms->mirror[m])));
+		}
+
+		DMEMIT(",handle_errors=%c", errors_handled(ms) ? 'y' : 'n');
+		DMEMIT(",keep_log=%c", keep_log(ms) ? 'y' : 'n');
+
+		DMEMIT(",log_type_status=");
+		sz += log->type->status(log, type, result+sz, maxlen-sz);
+		DMEMIT(";");
+		break;
 	}
 }
 

drivers/md/dm-snap-persistent.c

@@ -908,6 +908,10 @@ static unsigned persistent_status(struct dm_exception_store *store,
 	case STATUSTYPE_TABLE:
 		DMEMIT(" %s %llu", store->userspace_supports_overflow ? "PO" : "P",
 		       (unsigned long long)store->chunk_size);
+		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return sz;

drivers/md/dm-snap-transient.c

@@ -95,6 +95,10 @@ static unsigned transient_status(struct dm_exception_store *store,
 		break;
 	case STATUSTYPE_TABLE:
 		DMEMIT(" N %llu", (unsigned long long)store->chunk_size);
+		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 
 	return sz;

drivers/md/dm-snap.c

@@ -2390,6 +2390,16 @@ static void snapshot_status(struct dm_target *ti, status_type_t type,
 				DMEMIT(" discard_passdown_origin");
 		}
 		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",snap_origin_name=%s", snap->origin->name);
+		DMEMIT(",snap_cow_name=%s", snap->cow->name);
+		DMEMIT(",snap_valid=%c", snap->valid ? 'y' : 'n');
+		DMEMIT(",snap_merge_failed=%c", snap->merge_failed ? 'y' : 'n');
+		DMEMIT(",snapshot_overflowed=%c", snap->snapshot_overflowed ? 'y' : 'n');
+		DMEMIT(";");
+		break;
 	}
 }
 
@@ -2734,6 +2744,9 @@ static void origin_status(struct dm_target *ti, status_type_t type,
 	case STATUSTYPE_TABLE:
 		snprintf(result, maxlen, "%s", o->dev->name);
 		break;
+	case STATUSTYPE_IMA:
+		result[0] = '\0';
+		break;
 	}
 }
 

drivers/md/dm-stripe.c

@@ -428,6 +428,21 @@ static void stripe_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" %s %llu", sc->stripe[i].dev->name,
 			    (unsigned long long)sc->stripe[i].physical_start);
 		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",stripes=%d,chunk_size=%llu", sc->stripes,
+		       (unsigned long long)sc->chunk_size);
+
+		for (i = 0; i < sc->stripes; i++) {
+			DMEMIT(",stripe_%d_device_name=%s", i, sc->stripe[i].dev->name);
+			DMEMIT(",stripe_%d_physical_start=%llu", i,
+			       (unsigned long long)sc->stripe[i].physical_start);
+			DMEMIT(",stripe_%d_status=%c", i,
+			       atomic_read(&(sc->stripe[i].error_count)) ? 'D' : 'A');
+		}
+		DMEMIT(";");
+		break;
 	}
 }
 

drivers/md/dm-switch.c

@@ -504,6 +504,10 @@ static void switch_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" %s %llu", sctx->path_list[path_nr].dmdev->name,
 			       (unsigned long long)sctx->path_list[path_nr].start);
 		break;
+
+	case STATUSTYPE_IMA:
+		result[0] = '\0';
+		break;
 	}
 }
 

drivers/md/dm-thin.c

@@ -4012,6 +4012,10 @@ static void pool_status(struct dm_target *ti, status_type_t type,
 		       (unsigned long long)pt->low_water_blocks);
 		emit_flags(&pt->requested_pf, result, sz, maxlen);
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 	return;
 
@@ -4423,6 +4427,10 @@ static void thin_status(struct dm_target *ti, status_type_t type,
 			if (tc->origin_dev)
 				DMEMIT(" %s", format_dev_t(buf, tc->origin_dev->bdev->bd_dev));
 			break;
+
+		case STATUSTYPE_IMA:
+			*result = '\0';
+			break;
 		}
 	}
 

drivers/md/dm-unstripe.c

@@ -156,6 +156,10 @@ static void unstripe_status(struct dm_target *ti, status_type_t type,
 		       uc->stripes, (unsigned long long)uc->chunk_size, uc->unstripe,
 		       uc->dev->name, (unsigned long long)uc->physical_start);
 		break;
+
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 

drivers/md/dm-verity-target.c

@@ -772,6 +772,49 @@ static void verity_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
 				" %s", v->signature_key_desc);
 		break;
+
+	case STATUSTYPE_IMA:
+		DMEMIT_TARGET_NAME_VERSION(ti->type);
+		DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V');
+		DMEMIT(",verity_version=%u", v->version);
+		DMEMIT(",data_device_name=%s", v->data_dev->name);
+		DMEMIT(",hash_device_name=%s", v->hash_dev->name);
+		DMEMIT(",verity_algorithm=%s", v->alg_name);
+
+		DMEMIT(",root_digest=");
+		for (x = 0; x < v->digest_size; x++)
+			DMEMIT("%02x", v->root_digest[x]);
+
+		DMEMIT(",salt=");
+		if (!v->salt_size)
+			DMEMIT("-");
+		else
+			for (x = 0; x < v->salt_size; x++)
+				DMEMIT("%02x", v->salt[x]);
+
+		DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n');
+		DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n');
+		if (v->signature_key_desc)
+			DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc);
+
+		if (v->mode != DM_VERITY_MODE_EIO) {
+			DMEMIT(",verity_mode=");
+			switch (v->mode) {
+			case DM_VERITY_MODE_LOGGING:
+				DMEMIT(DM_VERITY_OPT_LOGGING);
+				break;
+			case DM_VERITY_MODE_RESTART:
+				DMEMIT(DM_VERITY_OPT_RESTART);
+				break;
+			case DM_VERITY_MODE_PANIC:
+				DMEMIT(DM_VERITY_OPT_PANIC);
+				break;
+			default:
+				DMEMIT("invalid");
+			}
+		}
+		DMEMIT(";");
+		break;
 	}
 }
 

drivers/md/dm-writecache.c

@@ -206,6 +206,19 @@ struct dm_writecache {
 
 	struct bio_set bio_set;
 	mempool_t copy_pool;
+
+	struct {
+		unsigned long long reads;
+		unsigned long long read_hits;
+		unsigned long long writes;
+		unsigned long long write_hits_uncommitted;
+		unsigned long long write_hits_committed;
+		unsigned long long writes_around;
+		unsigned long long writes_allocate;
+		unsigned long long writes_blocked_on_freelist;
+		unsigned long long flushes;
+		unsigned long long discards;
+	} stats;
 };
 
 #define WB_LIST_INLINE		16
@@ -1157,6 +1170,18 @@ static int process_cleaner_mesg(unsigned argc, char **argv, struct dm_writecache
 	return 0;
 }
 
+static int process_clear_stats_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
+{
+	if (argc != 1)
+		return -EINVAL;
+
+	wc_lock(wc);
+	memset(&wc->stats, 0, sizeof wc->stats);
+	wc_unlock(wc);
+
+	return 0;
+}
+
 static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
 			      char *result, unsigned maxlen)
 {
@@ -1169,6 +1194,8 @@ static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
 		r = process_flush_on_suspend_mesg(argc, argv, wc);
 	else if (!strcasecmp(argv[0], "cleaner"))
 		r = process_cleaner_mesg(argc, argv, wc);
+	else if (!strcasecmp(argv[0], "clear_stats"))
+		r = process_clear_stats_mesg(argc, argv, wc);
 	else
 		DMERR("unrecognised message received: %s", argv[0]);
 
@@ -1293,216 +1320,278 @@ static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
 	bio_list_add(&wc->flush_list, bio);
 }
 
-static int writecache_map(struct dm_target *ti, struct bio *bio)
+enum wc_map_op {
+	WC_MAP_SUBMIT,
+	WC_MAP_REMAP,
+	WC_MAP_REMAP_ORIGIN,
+	WC_MAP_RETURN,
+	WC_MAP_ERROR,
+};
+
+static enum wc_map_op writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
+						  struct wc_entry *e)
 {
-	struct wc_entry *e;
-	struct dm_writecache *wc = ti->private;
+	if (e) {
+		sector_t next_boundary =
+			read_original_sector(wc, e) - bio->bi_iter.bi_sector;
+		if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
+			dm_accept_partial_bio(bio, next_boundary);
+	}
 
-	bio->bi_private = NULL;
+	return WC_MAP_REMAP_ORIGIN;
+}
 
-	wc_lock(wc);
+static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
+{
+	enum wc_map_op map_op;
+	struct wc_entry *e;
 
-	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
-		if (writecache_has_error(wc))
-			goto unlock_error;
+read_next_block:
+	wc->stats.reads++;
+	e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
+	if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
+		wc->stats.read_hits++;
 		if (WC_MODE_PMEM(wc)) {
-			writecache_flush(wc);
-			if (writecache_has_error(wc))
-				goto unlock_error;
-			if (unlikely(wc->cleaner) || unlikely(wc->metadata_only))
-				goto unlock_remap_origin;
-			goto unlock_submit;
+			bio_copy_block(wc, bio, memory_data(wc, e));
+			if (bio->bi_iter.bi_size)
+				goto read_next_block;
+			map_op = WC_MAP_SUBMIT;
 		} else {
-			if (dm_bio_get_target_bio_nr(bio))
-				goto unlock_remap_origin;
-			writecache_offload_bio(wc, bio);
-			goto unlock_return;
+			dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
+			bio_set_dev(bio, wc->ssd_dev->bdev);
+			bio->bi_iter.bi_sector = cache_sector(wc, e);
+			if (!writecache_entry_is_committed(wc, e))
+				writecache_wait_for_ios(wc, WRITE);
+			map_op = WC_MAP_REMAP;
 		}
+	} else {
+		map_op = writecache_map_remap_origin(wc, bio, e);
 	}
 
-	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
+	return map_op;
+}
 
-	if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
-				(wc->block_size / 512 - 1)) != 0)) {
-		DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
-		      (unsigned long long)bio->bi_iter.bi_sector,
-		      bio->bi_iter.bi_size, wc->block_size);
-		goto unlock_error;
-	}
+static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
+					      struct wc_entry *e, bool search_used)
+{
+	unsigned bio_size = wc->block_size;
+	sector_t start_cache_sec = cache_sector(wc, e);
+	sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
 
-	if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
-		if (writecache_has_error(wc))
-			goto unlock_error;
-		if (WC_MODE_PMEM(wc)) {
-			writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
-			goto unlock_remap_origin;
+	while (bio_size < bio->bi_iter.bi_size) {
+		if (!search_used) {
+			struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
+			if (!f)
+				break;
+			write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
+							(bio_size >> SECTOR_SHIFT), wc->seq_count);
+			writecache_insert_entry(wc, f);
+			wc->uncommitted_blocks++;
 		} else {
-			writecache_offload_bio(wc, bio);
-			goto unlock_return;
+			struct wc_entry *f;
+			struct rb_node *next = rb_next(&e->rb_node);
+			if (!next)
+				break;
+			f = container_of(next, struct wc_entry, rb_node);
+			if (f != e + 1)
+				break;
+			if (read_original_sector(wc, f) !=
+			    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
+				break;
+			if (unlikely(f->write_in_progress))
+				break;
+			if (writecache_entry_is_committed(wc, f))
+				wc->overwrote_committed = true;
+			e = f;
 		}
+		bio_size += wc->block_size;
+		current_cache_sec += wc->block_size >> SECTOR_SHIFT;
 	}
 
-	if (bio_data_dir(bio) == READ) {
-read_next_block:
-		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
-		if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
-			if (WC_MODE_PMEM(wc)) {
-				bio_copy_block(wc, bio, memory_data(wc, e));
-				if (bio->bi_iter.bi_size)
-					goto read_next_block;
-				goto unlock_submit;
-			} else {
-				dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
-				bio_set_dev(bio, wc->ssd_dev->bdev);
-				bio->bi_iter.bi_sector = cache_sector(wc, e);
-				if (!writecache_entry_is_committed(wc, e))
-					writecache_wait_for_ios(wc, WRITE);
-				goto unlock_remap;
+	bio_set_dev(bio, wc->ssd_dev->bdev);
+	bio->bi_iter.bi_sector = start_cache_sec;
+	dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
+
+	if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
+		wc->uncommitted_blocks = 0;
+		queue_work(wc->writeback_wq, &wc->flush_work);
+	} else {
+		writecache_schedule_autocommit(wc);
+	}
+
+	return WC_MAP_REMAP;
+}
+
+static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
+{
+	struct wc_entry *e;
+
+	do {
+		bool found_entry = false;
+		bool search_used = false;
+		wc->stats.writes++;
+		if (writecache_has_error(wc))
+			return WC_MAP_ERROR;
+		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
+		if (e) {
+			if (!writecache_entry_is_committed(wc, e)) {
+				wc->stats.write_hits_uncommitted++;
+				search_used = true;
+				goto bio_copy;
 			}
-		} else {
-			if (e) {
-				sector_t next_boundary =
-					read_original_sector(wc, e) - bio->bi_iter.bi_sector;
-				if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
-					dm_accept_partial_bio(bio, next_boundary);
-				}
+			wc->stats.write_hits_committed++;
+			if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
+				wc->overwrote_committed = true;
+				search_used = true;
+				goto bio_copy;
 			}
-			goto unlock_remap_origin;
+			found_entry = true;
+		} else {
+			if (unlikely(wc->cleaner) ||
+			    (wc->metadata_only && !(bio->bi_opf & REQ_META)))
+				goto direct_write;
 		}
-	} else {
-		do {
-			bool found_entry = false;
-			bool search_used = false;
-			if (writecache_has_error(wc))
-				goto unlock_error;
-			e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
-			if (e) {
-				if (!writecache_entry_is_committed(wc, e)) {
-					search_used = true;
-					goto bio_copy;
-				}
-				if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
-					wc->overwrote_committed = true;
-					search_used = true;
-					goto bio_copy;
-				}
-				found_entry = true;
-			} else {
-				if (unlikely(wc->cleaner) ||
-				    (wc->metadata_only && !(bio->bi_opf & REQ_META)))
-					goto direct_write;
-			}
-			e = writecache_pop_from_freelist(wc, (sector_t)-1);
-			if (unlikely(!e)) {
-				if (!WC_MODE_PMEM(wc) && !found_entry) {
+		e = writecache_pop_from_freelist(wc, (sector_t)-1);
+		if (unlikely(!e)) {
+			if (!WC_MODE_PMEM(wc) && !found_entry) {
 direct_write:
-					e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
-					if (e) {
-						sector_t next_boundary = read_original_sector(wc, e) - bio->bi_iter.bi_sector;
-						BUG_ON(!next_boundary);
-						if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
-							dm_accept_partial_bio(bio, next_boundary);
-						}
-					}
-					goto unlock_remap_origin;
-				}
-				writecache_wait_on_freelist(wc);
-				continue;
+				wc->stats.writes_around++;
+				e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
+				return writecache_map_remap_origin(wc, bio, e);
 			}
-			write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
-			writecache_insert_entry(wc, e);
-			wc->uncommitted_blocks++;
+			wc->stats.writes_blocked_on_freelist++;
+			writecache_wait_on_freelist(wc);
+			continue;
+		}
+		write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
+		writecache_insert_entry(wc, e);
+		wc->uncommitted_blocks++;
+		wc->stats.writes_allocate++;
 bio_copy:
-			if (WC_MODE_PMEM(wc)) {
-				bio_copy_block(wc, bio, memory_data(wc, e));
-			} else {
-				unsigned bio_size = wc->block_size;
-				sector_t start_cache_sec = cache_sector(wc, e);
-				sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
-
-				while (bio_size < bio->bi_iter.bi_size) {
-					if (!search_used) {
-						struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
-						if (!f)
-							break;
-						write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
-										(bio_size >> SECTOR_SHIFT), wc->seq_count);
-						writecache_insert_entry(wc, f);
-						wc->uncommitted_blocks++;
-					} else {
-						struct wc_entry *f;
-						struct rb_node *next = rb_next(&e->rb_node);
-						if (!next)
-							break;
-						f = container_of(next, struct wc_entry, rb_node);
-						if (f != e + 1)
-							break;
-						if (read_original_sector(wc, f) !=
-						    read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
-							break;
-						if (unlikely(f->write_in_progress))
-							break;
-						if (writecache_entry_is_committed(wc, f))
-							wc->overwrote_committed = true;
-						e = f;
-					}
-					bio_size += wc->block_size;
-					current_cache_sec += wc->block_size >> SECTOR_SHIFT;
-				}
+		if (WC_MODE_PMEM(wc))
+			bio_copy_block(wc, bio, memory_data(wc, e));
+		else
+			return writecache_bio_copy_ssd(wc, bio, e, search_used);
+	} while (bio->bi_iter.bi_size);
 
-				bio_set_dev(bio, wc->ssd_dev->bdev);
-				bio->bi_iter.bi_sector = start_cache_sec;
-				dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
+	if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks))
+		writecache_flush(wc);
+	else
+		writecache_schedule_autocommit(wc);
 
-				if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
-					wc->uncommitted_blocks = 0;
-					queue_work(wc->writeback_wq, &wc->flush_work);
-				} else {
-					writecache_schedule_autocommit(wc);
-				}
-				goto unlock_remap;
-			}
-		} while (bio->bi_iter.bi_size);
+	return WC_MAP_SUBMIT;
+}
 
-		if (unlikely(bio->bi_opf & REQ_FUA ||
-			     wc->uncommitted_blocks >= wc->autocommit_blocks))
-			writecache_flush(wc);
-		else
-			writecache_schedule_autocommit(wc);
-		goto unlock_submit;
+static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio *bio)
+{
+	if (writecache_has_error(wc))
+		return WC_MAP_ERROR;
+
+	if (WC_MODE_PMEM(wc)) {
+		wc->stats.flushes++;
+		writecache_flush(wc);
+		if (writecache_has_error(wc))
+			return WC_MAP_ERROR;
+		else if (unlikely(wc->cleaner) || unlikely(wc->metadata_only))
+			return WC_MAP_REMAP_ORIGIN;
+		return WC_MAP_SUBMIT;
 	}
+	/* SSD: */
+	if (dm_bio_get_target_bio_nr(bio))
+		return WC_MAP_REMAP_ORIGIN;
+	wc->stats.flushes++;
+	writecache_offload_bio(wc, bio);
+	return WC_MAP_RETURN;
+}
 
-unlock_remap_origin:
-	if (likely(wc->pause != 0)) {
-		 if (bio_op(bio) == REQ_OP_WRITE) {
-			dm_iot_io_begin(&wc->iot, 1);
-			bio->bi_private = (void *)2;
-		}
+static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
+{
+	wc->stats.discards++;
+
+	if (writecache_has_error(wc))
+		return WC_MAP_ERROR;
+
+	if (WC_MODE_PMEM(wc)) {
+		writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
+		return WC_MAP_REMAP_ORIGIN;
 	}
-	bio_set_dev(bio, wc->dev->bdev);
-	wc_unlock(wc);
-	return DM_MAPIO_REMAPPED;
+	/* SSD: */
+	writecache_offload_bio(wc, bio);
+	return WC_MAP_RETURN;
+}
 
-unlock_remap:
-	/* make sure that writecache_end_io decrements bio_in_progress: */
-	bio->bi_private = (void *)1;
-	atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
-	wc_unlock(wc);
-	return DM_MAPIO_REMAPPED;
+static int writecache_map(struct dm_target *ti, struct bio *bio)
+{
+	struct dm_writecache *wc = ti->private;
+	enum wc_map_op map_op;
 
-unlock_submit:
-	wc_unlock(wc);
-	bio_endio(bio);
-	return DM_MAPIO_SUBMITTED;
+	bio->bi_private = NULL;
 
-unlock_return:
-	wc_unlock(wc);
-	return DM_MAPIO_SUBMITTED;
+	wc_lock(wc);
 
-unlock_error:
-	wc_unlock(wc);
-	bio_io_error(bio);
-	return DM_MAPIO_SUBMITTED;
+	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
+		map_op = writecache_map_flush(wc, bio);
+		goto done;
+	}
+
+	bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
+
+	if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
+				(wc->block_size / 512 - 1)) != 0)) {
+		DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
+		      (unsigned long long)bio->bi_iter.bi_sector,
+		      bio->bi_iter.bi_size, wc->block_size);
+		map_op = WC_MAP_ERROR;
+		goto done;
+	}
+
+	if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
+		map_op = writecache_map_discard(wc, bio);
+		goto done;
+	}
+
+	if (bio_data_dir(bio) == READ)
+		map_op = writecache_map_read(wc, bio);
+	else
+		map_op = writecache_map_write(wc, bio);
+done:
+	switch (map_op) {
+	case WC_MAP_REMAP_ORIGIN:
+		if (likely(wc->pause != 0)) {
+			if (bio_op(bio) == REQ_OP_WRITE) {
+				dm_iot_io_begin(&wc->iot, 1);
+				bio->bi_private = (void *)2;
+			}
+		}
+		bio_set_dev(bio, wc->dev->bdev);
+		wc_unlock(wc);
+		return DM_MAPIO_REMAPPED;
+
+	case WC_MAP_REMAP:
+		/* make sure that writecache_end_io decrements bio_in_progress: */
+		bio->bi_private = (void *)1;
+		atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
+		wc_unlock(wc);
+		return DM_MAPIO_REMAPPED;
+
+	case WC_MAP_SUBMIT:
+		wc_unlock(wc);
+		bio_endio(bio);
+		return DM_MAPIO_SUBMITTED;
+
+	case WC_MAP_RETURN:
+		wc_unlock(wc);
+		return DM_MAPIO_SUBMITTED;
+
+	case WC_MAP_ERROR:
+		wc_unlock(wc);
+		bio_io_error(bio);
+		return DM_MAPIO_SUBMITTED;
+
+	default:
+		BUG();
+		return -1;
+	}
 }
 
 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
@@ -2568,9 +2657,20 @@ static void writecache_status(struct dm_target *ti, status_type_t type,
 
 	switch (type) {
 	case STATUSTYPE_INFO:
-		DMEMIT("%ld %llu %llu %llu", writecache_has_error(wc),
+		DMEMIT("%ld %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu",
+		       writecache_has_error(wc),
 		       (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
-		       (unsigned long long)wc->writeback_size);
+		       (unsigned long long)wc->writeback_size,
+		       wc->stats.reads,
+		       wc->stats.read_hits,
+		       wc->stats.writes,
+		       wc->stats.write_hits_uncommitted,
+		       wc->stats.write_hits_committed,
+		       wc->stats.writes_around,
+		       wc->stats.writes_allocate,
+		       wc->stats.writes_blocked_on_freelist,
+		       wc->stats.flushes,
+		       wc->stats.discards);
 		break;
 	case STATUSTYPE_TABLE:
 		DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
@@ -2623,12 +2723,15 @@ static void writecache_status(struct dm_target *ti, status_type_t type,
 		if (wc->pause_set)
 			DMEMIT(" pause_writeback %u", wc->pause_value);
 		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 }
 
 static struct target_type writecache_target = {
 	.name			= "writecache",
-	.version		= {1, 5, 0},
+	.version		= {1, 6, 0},
 	.module			= THIS_MODULE,
 	.ctr			= writecache_ctr,
 	.dtr			= writecache_dtr,

drivers/md/dm-zoned-target.c

@@ -1119,6 +1119,9 @@ static void dmz_status(struct dm_target *ti, status_type_t type,
 			DMEMIT(" %s", buf);
 		}
 		break;
+	case STATUSTYPE_IMA:
+		*result = '\0';
+		break;
 	}
 	return;
 }

drivers/md/dm.c

@@ -8,6 +8,7 @@
 #include "dm-core.h"
 #include "dm-rq.h"
 #include "dm-uevent.h"
+#include "dm-ima.h"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -261,9 +262,13 @@ static void (*_exits[])(void) = {
 static int __init dm_init(void)
 {
 	const int count = ARRAY_SIZE(_inits);
-
 	int r, i;
 
+#if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE))
+	DMWARN("CONFIG_IMA_DISABLE_HTABLE is disabled."
+	       " Duplicate IMA measurements will not be recorded in the IMA log.");
+#endif
+
 	for (i = 0; i < count; i++) {
 		r = _inits[i]();
 		if (r)
@@ -271,8 +276,7 @@ static int __init dm_init(void)
 	}
 
 	return 0;
-
-      bad:
+bad:
 	while (i--)
 		_exits[i]();
 
@@ -1997,6 +2001,8 @@ int dm_create(int minor, struct mapped_device **result)
 	if (!md)
 		return -ENXIO;
 
+	dm_ima_reset_data(md);
+
 	*result = md;
 	return 0;
 }

include/linux/device-mapper.h

@@ -31,7 +31,7 @@ enum dm_queue_mode {
 	DM_TYPE_DAX_BIO_BASED	 = 3,
 };
 
-typedef enum { STATUSTYPE_INFO, STATUSTYPE_TABLE } status_type_t;
+typedef enum { STATUSTYPE_INFO, STATUSTYPE_TABLE, STATUSTYPE_IMA } status_type_t;
 
 union map_info {
 	void *ptr;
@@ -602,6 +602,10 @@ void dm_destroy_keyslot_manager(struct blk_keyslot_manager *ksm);
 #define DMEMIT(x...) sz += ((sz >= maxlen) ? \
 			  0 : scnprintf(result + sz, maxlen - sz, x))
 
+#define DMEMIT_TARGET_NAME_VERSION(y) \
+		DMEMIT("target_name=%s,target_version=%u.%u.%u", \
+		       (y)->name, (y)->version[0], (y)->version[1], (y)->version[2])
+
 /*
  * Definitions of return values from target end_io function.
  */

include/uapi/linux/dm-ioctl.h

@@ -376,4 +376,10 @@ enum {
  */
 #define DM_INTERNAL_SUSPEND_FLAG	(1 << 18) /* Out */
 
+/*
+ * If set, returns in the in buffer passed by UM, the raw table information
+ * that would be measured by IMA subsystem on device state change.
+ */
+#define DM_IMA_MEASUREMENT_FLAG	(1 << 19) /* In */
+
 #endif				/* _LINUX_DM_IOCTL_H */

security/integrity/ima/ima_main.c

@@ -985,6 +985,7 @@ void ima_measure_critical_data(const char *event_label,
 				   CRITICAL_DATA, 0, event_label,
 				   hash);
 }
+EXPORT_SYMBOL_GPL(ima_measure_critical_data);
 
 static int __init init_ima(void)
 {