blk-mq: fix kabi broken due to request_wrapper

hulk inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I65K8D
CVE: NA

--------------------------------

Before commit f60df4a0 ("blk-mq: fix kabi broken in struct
request"), drivers will got cmd address right after request, however,
after this commit, drivers will got cmd address after request_wrapper
instead, which is bigger than request and will cause compatibility
issues.

Fix the problem by placing request_wrapper behind cmd, so that the
cmd address for drivers will stay the same.

Before commit:		|request|cmd|
After commit:		|request|request_wrapper|cmd|
With this patch:	|request|cmd|request_wrapper|

Performance test: arm64 Kunpeng-920 96 core

1) null_blk setup:
modprobe null_blk nr_devices=0 &&
    udevadm settle &&
    cd /sys/kernel/config/nullb &&
    mkdir nullb0 &&
    cd nullb0 &&
    echo 0 > completion_nsec &&
    echo 512 > blocksize &&
    echo 0 > home_node &&
    echo 0 > irqmode &&
    echo 1024 > size &&
    echo 0 > memory_backed &&
    echo 2 > queue_mode &&
	echo 4096 > hw_queue_depth &&
	echo 96 > submit_queues &&
    echo 1 > power

2) fio test script:
[global]
ioengine=libaio
direct=1
numjobs=96
iodepth=32
bs=4k
rw=randwrite
allow_mounted_write=0
time_based
runtime=60
group_reporting=1
ioscheduler=none
cpus_allowed_policy=split
cpus_allowed=0-95

[test]
filename=/dev/nullb0

3) iops test result:

without this patch:	23.9M
with this patch:	24.1M

Fixes: f60df4a0 ("blk-mq: fix kabi broken in struct request")
Signed-off-by: NYu Kuai <yukuai3@huawei.com>
Reviewed-by: NHou Tao <houtao1@huawei.com>
Signed-off-by: NZheng Zengkai <zhengzengkai@huawei.com>

block/blk-flush.c

@@ -470,7 +470,7 @@ struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
 					      gfp_t flags)
 {
 	struct blk_flush_queue *fq;
-	int rq_sz = sizeof(struct request_wrapper);
+	int rq_sz = sizeof(struct request) + sizeof(struct request_wrapper);
 
 	fq = kzalloc_node(sizeof(*fq), flags, node);
 	if (!fq)

block/blk-mq.c

@@ -2601,8 +2601,9 @@ static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
 	 * rq_size is the size of the request plus driver payload, rounded
 	 * to the cacheline size
 	 */
-	rq_size = round_up(sizeof(struct request_wrapper) + set->cmd_size,
-				cache_line_size());
+	rq_size = round_up(sizeof(struct request) +
+			   sizeof(struct request_wrapper) + set->cmd_size,
+			   cache_line_size());
 	left = rq_size * depth;
 
 	for (i = 0; i < depth; ) {

block/blk-mq.h

@@ -37,6 +37,19 @@ struct blk_mq_ctx {
 	struct kobject		kobj;
 } ____cacheline_aligned_in_smp;
 
+struct request_wrapper {
+	/* Time that I/O was counted in part_get_stat_info(). */
+	u64 stat_time_ns;
+};
+
+static inline struct request_wrapper *request_to_wrapper(struct request *rq)
+{
+	unsigned long addr = (unsigned long)rq;
+
+	addr += sizeof(*rq) + rq->q->tag_set->cmd_size;
+	return (struct request_wrapper *)addr;
+}
+
 void blk_mq_exit_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
 void blk_mq_wake_waiters(struct request_queue *q);

drivers/scsi/scsi_error.c

@@ -2359,7 +2359,7 @@ scsi_ioctl_reset(struct scsi_device *dev, int __user *arg)
 		return -EIO;
 
 	error = -EIO;
-	rq = kzalloc(sizeof(struct request_wrapper) + sizeof(struct scsi_cmnd) +
+	rq = kzalloc(sizeof(struct request) + sizeof(struct scsi_cmnd) +
 			shost->hostt->cmd_size, GFP_KERNEL);
 	if (!rq)
 		goto out_put_autopm_host;

include/linux/blk-mq.h

@@ -303,15 +303,6 @@ struct blk_mq_queue_data {
 	KABI_RESERVE(1)
 };
 
-struct request_wrapper {
-	struct request rq;
-
-	/* Time that I/O was counted in part_get_stat_info(). */
-	u64 stat_time_ns;
-};
-
-#define request_to_wrapper(_rq) container_of(_rq, struct request_wrapper, rq)
-
 typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
 		bool);
 typedef bool (busy_tag_iter_fn)(struct request *, void *, bool);
@@ -606,7 +597,7 @@ static inline bool blk_should_fake_timeout(struct request_queue *q)
  */
 static inline struct request *blk_mq_rq_from_pdu(void *pdu)
 {
-	return pdu - sizeof(struct request_wrapper);
+	return pdu - sizeof(struct request);
 }
 
 /**
@@ -620,7 +611,7 @@ static inline struct request *blk_mq_rq_from_pdu(void *pdu)
  */
 static inline void *blk_mq_rq_to_pdu(struct request *rq)
 {
-	return request_to_wrapper(rq) + 1;
+	return rq + 1;
 }
 
 static inline struct blk_mq_hw_ctx *queue_hctx(struct request_queue *q, int id)