diff --git a/block/blk-core.c b/block/blk-core.c index 7b6d5c8c036f00bd543ed04b5c39e67bae92dfc3..a4ec5e168312f455faebbc66fe191b78a80967cc 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -62,6 +62,21 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); DEFINE_IDA(blk_queue_ida); +bool precise_iostat; + +static int __init precise_iostat_setup(char *str) +{ + bool precise; + + if (!strtobool(str, &precise)) { + precise_iostat = precise; + pr_info("precise iostat %d\n", precise_iostat); + } + + return 1; +} +__setup("precise_iostat=", precise_iostat_setup); + /* * For queue allocation */ @@ -1258,9 +1273,14 @@ void update_io_ticks(struct hd_struct *part, unsigned long now, bool end) unsigned long stamp; again: stamp = READ_ONCE(part->stamp); - if (unlikely(time_after(now, stamp))) { - if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) + if (unlikely(time_after(now, stamp)) && + likely(cmpxchg(&part->stamp, stamp, now) == stamp)) { + if (precise_iostat) { + if (end || part_in_flight(part)) + __part_stat_add(part, io_ticks, now - stamp); + } else { __part_stat_add(part, io_ticks, end ? now - stamp : 1); + } } if (part->partno) { part = &part_to_disk(part)->part0; @@ -1318,6 +1338,8 @@ void blk_account_io_done(struct request *req, u64 now) #else part_stat_add(part, nsecs[sgrp], now - req->start_time_ns); #endif + if (precise_iostat) + part_stat_local_dec(part, in_flight[rq_data_dir(req)]); part_stat_unlock(); hd_struct_put(part); @@ -1333,6 +1355,8 @@ void blk_account_io_start(struct request *rq) part_stat_lock(); update_io_ticks(rq->part, jiffies, false); + if (precise_iostat) + part_stat_local_inc(rq->part, in_flight[rq_data_dir(rq)]); part_stat_unlock(); } diff --git a/block/blk-merge.c b/block/blk-merge.c index 6518e0ae283516039419341010e242fe7d69654f..cfb88d2fbf389a8ee16a861d01c127d721d56624 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -696,6 +696,8 @@ static void blk_account_io_merge_request(struct request *req) if (blk_do_io_stat(req)) { part_stat_lock(); part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); + if (precise_iostat) + part_stat_local_dec(req->part, in_flight[rq_data_dir(req)]); part_stat_unlock(); hd_struct_put(req->part); diff --git a/block/genhd.c b/block/genhd.c index 021c9c2d7231ae1fad4f656be0c48bac7b14d675..4f6a0be74d852626de49dba5ccb3b5772610370d 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -112,7 +112,7 @@ static void part_stat_read_all(struct hd_struct *part, struct disk_stats *stat) } } -static unsigned int part_in_flight(struct hd_struct *part) +unsigned int part_in_flight(struct hd_struct *part) { unsigned int inflight = 0; int cpu; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 58475edd65f4cfffeddbb6606f9c3581100734f8..e4bcb11d6202622dfb5c182539516cfb136f7800 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -29,6 +29,7 @@ #include #include +extern bool precise_iostat; struct module; struct scsi_ioctl_command; diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 05927a1c6b5b18c8ae00626edfc65ccdca7cc998..959add98b6865aa42e66132df25e55b4b94894a4 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -304,6 +304,7 @@ extern void disk_part_iter_exit(struct disk_part_iter *piter); extern bool disk_has_partitions(struct gendisk *disk); /* block/genhd.c */ +extern unsigned int part_in_flight(struct hd_struct *part); extern void device_add_disk(struct device *parent, struct gendisk *disk, const struct attribute_group **groups); static inline void add_disk(struct gendisk *disk)