diff --git a/block/bio.c b/block/bio.c index 3d757055305f98e7bb2a381b847cf4000ff5b094..48a8cf55fb7a2fb859a244ae5ca4fe535c91be24 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1668,13 +1668,28 @@ void bio_check_pages_dirty(struct bio *bio) } EXPORT_SYMBOL_GPL(bio_check_pages_dirty); +void update_io_ticks(int cpu, struct hd_struct *part, unsigned long now) +{ + unsigned long stamp; +again: + stamp = READ_ONCE(part->stamp); + if (unlikely(stamp != now)) { + if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) + __part_stat_add(cpu, part, io_ticks, 1); + } + if (part->partno) { + part = &part_to_disk(part)->part0; + goto again; + } +} + void generic_start_io_acct(struct request_queue *q, int op, unsigned long sectors, struct hd_struct *part) { const int sgrp = op_stat_group(op); int cpu = part_stat_lock(); - part_round_stats(q, cpu, part); + update_io_ticks(cpu, part, jiffies); part_stat_inc(cpu, part, ios[sgrp]); part_stat_add(cpu, part, sectors[sgrp], sectors); part_inc_in_flight(q, part, op_is_write(op)); @@ -1686,12 +1701,14 @@ EXPORT_SYMBOL(generic_start_io_acct); void generic_end_io_acct(struct request_queue *q, int req_op, struct hd_struct *part, unsigned long start_time) { - unsigned long duration = jiffies - start_time; + unsigned long now = jiffies; + unsigned long duration = now - start_time; const int sgrp = op_stat_group(req_op); int cpu = part_stat_lock(); + update_io_ticks(cpu, part, now); part_stat_add(cpu, part, nsecs[sgrp], jiffies_to_nsecs(duration)); - part_round_stats(q, cpu, part); + part_stat_add(cpu, part, time_in_queue, duration); part_dec_in_flight(q, part, op_is_write(req_op)); part_stat_unlock(); diff --git a/block/blk-core.c b/block/blk-core.c index b64dec2fd1ccb1b7ffaa36e3110e199354ab4ed3..d9e3ee68f3776591979ee74b87707c3acd3c79c4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2740,9 +2740,10 @@ void blk_account_io_done(struct request *req, u64 now) cpu = part_stat_lock(); part = req->part; + update_io_ticks(cpu, part, jiffies); part_stat_inc(cpu, part, ios[sgrp]); part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns); - part_round_stats(req->q, cpu, part); + part_stat_add(cpu, part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns)); part_dec_in_flight(req->q, part, rq_data_dir(req)); hd_struct_put(part); @@ -2790,11 +2791,12 @@ void blk_account_io_start(struct request *rq, bool new_io) part_stat_inc(cpu, part, merges[rw]); } else { part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); - part_round_stats(rq->q, cpu, part); part_inc_in_flight(rq->q, part, rw); rq->part = part; } + update_io_ticks(cpu, part, jiffies); + part_stat_unlock(); } diff --git a/block/blk-merge.c b/block/blk-merge.c index 7efa8c3e2b727f6367af02e0afb201d938f74b65..044bff9afa5e724147bddd530250b82c2abf1bff 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -656,7 +656,6 @@ static void blk_account_io_merge(struct request *req) cpu = part_stat_lock(); part = req->part; - part_round_stats(req->q, cpu, part); part_dec_in_flight(req->q, part, rq_data_dir(req)); hd_struct_put(part); diff --git a/block/genhd.c b/block/genhd.c index ff9d46d49da1dffe65aa4ce8613cf0e62912aebc..862a2f3c536f4e6f14ce07337a1ebc8ac984c9dd 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1342,7 +1342,6 @@ static int diskstats_show(struct seq_file *seqf, void *v) struct hd_struct *hd; char buf[BDEVNAME_SIZE]; unsigned int inflight[2]; - int cpu; /* if (&disk_to_dev(gp)->kobj.entry == block_class.devices.next) @@ -1354,9 +1353,6 @@ static int diskstats_show(struct seq_file *seqf, void *v) disk_part_iter_init(&piter, gp, DISK_PITER_INCL_EMPTY_PART0); while ((hd = disk_part_iter_next(&piter))) { - cpu = part_stat_lock(); - part_round_stats(gp->queue, cpu, hd); - part_stat_unlock(); part_in_flight(gp->queue, hd, inflight); seq_printf(seqf, "%4d %7d %s " "%lu %lu %lu %u " diff --git a/block/partition-generic.c b/block/partition-generic.c index 8ad6dcaa61c2c99b727ec045a7c7afb40271e775..d86d794d28e1a5cd586c961b763cf26c0882a74b 100644 --- a/block/partition-generic.c +++ b/block/partition-generic.c @@ -121,11 +121,7 @@ ssize_t part_stat_show(struct device *dev, struct hd_struct *p = dev_to_part(dev); struct request_queue *q = part_to_disk(p)->queue; unsigned int inflight[2]; - int cpu; - cpu = part_stat_lock(); - part_round_stats(q, cpu, p); - part_stat_unlock(); part_in_flight(q, p, inflight); return sprintf(buf, "%8lu %8lu %8llu %8u " diff --git a/include/linux/genhd.h b/include/linux/genhd.h index df0d01d45c8db3e9c41653465fb25eb4cd3afc39..666b23a88c6f609a87ff22ddfb26082f45641852 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -422,6 +422,7 @@ static inline void free_part_info(struct hd_struct *part) /* block/blk-core.c */ extern void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part); +void update_io_ticks(int cpu, struct hd_struct *part, unsigned long now); /* block/genhd.c */ extern void device_add_disk(struct device *parent, struct gendisk *disk);