diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index ad6843f2e0ab7f34b9f0e6ea891ea6b713473b02..9af7257f429c34dfa444f54d5e4907d7f06bd10a 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "blk-cgroup.h" static DEFINE_SPINLOCK(blkio_list_lock); @@ -55,6 +56,26 @@ struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup) } EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup); +/* + * Add to the appropriate stat variable depending on the request type. + * This should be called with the blkg->stats_lock held. + */ +void io_add_stat(uint64_t *stat, uint64_t add, unsigned int flags) +{ + if (flags & REQ_RW) + stat[IO_WRITE] += add; + else + stat[IO_READ] += add; + /* + * Everywhere in the block layer, an IO is treated as sync if it is a + * read or a SYNC write. We follow the same norm. + */ + if (!(flags & REQ_RW) || flags & REQ_RW_SYNC) + stat[IO_SYNC] += add; + else + stat[IO_ASYNC] += add; +} + void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time) { unsigned long flags; @@ -65,6 +86,41 @@ void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time) } EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used); +void blkiocg_update_request_dispatch_stats(struct blkio_group *blkg, + struct request *rq) +{ + struct blkio_group_stats *stats; + unsigned long flags; + + spin_lock_irqsave(&blkg->stats_lock, flags); + stats = &blkg->stats; + stats->sectors += blk_rq_sectors(rq); + io_add_stat(stats->io_serviced, 1, rq->cmd_flags); + io_add_stat(stats->io_service_bytes, blk_rq_sectors(rq) << 9, + rq->cmd_flags); + spin_unlock_irqrestore(&blkg->stats_lock, flags); +} + +void blkiocg_update_request_completion_stats(struct blkio_group *blkg, + struct request *rq) +{ + struct blkio_group_stats *stats; + unsigned long flags; + unsigned long long now = sched_clock(); + + spin_lock_irqsave(&blkg->stats_lock, flags); + stats = &blkg->stats; + if (time_after64(now, rq->io_start_time_ns)) + io_add_stat(stats->io_service_time, now - rq->io_start_time_ns, + rq->cmd_flags); + if (time_after64(rq->io_start_time_ns, rq->start_time_ns)) + io_add_stat(stats->io_wait_time, + rq->io_start_time_ns - rq->start_time_ns, + rq->cmd_flags); + spin_unlock_irqrestore(&blkg->stats_lock, flags); +} +EXPORT_SYMBOL_GPL(blkiocg_update_request_completion_stats); + void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg, struct blkio_group *blkg, void *key, dev_t dev) { @@ -325,12 +381,12 @@ SHOW_FUNCTION_PER_GROUP(dequeue, get_stat, get_dequeue_stat, 0); #undef SHOW_FUNCTION_PER_GROUP #ifdef CONFIG_DEBUG_BLK_CGROUP -void blkiocg_update_blkio_group_dequeue_stats(struct blkio_group *blkg, +void blkiocg_update_dequeue_stats(struct blkio_group *blkg, unsigned long dequeue) { blkg->stats.dequeue += dequeue; } -EXPORT_SYMBOL_GPL(blkiocg_update_blkio_group_dequeue_stats); +EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats); #endif struct cftype blkio_files[] = { diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 5c5e5294b5066b797221570b93c2f2f3640e8c2b..80010ef64ab0cc5e1ca2c30139d9fb54b7a620f5 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -112,12 +112,12 @@ static inline char *blkg_path(struct blkio_group *blkg) { return blkg->path; } -void blkiocg_update_blkio_group_dequeue_stats(struct blkio_group *blkg, +void blkiocg_update_dequeue_stats(struct blkio_group *blkg, unsigned long dequeue); #else static inline char *blkg_path(struct blkio_group *blkg) { return NULL; } -static inline void blkiocg_update_blkio_group_dequeue_stats( - struct blkio_group *blkg, unsigned long dequeue) {} +static inline void blkiocg_update_dequeue_stats(struct blkio_group *blkg, + unsigned long dequeue) {} #endif #if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE) @@ -130,6 +130,10 @@ extern struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key); void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time); +void blkiocg_update_request_dispatch_stats(struct blkio_group *blkg, + struct request *rq); +void blkiocg_update_request_completion_stats(struct blkio_group *blkg, + struct request *rq); #else struct cgroup; static inline struct blkio_cgroup * @@ -147,5 +151,9 @@ static inline struct blkio_group * blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key) { return NULL; } static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time) {} +static inline void blkiocg_update_request_dispatch_stats( + struct blkio_group *blkg, struct request *rq) {} +static inline void blkiocg_update_request_completion_stats( + struct blkio_group *blkg, struct request *rq) {} #endif #endif /* _BLK_CGROUP_H */ diff --git a/block/blk-core.c b/block/blk-core.c index 9fe174dc74d1dfecca76bb19eb9dda2af53f93bd..1d94f15d7f0d452eb3154e7ce4a9426aa31f1769 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -127,6 +127,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) rq->tag = -1; rq->ref_count = 1; rq->start_time = jiffies; + set_start_time_ns(rq); } EXPORT_SYMBOL(blk_rq_init); @@ -1855,8 +1856,10 @@ void blk_dequeue_request(struct request *rq) * and to it is freed is accounted as io that is in progress at * the driver side. */ - if (blk_account_rq(rq)) + if (blk_account_rq(rq)) { q->in_flight[rq_is_sync(rq)]++; + set_io_start_time_ns(rq); + } } /** @@ -2517,4 +2520,3 @@ int __init blk_dev_init(void) return 0; } - diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index c5161bbf2fe99e0db0a7d61be896ded72b3e8f99..42028e7128a7e71940d2c18aa0fd906fa15892f7 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -855,7 +855,7 @@ cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg) if (!RB_EMPTY_NODE(&cfqg->rb_node)) cfq_rb_erase(&cfqg->rb_node, st); cfqg->saved_workload_slice = 0; - blkiocg_update_blkio_group_dequeue_stats(&cfqg->blkg, 1); + blkiocg_update_dequeue_stats(&cfqg->blkg, 1); } static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq) @@ -1865,6 +1865,7 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq) elv_dispatch_sort(q, rq); cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++; + blkiocg_update_request_dispatch_stats(&cfqq->cfqg->blkg, rq); } /* @@ -3285,6 +3286,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) WARN_ON(!cfqq->dispatched); cfqd->rq_in_driver--; cfqq->dispatched--; + blkiocg_update_request_completion_stats(&cfqq->cfqg->blkg, rq); cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6690e8bae7bb5946396577fcc12a9c85836cb7b5..f3fff8bf85eeda737d71d266b31a87bb4223a4de 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -194,7 +194,10 @@ struct request { struct gendisk *rq_disk; unsigned long start_time; - +#ifdef CONFIG_BLK_CGROUP + unsigned long long start_time_ns; + unsigned long long io_start_time_ns; /* when passed to hardware */ +#endif /* Number of scatter-gather DMA addr+len pairs after * physical address coalescing is performed. */ @@ -1196,6 +1199,21 @@ static inline void put_dev_sector(Sector p) struct work_struct; int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); +#ifdef CONFIG_BLK_CGROUP +static inline void set_start_time_ns(struct request *req) +{ + req->start_time_ns = sched_clock(); +} + +static inline void set_io_start_time_ns(struct request *req) +{ + req->io_start_time_ns = sched_clock(); +} +#else +static inline void set_start_time_ns(struct request *req) {} +static inline void set_io_start_time_ns(struct request *req) {} +#endif + #define MODULE_ALIAS_BLOCKDEV(major,minor) \ MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \