diff --git a/block/blk-core.c b/block/blk-core.c index fe2f457ed27d7f9e627c87c8a143629a83a28aa2..33d5c7d85da1068fc0cf8eb44084cb9cba5098b4 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2991,7 +2991,10 @@ void blk_start_request(struct request *req) blk_dequeue_request(req); if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) { - blk_stat_set_issue(&req->issue_stat, blk_rq_sectors(req)); + req->io_start_time_ns = ktime_get_ns(); +#ifdef CONFIG_BLK_DEV_THROTTLING_LOW + req->throtl_size = blk_rq_sectors(req); +#endif req->rq_flags |= RQF_STATS; wbt_issue(req->q->rq_wb, req); } diff --git a/block/blk-mq.c b/block/blk-mq.c index deb85b5c6c21f716571c5edadb418ba62c125789..17612e04d0414aa2ae3e2bee117f43040ac08123 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -310,6 +310,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, rq->rq_disk = NULL; rq->part = NULL; rq->start_time = jiffies; + rq->io_start_time_ns = 0; rq->nr_phys_segments = 0; #if defined(CONFIG_BLK_DEV_INTEGRITY) rq->nr_integrity_segments = 0; @@ -329,7 +330,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, #ifdef CONFIG_BLK_CGROUP rq->rl = NULL; set_start_time_ns(rq); - rq->io_start_time_ns = 0; + rq->cgroup_io_start_time_ns = 0; #endif data->ctx->rq_dispatched[op_is_sync(op)]++; @@ -669,7 +670,10 @@ void blk_mq_start_request(struct request *rq) trace_block_rq_issue(q, rq); if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { - blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq)); + rq->io_start_time_ns = ktime_get_ns(); +#ifdef CONFIG_BLK_DEV_THROTTLING_LOW + rq->throtl_size = blk_rq_sectors(rq); +#endif rq->rq_flags |= RQF_STATS; wbt_issue(q->rq_wb, rq); } diff --git a/block/blk-stat.c b/block/blk-stat.c index bd365a95fcf86df90222ee0f4d676cf25cf27385..725a881723b0140833dc79ea6c5caa588e6256d5 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -55,11 +55,8 @@ void blk_stat_add(struct request *rq) int bucket; u64 now, value; - now = __blk_stat_time(ktime_to_ns(ktime_get())); - if (now < blk_stat_time(&rq->issue_stat)) - return; - - value = now - blk_stat_time(&rq->issue_stat); + now = ktime_get_ns(); + value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; blk_throtl_stat_add(rq, value); diff --git a/block/blk-stat.h b/block/blk-stat.h index c22049a8125edfba5a0170299ba78a563339a8d5..17c812db0acafab80aed4a62cc2fd1038f5051d9 100644 --- a/block/blk-stat.h +++ b/block/blk-stat.h @@ -8,21 +8,6 @@ #include #include -/* - * from upper: - * 4 bits: reserved for other usage - * 12 bits: size - * 48 bits: time - */ -#define BLK_STAT_RES_BITS 4 -#define BLK_STAT_SIZE_BITS 12 -#define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS) -#define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS) -#define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1) -#define BLK_STAT_SIZE_MASK \ - (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT) -#define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1)) - /** * struct blk_stat_callback - Block statistics callback. * @@ -82,34 +67,6 @@ void blk_free_queue_stats(struct blk_queue_stats *); void blk_stat_add(struct request *); -static inline u64 __blk_stat_time(u64 time) -{ - return time & BLK_STAT_TIME_MASK; -} - -static inline u64 blk_stat_time(struct blk_issue_stat *stat) -{ - return __blk_stat_time(stat->stat); -} - -static inline sector_t blk_capped_size(sector_t size) -{ - return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1); -} - -static inline sector_t blk_stat_size(struct blk_issue_stat *stat) -{ - return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT; -} - -static inline void blk_stat_set_issue(struct blk_issue_stat *stat, - sector_t size) -{ - stat->stat = (stat->stat & BLK_STAT_RES_MASK) | - (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) | - (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT); -} - /* record time/size info in request but not add a callback */ void blk_stat_enable_accounting(struct request_queue *q); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 35f9b8ff40d734bf2ac89c2316c4b204a70262b9..f63d88c92c3aeba8b22195dca1512fbaeb6c2b69 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -2279,8 +2279,7 @@ void blk_throtl_stat_add(struct request *rq, u64 time_ns) struct request_queue *q = rq->q; struct throtl_data *td = q->td; - throtl_track_latency(td, blk_stat_size(&rq->issue_stat), - req_op(rq), time_ns >> 10); + throtl_track_latency(td, rq->throtl_size, req_op(rq), time_ns >> 10); } void blk_throtl_bio_endio(struct bio *bio) diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 592e914c989070bf222dda6f6eff66d4d5ea7411..4f89b28fa6524d117b5ad6a53a605ebda8e57b93 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -31,22 +31,22 @@ static inline void wbt_clear_state(struct request *rq) { - rq->issue_stat.stat &= ~BLK_STAT_RES_MASK; + rq->wbt_flags = 0; } static inline enum wbt_flags wbt_flags(struct request *rq) { - return (rq->issue_stat.stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT; + return rq->wbt_flags; } static inline bool wbt_is_tracked(struct request *rq) { - return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED; + return rq->wbt_flags & WBT_TRACKED; } static inline bool wbt_is_read(struct request *rq) { - return (rq->issue_stat.stat >> BLK_STAT_RES_SHIFT) & WBT_READ; + return rq->wbt_flags & WBT_READ; } enum { @@ -657,7 +657,7 @@ void wbt_issue(struct rq_wb *rwb, struct request *rq) */ if (wbt_is_read(rq) && !rwb->sync_issue) { rwb->sync_cookie = rq; - rwb->sync_issue = blk_stat_time(&rq->issue_stat); + rwb->sync_issue = rq->io_start_time_ns; } } @@ -746,8 +746,6 @@ int wbt_init(struct request_queue *q) struct rq_wb *rwb; int i; - BUILD_BUG_ON(WBT_NR_BITS > BLK_STAT_RES_BITS); - rwb = kzalloc(sizeof(*rwb), GFP_KERNEL); if (!rwb) return -ENOMEM; diff --git a/block/blk-wbt.h b/block/blk-wbt.h index 85fbcccf12a549b845c20520c35209bbcf8a9a41..300df531d0a667ec3da37d04a2ad3f19da3ea868 100644 --- a/block/blk-wbt.h +++ b/block/blk-wbt.h @@ -63,7 +63,7 @@ struct rq_wb { struct blk_stat_callback *cb; - s64 sync_issue; + u64 sync_issue; void *sync_cookie; unsigned int wc; @@ -90,7 +90,7 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb) static inline void wbt_track(struct request *rq, enum wbt_flags flags) { - rq->issue_stat.stat |= ((u64)flags) << BLK_STAT_RES_SHIFT; + rq->wbt_flags |= flags; } void __wbt_done(struct rq_wb *, enum wbt_flags); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 0d6d25e32e1f44fda0049bb3e213b85a3debfa1f..564967fafe5f07a9887d035f3791b36208e18269 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -485,11 +485,11 @@ static void kyber_completed_request(struct request *rq) if (blk_stat_is_active(kqd->cb)) return; - now = __blk_stat_time(ktime_to_ns(ktime_get())); - if (now < blk_stat_time(&rq->issue_stat)) + now = ktime_get_ns(); + if (now < rq->io_start_time_ns) return; - latency = now - blk_stat_time(&rq->issue_stat); + latency = now - rq->io_start_time_ns; if (latency > target) blk_stat_activate_msecs(kqd->cb, 10); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index b6f1d53cf11395817e6607361bcf632cdc626a7a..4cb970cdcd11e55789fb29e5274432ee98521804 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -91,10 +91,6 @@ static inline bool blk_path_error(blk_status_t error) return true; } -struct blk_issue_stat { - u64 stat; -}; - /* * From most significant bit: * 1 bit: reserved for other usage, see below diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5c4eee04319174a291a6e76378e18c105da87fda..f2c2fc011e6b2b38dfec4bb6c2ccb18bde7d92e5 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -206,8 +206,18 @@ struct request { struct gendisk *rq_disk; struct hd_struct *part; unsigned long start_time; - struct blk_issue_stat issue_stat; - /* Number of scatter-gather DMA addr+len pairs after + /* Time that I/O was submitted to the device. */ + u64 io_start_time_ns; + +#ifdef CONFIG_BLK_WBT + unsigned short wbt_flags; +#endif +#ifdef CONFIG_BLK_DEV_THROTTLING_LOW + unsigned short throtl_size; +#endif + + /* + * Number of scatter-gather DMA addr+len pairs after * physical address coalescing is performed. */ unsigned short nr_phys_segments; @@ -267,8 +277,8 @@ struct request { #ifdef CONFIG_BLK_CGROUP struct request_list *rl; /* rl this rq is alloced from */ - unsigned long long start_time_ns; - unsigned long long io_start_time_ns; /* when passed to hardware */ + unsigned long long cgroup_start_time_ns; + unsigned long long cgroup_io_start_time_ns; /* when passed to hardware */ #endif }; @@ -1797,25 +1807,25 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned lo static inline void set_start_time_ns(struct request *req) { preempt_disable(); - req->start_time_ns = sched_clock(); + req->cgroup_start_time_ns = sched_clock(); preempt_enable(); } static inline void set_io_start_time_ns(struct request *req) { preempt_disable(); - req->io_start_time_ns = sched_clock(); + req->cgroup_io_start_time_ns = sched_clock(); preempt_enable(); } static inline uint64_t rq_start_time_ns(struct request *req) { - return req->start_time_ns; + return req->cgroup_start_time_ns; } static inline uint64_t rq_io_start_time_ns(struct request *req) { - return req->io_start_time_ns; + return req->cgroup_io_start_time_ns; } #else static inline void set_start_time_ns(struct request *req) {}