From 0111cff3be1b260336f25ee027c4b0a639a19124 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 27 Nov 2018 17:02:25 -0700 Subject: [PATCH] blk-mq: add mq_ops->commit_rqs() fix #28871358 commit d666ba98f849ad44c4405ecc2180390ebe80f4f9 upstream blk-mq passes information to the hardware about any given request being the last that we will issue in this sequence. The point is that hardware can defer costly doorbell type writes to the last request. But if we run into errors issuing a sequence of requests, we may never send the request with bd->last == true set. For that case, we need a hook that tells the hardware that nothing else is coming right now. For failures returned by the drivers ->queue_rq() hook, the driver is responsible for flushing pending requests, if it uses bd->last to optimize that part. This works like before, no changes there. Reviewed-by: Omar Sandoval Reviewed-by: Ming Lei Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe Signed-off-by: Baolin Wang Reviewed-by: Joseph Qi --- block/blk-mq.c | 16 ++++++++++++++++ include/linux/blk-mq.h | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/block/blk-mq.c b/block/blk-mq.c index fae3f4458123..71aad5a1af86 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1254,6 +1254,14 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list, if (!list_empty(list)) { bool needs_restart; + /* + * If we didn't flush the entire list, we could have told + * the driver there was more coming, but that turned out to + * be a lie. + */ + if (q->mq_ops->commit_rqs) + q->mq_ops->commit_rqs(hctx); + spin_lock(&hctx->lock); list_splice_init(list, &hctx->dispatch); spin_unlock(&hctx->lock); @@ -1854,6 +1862,14 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, blk_mq_end_request(rq, ret); } } + + /* + * If we didn't flush the entire list, we could have told + * the driver there was more coming, but that turned out to + * be a lie. + */ + if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs) + hctx->queue->mq_ops->commit_rqs(hctx); } static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 4d72b4ebfbc4..70a624c70adb 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -111,6 +111,7 @@ struct blk_mq_queue_data { typedef blk_status_t (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *); +typedef void (commit_rqs_fn)(struct blk_mq_hw_ctx *); typedef bool (get_budget_fn)(struct blk_mq_hw_ctx *); typedef void (put_budget_fn)(struct blk_mq_hw_ctx *); typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool); @@ -135,6 +136,15 @@ struct blk_mq_ops { */ queue_rq_fn *queue_rq; + /* + * If a driver uses bd->last to judge when to submit requests to + * hardware, it must define this function. In case of errors that + * make us stop issuing further requests, this hook serves the + * purpose of kicking the hardware (which the last request otherwise + * would have done). + */ + commit_rqs_fn *commit_rqs; + /* * Reserve budget before queue request, once .queue_rq is * run, it is driver's responsibility to release the -- GitLab