提交 f4829a9b 编写于 作者: C Christoph Hellwig 提交者: Jens Axboe

blk-mq: fix racy updates of rq->errors

blk_mq_complete_request may be a no-op if the request has already
been completed by others means (e.g. a timeout or cancellation), but
currently drivers have to set rq->errors before calling
blk_mq_complete_request, which might leave us with the wrong error value.

Add an error parameter to blk_mq_complete_request so that we can
defer setting rq->errors until we known we won the race to complete the
request.
Signed-off-by: NChristoph Hellwig <hch@lst.de>
Reviewed-by: NSagi Grimberg <sagig@mellanox.com>
Signed-off-by: NJens Axboe <axboe@fb.com>
上级 60de074b
...@@ -393,14 +393,16 @@ void __blk_mq_complete_request(struct request *rq) ...@@ -393,14 +393,16 @@ void __blk_mq_complete_request(struct request *rq)
* Ends all I/O on a request. It does not handle partial completions. * Ends all I/O on a request. It does not handle partial completions.
* The actual completion happens out-of-order, through a IPI handler. * The actual completion happens out-of-order, through a IPI handler.
**/ **/
void blk_mq_complete_request(struct request *rq) void blk_mq_complete_request(struct request *rq, int error)
{ {
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
if (unlikely(blk_should_fake_timeout(q))) if (unlikely(blk_should_fake_timeout(q)))
return; return;
if (!blk_mark_rq_complete(rq)) if (!blk_mark_rq_complete(rq)) {
rq->errors = error;
__blk_mq_complete_request(rq); __blk_mq_complete_request(rq);
}
} }
EXPORT_SYMBOL(blk_mq_complete_request); EXPORT_SYMBOL(blk_mq_complete_request);
...@@ -616,10 +618,8 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, ...@@ -616,10 +618,8 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
* If a request wasn't started before the queue was * If a request wasn't started before the queue was
* marked dying, kill it here or it'll go unnoticed. * marked dying, kill it here or it'll go unnoticed.
*/ */
if (unlikely(blk_queue_dying(rq->q))) { if (unlikely(blk_queue_dying(rq->q)))
rq->errors = -EIO; blk_mq_complete_request(rq, -EIO);
blk_mq_complete_request(rq);
}
return; return;
} }
if (rq->cmd_flags & REQ_NO_TIMEOUT) if (rq->cmd_flags & REQ_NO_TIMEOUT)
......
...@@ -1486,17 +1486,16 @@ static void loop_handle_cmd(struct loop_cmd *cmd) ...@@ -1486,17 +1486,16 @@ static void loop_handle_cmd(struct loop_cmd *cmd)
{ {
const bool write = cmd->rq->cmd_flags & REQ_WRITE; const bool write = cmd->rq->cmd_flags & REQ_WRITE;
struct loop_device *lo = cmd->rq->q->queuedata; struct loop_device *lo = cmd->rq->q->queuedata;
int ret = -EIO; int ret = 0;
if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
ret = -EIO;
goto failed; goto failed;
}
ret = do_req_filebacked(lo, cmd->rq); ret = do_req_filebacked(lo, cmd->rq);
failed: failed:
if (ret) blk_mq_complete_request(cmd->rq, ret ? -EIO : 0);
cmd->rq->errors = -EIO;
blk_mq_complete_request(cmd->rq);
} }
static void loop_queue_write_work(struct work_struct *work) static void loop_queue_write_work(struct work_struct *work)
......
...@@ -289,7 +289,7 @@ static inline void null_handle_cmd(struct nullb_cmd *cmd) ...@@ -289,7 +289,7 @@ static inline void null_handle_cmd(struct nullb_cmd *cmd)
case NULL_IRQ_SOFTIRQ: case NULL_IRQ_SOFTIRQ:
switch (queue_mode) { switch (queue_mode) {
case NULL_Q_MQ: case NULL_Q_MQ:
blk_mq_complete_request(cmd->rq); blk_mq_complete_request(cmd->rq, cmd->rq->errors);
break; break;
case NULL_Q_RQ: case NULL_Q_RQ:
blk_complete_request(cmd->rq); blk_complete_request(cmd->rq);
......
...@@ -618,16 +618,15 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx, ...@@ -618,16 +618,15 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx,
spin_unlock_irqrestore(req->q->queue_lock, flags); spin_unlock_irqrestore(req->q->queue_lock, flags);
return; return;
} }
if (req->cmd_type == REQ_TYPE_DRV_PRIV) { if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
if (cmd_rq->ctx == CMD_CTX_CANCELLED) if (cmd_rq->ctx == CMD_CTX_CANCELLED)
req->errors = -EINTR; status = -EINTR;
else
req->errors = status;
} else { } else {
req->errors = nvme_error_status(status); status = nvme_error_status(status);
} }
} else }
req->errors = 0;
if (req->cmd_type == REQ_TYPE_DRV_PRIV) { if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
u32 result = le32_to_cpup(&cqe->result); u32 result = le32_to_cpup(&cqe->result);
req->special = (void *)(uintptr_t)result; req->special = (void *)(uintptr_t)result;
...@@ -650,7 +649,7 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx, ...@@ -650,7 +649,7 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx,
} }
nvme_free_iod(nvmeq->dev, iod); nvme_free_iod(nvmeq->dev, iod);
blk_mq_complete_request(req); blk_mq_complete_request(req, status);
} }
/* length is in bytes. gfp flags indicates whether we may sleep. */ /* length is in bytes. gfp flags indicates whether we may sleep. */
...@@ -863,8 +862,7 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx, ...@@ -863,8 +862,7 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
if (ns && ns->ms && !blk_integrity_rq(req)) { if (ns && ns->ms && !blk_integrity_rq(req)) {
if (!(ns->pi_type && ns->ms == 8) && if (!(ns->pi_type && ns->ms == 8) &&
req->cmd_type != REQ_TYPE_DRV_PRIV) { req->cmd_type != REQ_TYPE_DRV_PRIV) {
req->errors = -EFAULT; blk_mq_complete_request(req, -EFAULT);
blk_mq_complete_request(req);
return BLK_MQ_RQ_QUEUE_OK; return BLK_MQ_RQ_QUEUE_OK;
} }
} }
......
...@@ -144,7 +144,7 @@ static void virtblk_done(struct virtqueue *vq) ...@@ -144,7 +144,7 @@ static void virtblk_done(struct virtqueue *vq)
do { do {
virtqueue_disable_cb(vq); virtqueue_disable_cb(vq);
while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) { while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
blk_mq_complete_request(vbr->req); blk_mq_complete_request(vbr->req, vbr->req->errors);
req_done = true; req_done = true;
} }
if (unlikely(virtqueue_is_broken(vq))) if (unlikely(virtqueue_is_broken(vq)))
......
...@@ -1142,6 +1142,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) ...@@ -1142,6 +1142,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
RING_IDX i, rp; RING_IDX i, rp;
unsigned long flags; unsigned long flags;
struct blkfront_info *info = (struct blkfront_info *)dev_id; struct blkfront_info *info = (struct blkfront_info *)dev_id;
int error;
spin_lock_irqsave(&info->io_lock, flags); spin_lock_irqsave(&info->io_lock, flags);
...@@ -1182,37 +1183,37 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) ...@@ -1182,37 +1183,37 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
continue; continue;
} }
req->errors = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO; error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
switch (bret->operation) { switch (bret->operation) {
case BLKIF_OP_DISCARD: case BLKIF_OP_DISCARD:
if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
struct request_queue *rq = info->rq; struct request_queue *rq = info->rq;
printk(KERN_WARNING "blkfront: %s: %s op failed\n", printk(KERN_WARNING "blkfront: %s: %s op failed\n",
info->gd->disk_name, op_name(bret->operation)); info->gd->disk_name, op_name(bret->operation));
req->errors = -EOPNOTSUPP; error = -EOPNOTSUPP;
info->feature_discard = 0; info->feature_discard = 0;
info->feature_secdiscard = 0; info->feature_secdiscard = 0;
queue_flag_clear(QUEUE_FLAG_DISCARD, rq); queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq); queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq);
} }
blk_mq_complete_request(req); blk_mq_complete_request(req, error);
break; break;
case BLKIF_OP_FLUSH_DISKCACHE: case BLKIF_OP_FLUSH_DISKCACHE:
case BLKIF_OP_WRITE_BARRIER: case BLKIF_OP_WRITE_BARRIER:
if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
printk(KERN_WARNING "blkfront: %s: %s op failed\n", printk(KERN_WARNING "blkfront: %s: %s op failed\n",
info->gd->disk_name, op_name(bret->operation)); info->gd->disk_name, op_name(bret->operation));
req->errors = -EOPNOTSUPP; error = -EOPNOTSUPP;
} }
if (unlikely(bret->status == BLKIF_RSP_ERROR && if (unlikely(bret->status == BLKIF_RSP_ERROR &&
info->shadow[id].req.u.rw.nr_segments == 0)) { info->shadow[id].req.u.rw.nr_segments == 0)) {
printk(KERN_WARNING "blkfront: %s: empty %s op failed\n", printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
info->gd->disk_name, op_name(bret->operation)); info->gd->disk_name, op_name(bret->operation));
req->errors = -EOPNOTSUPP; error = -EOPNOTSUPP;
} }
if (unlikely(req->errors)) { if (unlikely(error)) {
if (req->errors == -EOPNOTSUPP) if (error == -EOPNOTSUPP)
req->errors = 0; error = 0;
info->feature_flush = 0; info->feature_flush = 0;
xlvbd_flush(info); xlvbd_flush(info);
} }
...@@ -1223,7 +1224,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) ...@@ -1223,7 +1224,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
dev_dbg(&info->xbdev->dev, "Bad return from blkdev data " dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
"request: %x\n", bret->status); "request: %x\n", bret->status);
blk_mq_complete_request(req); blk_mq_complete_request(req, error);
break; break;
default: default:
BUG(); BUG();
......
...@@ -1957,7 +1957,7 @@ static int scsi_mq_prep_fn(struct request *req) ...@@ -1957,7 +1957,7 @@ static int scsi_mq_prep_fn(struct request *req)
static void scsi_mq_done(struct scsi_cmnd *cmd) static void scsi_mq_done(struct scsi_cmnd *cmd)
{ {
trace_scsi_dispatch_cmd_done(cmd); trace_scsi_dispatch_cmd_done(cmd);
blk_mq_complete_request(cmd->request); blk_mq_complete_request(cmd->request, cmd->request->errors);
} }
static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx, static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
......
...@@ -214,7 +214,7 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); ...@@ -214,7 +214,7 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head);
void blk_mq_cancel_requeue_work(struct request_queue *q); void blk_mq_cancel_requeue_work(struct request_queue *q);
void blk_mq_kick_requeue_list(struct request_queue *q); void blk_mq_kick_requeue_list(struct request_queue *q);
void blk_mq_abort_requeue_list(struct request_queue *q); void blk_mq_abort_requeue_list(struct request_queue *q);
void blk_mq_complete_request(struct request *rq); void blk_mq_complete_request(struct request *rq, int error);
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册