blk-core.c 68.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6 7
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *	-  July2000
L
Linus Torvalds 已提交
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
28
#include <linux/task_io_accounting_ops.h>
29
#include <linux/fault-inject.h>
30 31 32

#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
L
Linus Torvalds 已提交
33

34 35
#include "blk.h"

36
EXPORT_TRACEPOINT_SYMBOL_GPL(block_remap);
37
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
38
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
39

40
static int __make_request(struct request_queue *q, struct bio *bio);
L
Linus Torvalds 已提交
41 42 43 44

/*
 * For the allocated request tables
 */
45
static struct kmem_cache *request_cachep;
L
Linus Torvalds 已提交
46 47 48 49

/*
 * For queue allocation
 */
50
struct kmem_cache *blk_requestq_cachep;
L
Linus Torvalds 已提交
51 52 53 54

/*
 * Controlling structure to kblockd
 */
55
static struct workqueue_struct *kblockd_workqueue;
L
Linus Torvalds 已提交
56

57 58
static void drive_stat_acct(struct request *rq, int new_io)
{
59
	struct hd_struct *part;
60
	int rw = rq_data_dir(rq);
T
Tejun Heo 已提交
61
	int cpu;
62

63
	if (!blk_do_io_stat(rq))
64 65
		return;

T
Tejun Heo 已提交
66
	cpu = part_stat_lock();
67
	part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
T
Tejun Heo 已提交
68

69
	if (!new_io)
T
Tejun Heo 已提交
70
		part_stat_inc(cpu, part, merges[rw]);
71
	else {
T
Tejun Heo 已提交
72
		part_round_stats(cpu, part);
73
		part_inc_in_flight(part, rw);
74
	}
75

T
Tejun Heo 已提交
76
	part_stat_unlock();
77 78
}

79
void blk_queue_congestion_threshold(struct request_queue *q)
L
Linus Torvalds 已提交
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

/**
 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
 * @bdev:	device
 *
 * Locates the passed device's request queue and returns the address of its
 * backing_dev_info
 *
 * Will return NULL if the request queue cannot be located.
 */
struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
{
	struct backing_dev_info *ret = NULL;
106
	struct request_queue *q = bdev_get_queue(bdev);
L
Linus Torvalds 已提交
107 108 109 110 111 112 113

	if (q)
		ret = &q->backing_dev_info;
	return ret;
}
EXPORT_SYMBOL(blk_get_backing_dev_info);

114
void blk_rq_init(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
115
{
116 117
	memset(rq, 0, sizeof(*rq));

L
Linus Torvalds 已提交
118
	INIT_LIST_HEAD(&rq->queuelist);
J
Jens Axboe 已提交
119
	INIT_LIST_HEAD(&rq->timeout_list);
120
	rq->cpu = -1;
J
Jens Axboe 已提交
121
	rq->q = q;
122
	rq->__sector = (sector_t) -1;
123 124
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
125
	rq->cmd = rq->__cmd;
126
	rq->cmd_len = BLK_MAX_CDB;
J
Jens Axboe 已提交
127
	rq->tag = -1;
L
Linus Torvalds 已提交
128
	rq->ref_count = 1;
129
	rq->start_time = jiffies;
130
	set_start_time_ns(rq);
L
Linus Torvalds 已提交
131
}
132
EXPORT_SYMBOL(blk_rq_init);
L
Linus Torvalds 已提交
133

N
NeilBrown 已提交
134 135
static void req_bio_endio(struct request *rq, struct bio *bio,
			  unsigned int nbytes, int error)
L
Linus Torvalds 已提交
136
{
137
	struct request_queue *q = rq->q;
138

N
NeilBrown 已提交
139 140 141 142 143
	if (&q->bar_rq != rq) {
		if (error)
			clear_bit(BIO_UPTODATE, &bio->bi_flags);
		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
			error = -EIO;
144

N
NeilBrown 已提交
145
		if (unlikely(nbytes > bio->bi_size)) {
146
			printk(KERN_ERR "%s: want %u bytes done, %u left\n",
147
			       __func__, nbytes, bio->bi_size);
N
NeilBrown 已提交
148 149
			nbytes = bio->bi_size;
		}
150

151 152 153
		if (unlikely(rq->cmd_flags & REQ_QUIET))
			set_bit(BIO_QUIET, &bio->bi_flags);

N
NeilBrown 已提交
154 155
		bio->bi_size -= nbytes;
		bio->bi_sector += (nbytes >> 9);
156 157 158 159

		if (bio_integrity(bio))
			bio_integrity_advance(bio, nbytes);

N
NeilBrown 已提交
160
		if (bio->bi_size == 0)
161
			bio_endio(bio, error);
N
NeilBrown 已提交
162 163 164 165 166 167 168 169 170
	} else {

		/*
		 * Okay, this is the barrier request in progress, just
		 * record the error;
		 */
		if (error && !q->orderr)
			q->orderr = error;
	}
L
Linus Torvalds 已提交
171 172 173 174 175 176
}

void blk_dump_rq_flags(struct request *rq, char *msg)
{
	int bit;

177
	printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
178 179
		rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
		rq->cmd_flags);
L
Linus Torvalds 已提交
180

181 182 183
	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
	       (unsigned long long)blk_rq_pos(rq),
	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
T
Tejun Heo 已提交
184
	printk(KERN_INFO "  bio %p, biotail %p, buffer %p, len %u\n",
185
	       rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
L
Linus Torvalds 已提交
186

187
	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
188
		printk(KERN_INFO "  cdb: ");
189
		for (bit = 0; bit < BLK_MAX_CDB; bit++)
L
Linus Torvalds 已提交
190 191 192 193 194 195 196 197 198 199 200 201 202 203
			printk("%02x ", rq->cmd[bit]);
		printk("\n");
	}
}
EXPORT_SYMBOL(blk_dump_rq_flags);

/*
 * "plug" the device if there are no outstanding requests: this will
 * force the transfer to start only after we have put all the requests
 * on the list.
 *
 * This is called with interrupts off and no requests on the queue and
 * with the queue lock held.
 */
204
void blk_plug_device(struct request_queue *q)
L
Linus Torvalds 已提交
205 206 207 208 209 210 211
{
	WARN_ON(!irqs_disabled());

	/*
	 * don't plug a stopped queue, it must be paired with blk_start_queue()
	 * which will restart the queueing
	 */
212
	if (blk_queue_stopped(q))
L
Linus Torvalds 已提交
213 214
		return;

J
Jens Axboe 已提交
215
	if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) {
L
Linus Torvalds 已提交
216
		mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
217
		trace_block_plug(q);
218
	}
L
Linus Torvalds 已提交
219 220 221
}
EXPORT_SYMBOL(blk_plug_device);

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
/**
 * blk_plug_device_unlocked - plug a device without queue lock held
 * @q:    The &struct request_queue to plug
 *
 * Description:
 *   Like @blk_plug_device(), but grabs the queue lock and disables
 *   interrupts.
 **/
void blk_plug_device_unlocked(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	blk_plug_device(q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_plug_device_unlocked);

L
Linus Torvalds 已提交
240 241 242 243
/*
 * remove the queue from the plugged list, if present. called with
 * queue lock held and interrupts disabled.
 */
244
int blk_remove_plug(struct request_queue *q)
L
Linus Torvalds 已提交
245 246 247
{
	WARN_ON(!irqs_disabled());

J
Jens Axboe 已提交
248
	if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q))
L
Linus Torvalds 已提交
249 250 251 252 253 254 255 256 257 258
		return 0;

	del_timer(&q->unplug_timer);
	return 1;
}
EXPORT_SYMBOL(blk_remove_plug);

/*
 * remove the plug and let it rip..
 */
259
void __generic_unplug_device(struct request_queue *q)
L
Linus Torvalds 已提交
260
{
261
	if (unlikely(blk_queue_stopped(q)))
L
Linus Torvalds 已提交
262
		return;
263
	if (!blk_remove_plug(q) && !blk_queue_nonrot(q))
L
Linus Torvalds 已提交
264 265
		return;

266
	q->request_fn(q);
L
Linus Torvalds 已提交
267 268 269 270
}

/**
 * generic_unplug_device - fire a request queue
271
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
272 273 274 275 276 277 278 279
 *
 * Description:
 *   Linux uses plugging to build bigger requests queues before letting
 *   the device have at them. If a queue is plugged, the I/O scheduler
 *   is still adding and merging requests on the queue. Once the queue
 *   gets unplugged, the request_fn defined for the queue is invoked and
 *   transfers started.
 **/
280
void generic_unplug_device(struct request_queue *q)
L
Linus Torvalds 已提交
281
{
282 283 284 285 286
	if (blk_queue_plugged(q)) {
		spin_lock_irq(q->queue_lock);
		__generic_unplug_device(q);
		spin_unlock_irq(q->queue_lock);
	}
L
Linus Torvalds 已提交
287 288 289 290 291 292
}
EXPORT_SYMBOL(generic_unplug_device);

static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
				   struct page *page)
{
293
	struct request_queue *q = bdi->unplug_io_data;
L
Linus Torvalds 已提交
294

295
	blk_unplug(q);
L
Linus Torvalds 已提交
296 297
}

J
Jens Axboe 已提交
298
void blk_unplug_work(struct work_struct *work)
L
Linus Torvalds 已提交
299
{
300 301
	struct request_queue *q =
		container_of(work, struct request_queue, unplug_work);
L
Linus Torvalds 已提交
302

303
	trace_block_unplug_io(q);
L
Linus Torvalds 已提交
304 305 306
	q->unplug_fn(q);
}

J
Jens Axboe 已提交
307
void blk_unplug_timeout(unsigned long data)
L
Linus Torvalds 已提交
308
{
309
	struct request_queue *q = (struct request_queue *)data;
L
Linus Torvalds 已提交
310

311
	trace_block_unplug_timer(q);
312
	kblockd_schedule_work(q, &q->unplug_work);
L
Linus Torvalds 已提交
313 314
}

315 316 317 318 319 320
void blk_unplug(struct request_queue *q)
{
	/*
	 * devices don't necessarily have an ->unplug_fn defined
	 */
	if (q->unplug_fn) {
321
		trace_block_unplug_io(q);
322 323 324 325 326
		q->unplug_fn(q);
	}
}
EXPORT_SYMBOL(blk_unplug);

L
Linus Torvalds 已提交
327 328
/**
 * blk_start_queue - restart a previously stopped queue
329
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
330 331 332 333 334 335
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
 *   entered. Also see blk_stop_queue(). Queue lock must be held.
 **/
336
void blk_start_queue(struct request_queue *q)
L
Linus Torvalds 已提交
337
{
338 339
	WARN_ON(!irqs_disabled());

N
Nick Piggin 已提交
340
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
341
	__blk_run_queue(q);
L
Linus Torvalds 已提交
342 343 344 345 346
}
EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
347
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
348 349 350 351 352 353 354 355 356 357 358
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
 *   blk_start_queue() to restart queue operations. Queue lock must be held.
 **/
359
void blk_stop_queue(struct request_queue *q)
L
Linus Torvalds 已提交
360 361
{
	blk_remove_plug(q);
N
Nick Piggin 已提交
362
	queue_flag_set(QUEUE_FLAG_STOPPED, q);
L
Linus Torvalds 已提交
363 364 365 366 367 368 369 370 371 372 373 374
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
375
 *     that the callbacks might use. The caller must already have made sure
L
Linus Torvalds 已提交
376 377 378 379 380 381 382
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
 */
void blk_sync_queue(struct request_queue *q)
{
	del_timer_sync(&q->unplug_timer);
383
	del_timer_sync(&q->timeout);
384
	cancel_work_sync(&q->unplug_work);
L
Linus Torvalds 已提交
385 386 387 388
}
EXPORT_SYMBOL(blk_sync_queue);

/**
389
 * __blk_run_queue - run a single device queue
L
Linus Torvalds 已提交
390
 * @q:	The queue to run
391 392 393 394 395
 *
 * Description:
 *    See @blk_run_queue. This variant must be called with the queue lock
 *    held and interrupts disabled.
 *
L
Linus Torvalds 已提交
396
 */
N
Nick Piggin 已提交
397
void __blk_run_queue(struct request_queue *q)
L
Linus Torvalds 已提交
398 399
{
	blk_remove_plug(q);
J
Jens Axboe 已提交
400

401 402 403 404 405 406
	if (unlikely(blk_queue_stopped(q)))
		return;

	if (elv_queue_empty(q))
		return;

J
Jens Axboe 已提交
407 408 409 410
	/*
	 * Only recurse once to avoid overrunning the stack, let the unplug
	 * handling reinvoke the handler shortly if we already got there.
	 */
411 412 413 414 415 416 417
	if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
		q->request_fn(q);
		queue_flag_clear(QUEUE_FLAG_REENTER, q);
	} else {
		queue_flag_set(QUEUE_FLAG_PLUGGED, q);
		kblockd_schedule_work(q, &q->unplug_work);
	}
N
Nick Piggin 已提交
418 419
}
EXPORT_SYMBOL(__blk_run_queue);
J
Jens Axboe 已提交
420

N
Nick Piggin 已提交
421 422 423
/**
 * blk_run_queue - run a single device queue
 * @q: The queue to run
424 425 426
 *
 * Description:
 *    Invoke request handling on this queue, if it has pending work to do.
T
Tejun Heo 已提交
427
 *    May be used to restart queueing when a request has completed.
N
Nick Piggin 已提交
428 429 430 431 432 433 434
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	__blk_run_queue(q);
L
Linus Torvalds 已提交
435 436 437 438
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

439
void blk_put_queue(struct request_queue *q)
440 441 442 443
{
	kobject_put(&q->kobj);
}

444
void blk_cleanup_queue(struct request_queue *q)
445
{
446 447 448 449 450 451 452 453
	/*
	 * We know we have process context here, so we can be a little
	 * cautious and ensure that pending block actions on this device
	 * are done before moving on. Going into this function, we should
	 * not have processes doing IO to this device.
	 */
	blk_sync_queue(q);

454
	del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
455
	mutex_lock(&q->sysfs_lock);
N
Nick Piggin 已提交
456
	queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
457 458 459 460 461 462 463
	mutex_unlock(&q->sysfs_lock);

	if (q->elevator)
		elevator_exit(q->elevator);

	blk_put_queue(q);
}
L
Linus Torvalds 已提交
464 465
EXPORT_SYMBOL(blk_cleanup_queue);

466
static int blk_init_free_list(struct request_queue *q)
L
Linus Torvalds 已提交
467 468 469
{
	struct request_list *rl = &q->rq;

470 471 472
	if (unlikely(rl->rq_pool))
		return 0;

473 474
	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
T
Tejun Heo 已提交
475
	rl->elvpriv = 0;
476 477
	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
L
Linus Torvalds 已提交
478

479 480
	rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
				mempool_free_slab, request_cachep, q->node);
L
Linus Torvalds 已提交
481 482 483 484 485 486 487

	if (!rl->rq_pool)
		return -ENOMEM;

	return 0;
}

488
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
L
Linus Torvalds 已提交
489
{
490 491 492
	return blk_alloc_queue_node(gfp_mask, -1);
}
EXPORT_SYMBOL(blk_alloc_queue);
L
Linus Torvalds 已提交
493

494
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
495
{
496
	struct request_queue *q;
P
Peter Zijlstra 已提交
497
	int err;
498

499
	q = kmem_cache_alloc_node(blk_requestq_cachep,
500
				gfp_mask | __GFP_ZERO, node_id);
L
Linus Torvalds 已提交
501 502 503
	if (!q)
		return NULL;

P
Peter Zijlstra 已提交
504 505
	q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
	q->backing_dev_info.unplug_io_data = q;
506 507 508 509
	q->backing_dev_info.ra_pages =
			(VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
	q->backing_dev_info.state = 0;
	q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
510
	q->backing_dev_info.name = "block";
511

P
Peter Zijlstra 已提交
512 513
	err = bdi_init(&q->backing_dev_info);
	if (err) {
514
		kmem_cache_free(blk_requestq_cachep, q);
P
Peter Zijlstra 已提交
515 516 517
		return NULL;
	}

518 519
	setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
		    laptop_mode_timer_fn, (unsigned long) q);
L
Linus Torvalds 已提交
520
	init_timer(&q->unplug_timer);
J
Jens Axboe 已提交
521 522
	setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
	INIT_LIST_HEAD(&q->timeout_list);
523
	INIT_WORK(&q->unplug_work, blk_unplug_work);
524

525
	kobject_init(&q->kobj, &blk_queue_ktype);
L
Linus Torvalds 已提交
526

527
	mutex_init(&q->sysfs_lock);
528
	spin_lock_init(&q->__queue_lock);
529

L
Linus Torvalds 已提交
530 531
	return q;
}
532
EXPORT_SYMBOL(blk_alloc_queue_node);
L
Linus Torvalds 已提交
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
556 557
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
L
Linus Torvalds 已提交
558
 *
559
 *    Function returns a pointer to the initialized request queue, or %NULL if
L
Linus Torvalds 已提交
560 561 562 563 564 565
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
566

567
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
L
Linus Torvalds 已提交
568
{
569 570 571 572
	return blk_init_queue_node(rfn, lock, -1);
}
EXPORT_SYMBOL(blk_init_queue);

573
struct request_queue *
574 575
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
576
	struct request_queue *uninit_q, *q;
L
Linus Torvalds 已提交
577

578 579 580 581 582 583 584 585 586
	uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
	if (!uninit_q)
		return NULL;

	q = blk_init_allocated_queue_node(uninit_q, rfn, lock, node_id);
	if (!q)
		blk_cleanup_queue(uninit_q);

	return q;
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601
}
EXPORT_SYMBOL(blk_init_queue_node);

struct request_queue *
blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
			 spinlock_t *lock)
{
	return blk_init_allocated_queue_node(q, rfn, lock, -1);
}
EXPORT_SYMBOL(blk_init_allocated_queue);

struct request_queue *
blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn,
			      spinlock_t *lock, int node_id)
{
L
Linus Torvalds 已提交
602 603 604
	if (!q)
		return NULL;

605
	q->node = node_id;
606
	if (blk_init_free_list(q))
607
		return NULL;
L
Linus Torvalds 已提交
608 609 610

	q->request_fn		= rfn;
	q->prep_rq_fn		= NULL;
611
	q->unprep_rq_fn		= NULL;
L
Linus Torvalds 已提交
612
	q->unplug_fn		= generic_unplug_device;
613
	q->queue_flags		= QUEUE_FLAG_DEFAULT;
L
Linus Torvalds 已提交
614 615
	q->queue_lock		= lock;

616 617 618
	/*
	 * This also sets hw/phys segments, boundary and size
	 */
L
Linus Torvalds 已提交
619 620
	blk_queue_make_request(q, __make_request);

621 622
	q->sg_reserved_size = INT_MAX;

L
Linus Torvalds 已提交
623 624 625 626 627 628 629 630 631 632
	/*
	 * all done
	 */
	if (!elevator_init(q, NULL)) {
		blk_queue_congestion_threshold(q);
		return q;
	}

	return NULL;
}
633
EXPORT_SYMBOL(blk_init_allocated_queue_node);
L
Linus Torvalds 已提交
634

635
int blk_get_queue(struct request_queue *q)
L
Linus Torvalds 已提交
636
{
N
Nick Piggin 已提交
637
	if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
638
		kobject_get(&q->kobj);
L
Linus Torvalds 已提交
639 640 641 642 643 644
		return 0;
	}

	return 1;
}

645
static inline void blk_free_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
646
{
647
	if (rq->cmd_flags & REQ_ELVPRIV)
T
Tejun Heo 已提交
648
		elv_put_request(q, rq);
L
Linus Torvalds 已提交
649 650 651
	mempool_free(rq, q->rq.rq_pool);
}

J
Jens Axboe 已提交
652
static struct request *
653
blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask)
L
Linus Torvalds 已提交
654 655 656 657 658 659
{
	struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);

	if (!rq)
		return NULL;

660
	blk_rq_init(q, rq);
661

662
	rq->cmd_flags = flags | REQ_ALLOCED;
L
Linus Torvalds 已提交
663

T
Tejun Heo 已提交
664
	if (priv) {
665
		if (unlikely(elv_set_request(q, rq, gfp_mask))) {
T
Tejun Heo 已提交
666 667 668
			mempool_free(rq, q->rq.rq_pool);
			return NULL;
		}
669
		rq->cmd_flags |= REQ_ELVPRIV;
T
Tejun Heo 已提交
670
	}
L
Linus Torvalds 已提交
671

T
Tejun Heo 已提交
672
	return rq;
L
Linus Torvalds 已提交
673 674 675 676 677 678
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
679
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
700
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
701 702 703 704 705 706 707 708
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

709
static void __freed_request(struct request_queue *q, int sync)
L
Linus Torvalds 已提交
710 711 712
{
	struct request_list *rl = &q->rq;

713 714
	if (rl->count[sync] < queue_congestion_off_threshold(q))
		blk_clear_queue_congested(q, sync);
L
Linus Torvalds 已提交
715

716 717 718
	if (rl->count[sync] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[sync]))
			wake_up(&rl->wait[sync]);
L
Linus Torvalds 已提交
719

720
		blk_clear_queue_full(q, sync);
L
Linus Torvalds 已提交
721 722 723 724 725 726 727
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
728
static void freed_request(struct request_queue *q, int sync, int priv)
L
Linus Torvalds 已提交
729 730 731
{
	struct request_list *rl = &q->rq;

732
	rl->count[sync]--;
T
Tejun Heo 已提交
733 734
	if (priv)
		rl->elvpriv--;
L
Linus Torvalds 已提交
735

736
	__freed_request(q, sync);
L
Linus Torvalds 已提交
737

738 739
	if (unlikely(rl->starved[sync ^ 1]))
		__freed_request(q, sync ^ 1);
L
Linus Torvalds 已提交
740 741 742
}

/*
N
Nick Piggin 已提交
743 744 745
 * Get a free request, queue_lock must be held.
 * Returns NULL on failure, with queue_lock held.
 * Returns !NULL on success, with queue_lock *not held*.
L
Linus Torvalds 已提交
746
 */
747
static struct request *get_request(struct request_queue *q, int rw_flags,
748
				   struct bio *bio, gfp_t gfp_mask)
L
Linus Torvalds 已提交
749 750 751
{
	struct request *rq = NULL;
	struct request_list *rl = &q->rq;
752
	struct io_context *ioc = NULL;
753
	const bool is_sync = rw_is_sync(rw_flags) != 0;
754 755
	int may_queue, priv;

756
	may_queue = elv_may_queue(q, rw_flags);
757 758 759
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

760 761
	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[is_sync]+1 >= q->nr_requests) {
762
			ioc = current_io_context(GFP_ATOMIC, q->node);
763 764 765 766 767 768
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
769
			if (!blk_queue_full(q, is_sync)) {
770
				ioc_set_batching(q, ioc);
771
				blk_set_queue_full(q, is_sync);
772 773 774 775 776 777 778 779 780 781 782
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
					goto out;
				}
			}
L
Linus Torvalds 已提交
783
		}
784
		blk_set_queue_congested(q, is_sync);
L
Linus Torvalds 已提交
785 786
	}

787 788 789 790 791
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
792
	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
793
		goto out;
H
Hugh Dickins 已提交
794

795 796
	rl->count[is_sync]++;
	rl->starved[is_sync] = 0;
T
Tejun Heo 已提交
797

J
Jens Axboe 已提交
798
	priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
T
Tejun Heo 已提交
799 800 801
	if (priv)
		rl->elvpriv++;

802 803
	if (blk_queue_io_stat(q))
		rw_flags |= REQ_IO_STAT;
L
Linus Torvalds 已提交
804 805
	spin_unlock_irq(q->queue_lock);

806
	rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
807
	if (unlikely(!rq)) {
L
Linus Torvalds 已提交
808 809 810 811 812 813 814 815
		/*
		 * Allocation failed presumably due to memory. Undo anything
		 * we might have messed up.
		 *
		 * Allocating task should really be put onto the front of the
		 * wait queue, but this is pretty rare.
		 */
		spin_lock_irq(q->queue_lock);
816
		freed_request(q, is_sync, priv);
L
Linus Torvalds 已提交
817 818 819 820 821 822 823 824 825

		/*
		 * in the very unlikely event that allocation failed and no
		 * requests for this direction was pending, mark us starved
		 * so that freeing of a request in the other direction will
		 * notice us. another possible fix would be to split the
		 * rq mempool into READ and WRITE
		 */
rq_starved:
826 827
		if (unlikely(rl->count[is_sync] == 0))
			rl->starved[is_sync] = 1;
L
Linus Torvalds 已提交
828 829 830 831

		goto out;
	}

832 833 834 835 836 837
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
L
Linus Torvalds 已提交
838 839
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
840

841
	trace_block_getrq(q, bio, rw_flags & 1);
L
Linus Torvalds 已提交
842 843 844 845 846 847 848
out:
	return rq;
}

/*
 * No available requests for this queue, unplug the device and wait for some
 * requests to become available.
N
Nick Piggin 已提交
849 850
 *
 * Called with q->queue_lock held, and returns with it unlocked.
L
Linus Torvalds 已提交
851
 */
852
static struct request *get_request_wait(struct request_queue *q, int rw_flags,
853
					struct bio *bio)
L
Linus Torvalds 已提交
854
{
855
	const bool is_sync = rw_is_sync(rw_flags) != 0;
L
Linus Torvalds 已提交
856 857
	struct request *rq;

858
	rq = get_request(q, rw_flags, bio, GFP_NOIO);
859 860
	while (!rq) {
		DEFINE_WAIT(wait);
861
		struct io_context *ioc;
L
Linus Torvalds 已提交
862 863
		struct request_list *rl = &q->rq;

864
		prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
L
Linus Torvalds 已提交
865 866
				TASK_UNINTERRUPTIBLE);

867
		trace_block_sleeprq(q, bio, rw_flags & 1);
L
Linus Torvalds 已提交
868

869 870 871
		__generic_unplug_device(q);
		spin_unlock_irq(q->queue_lock);
		io_schedule();
L
Linus Torvalds 已提交
872

873 874 875 876 877 878 879 880
		/*
		 * After sleeping, we become a "batching" process and
		 * will be able to allocate at least one request, and
		 * up to a big batch of them for a small period time.
		 * See ioc_batching, ioc_set_batching
		 */
		ioc = current_io_context(GFP_NOIO, q->node);
		ioc_set_batching(q, ioc);
N
Nick Piggin 已提交
881

882
		spin_lock_irq(q->queue_lock);
883
		finish_wait(&rl->wait[is_sync], &wait);
884 885 886

		rq = get_request(q, rw_flags, bio, GFP_NOIO);
	};
L
Linus Torvalds 已提交
887 888 889 890

	return rq;
}

891
struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
L
Linus Torvalds 已提交
892 893 894 895 896
{
	struct request *rq;

	BUG_ON(rw != READ && rw != WRITE);

N
Nick Piggin 已提交
897 898
	spin_lock_irq(q->queue_lock);
	if (gfp_mask & __GFP_WAIT) {
899
		rq = get_request_wait(q, rw, NULL);
N
Nick Piggin 已提交
900
	} else {
901
		rq = get_request(q, rw, NULL, gfp_mask);
N
Nick Piggin 已提交
902 903 904 905
		if (!rq)
			spin_unlock_irq(q->queue_lock);
	}
	/* q->queue_lock is unlocked at this point */
L
Linus Torvalds 已提交
906 907 908 909 910

	return rq;
}
EXPORT_SYMBOL(blk_get_request);

911
/**
912
 * blk_make_request - given a bio, allocate a corresponding struct request.
913
 * @q: target request queue
914 915
 * @bio:  The bio describing the memory mappings that will be submitted for IO.
 *        It may be a chained-bio properly constructed by block/bio layer.
916
 * @gfp_mask: gfp flags to be used for memory allocation
917
 *
918 919 920 921
 * blk_make_request is the parallel of generic_make_request for BLOCK_PC
 * type commands. Where the struct request needs to be farther initialized by
 * the caller. It is passed a &struct bio, which describes the memory info of
 * the I/O transfer.
922
 *
923 924 925 926 927 928 929 930 931
 * The caller of blk_make_request must make sure that bi_io_vec
 * are set to describe the memory buffers. That bio_data_dir() will return
 * the needed direction of the request. (And all bio's in the passed bio-chain
 * are properly set accordingly)
 *
 * If called under none-sleepable conditions, mapped bio buffers must not
 * need bouncing, by calling the appropriate masked or flagged allocator,
 * suitable for the target device. Otherwise the call to blk_queue_bounce will
 * BUG.
932 933 934 935 936 937 938 939 940
 *
 * WARNING: When allocating/cloning a bio-chain, careful consideration should be
 * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
 * anything but the first bio in the chain. Otherwise you risk waiting for IO
 * completion of a bio that hasn't been submitted yet, thus resulting in a
 * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
 * of bio_alloc(), as that avoids the mempool deadlock.
 * If possible a big IO should be split into smaller parts when allocation
 * fails. Partial allocation should not be an error, or you risk a live-lock.
941
 */
942 943
struct request *blk_make_request(struct request_queue *q, struct bio *bio,
				 gfp_t gfp_mask)
944
{
945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
	struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);

	if (unlikely(!rq))
		return ERR_PTR(-ENOMEM);

	for_each_bio(bio) {
		struct bio *bounce_bio = bio;
		int ret;

		blk_queue_bounce(q, &bounce_bio);
		ret = blk_rq_append_bio(q, rq, bounce_bio);
		if (unlikely(ret)) {
			blk_put_request(rq);
			return ERR_PTR(ret);
		}
	}

	return rq;
963
}
964
EXPORT_SYMBOL(blk_make_request);
965

L
Linus Torvalds 已提交
966 967 968 969 970 971 972 973 974 975
/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
976
void blk_requeue_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
977
{
J
Jens Axboe 已提交
978 979
	blk_delete_timer(rq);
	blk_clear_rq_complete(rq);
980
	trace_block_rq_requeue(q, rq);
981

L
Linus Torvalds 已提交
982 983 984
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);

985 986
	BUG_ON(blk_queued_rq(rq));

L
Linus Torvalds 已提交
987 988 989 990 991
	elv_requeue_request(q, rq);
}
EXPORT_SYMBOL(blk_requeue_request);

/**
992
 * blk_insert_request - insert a special request into a request queue
L
Linus Torvalds 已提交
993 994 995 996 997 998 999 1000 1001
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 * @at_head:	insert request at head or tail of queue
 * @data:	private data
 *
 * Description:
 *    Many block devices need to execute commands asynchronously, so they don't
 *    block the whole kernel from preemption during request execution.  This is
 *    accomplished normally by inserting aritficial requests tagged as
1002 1003
 *    REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
 *    be scheduled for actual execution by the request queue.
L
Linus Torvalds 已提交
1004 1005 1006 1007 1008 1009
 *
 *    We have the option of inserting the head or the tail of the queue.
 *    Typically we use the tail for new ioctls and so forth.  We use the head
 *    of the queue for things like a QUEUE_FULL message from a device, or a
 *    host that is unable to accept a particular command.
 */
1010
void blk_insert_request(struct request_queue *q, struct request *rq,
1011
			int at_head, void *data)
L
Linus Torvalds 已提交
1012
{
1013
	int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
L
Linus Torvalds 已提交
1014 1015 1016 1017 1018 1019 1020
	unsigned long flags;

	/*
	 * tell I/O scheduler that this isn't a regular read/write (ie it
	 * must not attempt merges on this) and that it acts as a soft
	 * barrier
	 */
1021
	rq->cmd_type = REQ_TYPE_SPECIAL;
L
Linus Torvalds 已提交
1022 1023 1024 1025 1026 1027 1028 1029

	rq->special = data;

	spin_lock_irqsave(q->queue_lock, flags);

	/*
	 * If command is tagged, release the tag
	 */
1030 1031
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);
L
Linus Torvalds 已提交
1032

1033
	drive_stat_acct(rq, 1);
1034
	__elv_add_request(q, rq, where, 0);
T
Tejun Heo 已提交
1035
	__blk_run_queue(q);
L
Linus Torvalds 已提交
1036 1037 1038 1039 1040 1041 1042 1043 1044
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_insert_request);

/*
 * add-request adds a request to the linked list.
 * queue lock is held and interrupts disabled, as we muck with the
 * request queue list.
 */
1045
static inline void add_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
1046
{
1047
	drive_stat_acct(req, 1);
L
Linus Torvalds 已提交
1048 1049 1050 1051 1052 1053 1054

	/*
	 * elevator indicated where it wants this request to be
	 * inserted at elevator_merge time
	 */
	__elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
}
1055

T
Tejun Heo 已提交
1056 1057 1058 1059 1060 1061
static void part_round_stats_single(int cpu, struct hd_struct *part,
				    unsigned long now)
{
	if (now == part->stamp)
		return;

1062
	if (part_in_flight(part)) {
T
Tejun Heo 已提交
1063
		__part_stat_add(cpu, part, time_in_queue,
1064
				part_in_flight(part) * (now - part->stamp));
T
Tejun Heo 已提交
1065 1066 1067 1068 1069 1070
		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
	}
	part->stamp = now;
}

/**
1071 1072 1073
 * part_round_stats() - Round off the performance stats on a struct disk_stats.
 * @cpu: cpu number for stats access
 * @part: target partition
L
Linus Torvalds 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
T
Tejun Heo 已提交
1086
void part_round_stats(int cpu, struct hd_struct *part)
1087 1088 1089
{
	unsigned long now = jiffies;

T
Tejun Heo 已提交
1090 1091 1092
	if (part->partno)
		part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
	part_round_stats_single(cpu, part, now);
1093
}
T
Tejun Heo 已提交
1094
EXPORT_SYMBOL_GPL(part_round_stats);
1095

L
Linus Torvalds 已提交
1096 1097 1098
/*
 * queue lock must be held
 */
1099
void __blk_put_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
1100 1101 1102 1103 1104 1105
{
	if (unlikely(!q))
		return;
	if (unlikely(--req->ref_count))
		return;

1106 1107
	elv_completed_request(q, req);

1108 1109 1110
	/* this is a bio leak */
	WARN_ON(req->bio != NULL);

L
Linus Torvalds 已提交
1111 1112 1113 1114
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
1115
	if (req->cmd_flags & REQ_ALLOCED) {
1116
		int is_sync = rq_is_sync(req) != 0;
1117
		int priv = req->cmd_flags & REQ_ELVPRIV;
L
Linus Torvalds 已提交
1118 1119

		BUG_ON(!list_empty(&req->queuelist));
1120
		BUG_ON(!hlist_unhashed(&req->hash));
L
Linus Torvalds 已提交
1121 1122

		blk_free_request(q, req);
1123
		freed_request(q, is_sync, priv);
L
Linus Torvalds 已提交
1124 1125
	}
}
1126 1127
EXPORT_SYMBOL_GPL(__blk_put_request);

L
Linus Torvalds 已提交
1128 1129
void blk_put_request(struct request *req)
{
1130
	unsigned long flags;
1131
	struct request_queue *q = req->q;
1132

1133 1134 1135
	spin_lock_irqsave(q->queue_lock, flags);
	__blk_put_request(q, req);
	spin_unlock_irqrestore(q->queue_lock, flags);
L
Linus Torvalds 已提交
1136 1137 1138
}
EXPORT_SYMBOL(blk_put_request);

1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
/**
 * blk_add_request_payload - add a payload to a request
 * @rq: request to update
 * @page: page backing the payload
 * @len: length of the payload.
 *
 * This allows to later add a payload to an already submitted request by
 * a block driver.  The driver needs to take care of freeing the payload
 * itself.
 *
 * Note that this is a quite horrible hack and nothing but handling of
 * discard requests should ever use it.
 */
void blk_add_request_payload(struct request *rq, struct page *page,
		unsigned int len)
{
	struct bio *bio = rq->bio;

	bio->bi_io_vec->bv_page = page;
	bio->bi_io_vec->bv_offset = 0;
	bio->bi_io_vec->bv_len = len;

	bio->bi_size = len;
	bio->bi_vcnt = 1;
	bio->bi_phys_segments = 1;

	rq->__data_len = rq->resid_len = len;
	rq->nr_phys_segments = 1;
	rq->buffer = bio_data(bio);
}
EXPORT_SYMBOL_GPL(blk_add_request_payload);

J
Jens Axboe 已提交
1171
void init_request_from_bio(struct request *req, struct bio *bio)
1172
{
1173
	req->cpu = bio->bi_comp_cpu;
1174
	req->cmd_type = REQ_TYPE_FS;
1175

1176 1177
	req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
	if (bio->bi_rw & REQ_RAHEAD)
1178
		req->cmd_flags |= REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
1179

1180
	req->errors = 0;
1181
	req->__sector = bio->bi_sector;
1182
	req->ioprio = bio_prio(bio);
1183
	blk_rq_bio_prep(req->q, req, bio);
1184 1185
}

1186 1187 1188 1189 1190 1191
/*
 * Only disabling plugging for non-rotational devices if it does tagging
 * as well, otherwise we do need the proper merging
 */
static inline bool queue_should_plug(struct request_queue *q)
{
1192
	return !(blk_queue_nonrot(q) && blk_queue_tagged(q));
1193 1194
}

1195
static int __make_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
1196
{
1197
	struct request *req;
1198 1199
	int el_ret;
	unsigned int bytes = bio->bi_size;
1200
	const unsigned short prio = bio_prio(bio);
1201 1202
	const bool sync = (bio->bi_rw & REQ_SYNC);
	const bool unplug = (bio->bi_rw & REQ_UNPLUG);
1203
	const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1204
	int rw_flags;
L
Linus Torvalds 已提交
1205

1206
	if ((bio->bi_rw & REQ_HARDBARRIER) &&
1207 1208 1209 1210
	    (q->next_ordered == QUEUE_ORDERED_NONE)) {
		bio_endio(bio, -EOPNOTSUPP);
		return 0;
	}
L
Linus Torvalds 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219
	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

	spin_lock_irq(q->queue_lock);

1220
	if (unlikely((bio->bi_rw & REQ_HARDBARRIER)) || elv_queue_empty(q))
L
Linus Torvalds 已提交
1221 1222 1223 1224
		goto get_rq;

	el_ret = elv_merge(q, &req, bio);
	switch (el_ret) {
1225 1226
	case ELEVATOR_BACK_MERGE:
		BUG_ON(!rq_mergeable(req));
L
Linus Torvalds 已提交
1227

1228 1229
		if (!ll_back_merge_fn(q, req, bio))
			break;
L
Linus Torvalds 已提交
1230

1231
		trace_block_bio_backmerge(q, bio);
1232

1233 1234 1235
		if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
			blk_rq_set_mixed_merge(req);

1236 1237
		req->biotail->bi_next = bio;
		req->biotail = bio;
1238
		req->__data_len += bytes;
1239
		req->ioprio = ioprio_best(req->ioprio, prio);
1240 1241
		if (!blk_rq_cpu_valid(req))
			req->cpu = bio->bi_comp_cpu;
1242
		drive_stat_acct(req, 0);
D
Divyesh Shah 已提交
1243
		elv_bio_merged(q, req, bio);
1244 1245 1246
		if (!attempt_back_merge(q, req))
			elv_merged_request(q, req, el_ret);
		goto out;
L
Linus Torvalds 已提交
1247

1248 1249
	case ELEVATOR_FRONT_MERGE:
		BUG_ON(!rq_mergeable(req));
L
Linus Torvalds 已提交
1250

1251 1252
		if (!ll_front_merge_fn(q, req, bio))
			break;
L
Linus Torvalds 已提交
1253

1254
		trace_block_bio_frontmerge(q, bio);
1255

1256 1257 1258 1259 1260 1261
		if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) {
			blk_rq_set_mixed_merge(req);
			req->cmd_flags &= ~REQ_FAILFAST_MASK;
			req->cmd_flags |= ff;
		}

1262 1263
		bio->bi_next = req->bio;
		req->bio = bio;
L
Linus Torvalds 已提交
1264

1265 1266 1267 1268 1269 1270
		/*
		 * may not be valid. if the low level driver said
		 * it didn't need a bounce buffer then it better
		 * not touch req->buffer either...
		 */
		req->buffer = bio_data(bio);
1271 1272
		req->__sector = bio->bi_sector;
		req->__data_len += bytes;
1273
		req->ioprio = ioprio_best(req->ioprio, prio);
1274 1275
		if (!blk_rq_cpu_valid(req))
			req->cpu = bio->bi_comp_cpu;
1276
		drive_stat_acct(req, 0);
D
Divyesh Shah 已提交
1277
		elv_bio_merged(q, req, bio);
1278 1279 1280 1281 1282 1283 1284
		if (!attempt_front_merge(q, req))
			elv_merged_request(q, req, el_ret);
		goto out;

	/* ELV_NO_MERGE: elevator says don't/can't merge. */
	default:
		;
L
Linus Torvalds 已提交
1285 1286
	}

1287
get_rq:
1288 1289 1290 1291 1292 1293 1294
	/*
	 * This sync check and mask will be re-done in init_request_from_bio(),
	 * but we need to set it earlier to expose the sync flag to the
	 * rq allocator and io schedulers.
	 */
	rw_flags = bio_data_dir(bio);
	if (sync)
1295
		rw_flags |= REQ_SYNC;
1296

L
Linus Torvalds 已提交
1297
	/*
1298
	 * Grab a free request. This is might sleep but can not fail.
N
Nick Piggin 已提交
1299
	 * Returns with the queue unlocked.
1300
	 */
1301
	req = get_request_wait(q, rw_flags, bio);
N
Nick Piggin 已提交
1302

1303 1304 1305 1306 1307
	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
L
Linus Torvalds 已提交
1308
	 */
1309
	init_request_from_bio(req, bio);
L
Linus Torvalds 已提交
1310

1311
	spin_lock_irq(q->queue_lock);
1312 1313 1314
	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) ||
	    bio_flagged(bio, BIO_CPU_AFFINE))
		req->cpu = blk_cpu_to_group(smp_processor_id());
1315
	if (queue_should_plug(q) && elv_queue_empty(q))
1316
		blk_plug_device(q);
L
Linus Torvalds 已提交
1317 1318
	add_request(q, req);
out:
1319
	if (unplug || !queue_should_plug(q))
L
Linus Torvalds 已提交
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331
		__generic_unplug_device(q);
	spin_unlock_irq(q->queue_lock);
	return 0;
}

/*
 * If bio->bi_dev is a partition, remap the location
 */
static inline void blk_partition_remap(struct bio *bio)
{
	struct block_device *bdev = bio->bi_bdev;

1332
	if (bio_sectors(bio) && bdev != bdev->bd_contains) {
L
Linus Torvalds 已提交
1333 1334 1335 1336
		struct hd_struct *p = bdev->bd_part;

		bio->bi_sector += p->start_sect;
		bio->bi_bdev = bdev->bd_contains;
1337

1338
		trace_block_remap(bdev_get_queue(bio->bi_bdev), bio,
1339
				    bdev->bd_dev,
1340
				    bio->bi_sector - p->start_sect);
L
Linus Torvalds 已提交
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
	}
}

static void handle_bad_sector(struct bio *bio)
{
	char b[BDEVNAME_SIZE];

	printk(KERN_INFO "attempt to access beyond end of device\n");
	printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
			bdevname(bio->bi_bdev, b),
			bio->bi_rw,
			(unsigned long long)bio->bi_sector + bio_sectors(bio),
			(long long)(bio->bi_bdev->bd_inode->i_size >> 9));

	set_bit(BIO_EOF, &bio->bi_flags);
}

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
	return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

static int should_fail_request(struct bio *bio)
{
1370 1371 1372
	struct hd_struct *part = bio->bi_bdev->bd_part;

	if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail)
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
		return should_fail(&fail_make_request, bio->bi_size);

	return 0;
}

static int __init fail_make_request_debugfs(void)
{
	return init_fault_attr_dentries(&fail_make_request,
					"fail_make_request");
}

late_initcall(fail_make_request_debugfs);

#else /* CONFIG_FAIL_MAKE_REQUEST */

static inline int should_fail_request(struct bio *bio)
{
	return 0;
}

#endif /* CONFIG_FAIL_MAKE_REQUEST */

J
Jens Axboe 已提交
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
/*
 * Check whether this bio extends beyond the end of the device.
 */
static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
{
	sector_t maxsector;

	if (!nr_sectors)
		return 0;

	/* Test device or partition size, when known. */
	maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
	if (maxsector) {
		sector_t sector = bio->bi_sector;

		if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
			/*
			 * This may well happen - the kernel calls bread()
			 * without checking the size of the device, e.g., when
			 * mounting a device.
			 */
			handle_bad_sector(bio);
			return 1;
		}
	}

	return 0;
}

L
Linus Torvalds 已提交
1424
/**
1425
 * generic_make_request - hand a buffer to its device driver for I/O
L
Linus Torvalds 已提交
1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
 * @bio:  The bio describing the location in memory and on the device.
 *
 * generic_make_request() is used to make I/O requests of block
 * devices. It is passed a &struct bio, which describes the I/O that needs
 * to be done.
 *
 * generic_make_request() does not return any status.  The
 * success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the bio->bi_end_io
 * function described (one day) else where.
 *
 * The caller of generic_make_request must make sure that bi_io_vec
 * are set to describe the memory buffer, and that bi_dev and bi_sector are
 * set to describe the device address, and the
 * bi_end_io and optionally bi_private are set to describe how
 * completion notification should be signaled.
 *
 * generic_make_request and the drivers it calls may use bi_next if this
 * bio happens to be merged with someone else, and may change bi_dev and
 * bi_sector for remaps as it sees fit.  So the values of these fields
 * should NOT be depended on after the call to generic_make_request.
 */
1448
static inline void __generic_make_request(struct bio *bio)
L
Linus Torvalds 已提交
1449
{
1450
	struct request_queue *q;
1451
	sector_t old_sector;
L
Linus Torvalds 已提交
1452
	int ret, nr_sectors = bio_sectors(bio);
1453
	dev_t old_dev;
1454
	int err = -EIO;
L
Linus Torvalds 已提交
1455 1456 1457

	might_sleep();

J
Jens Axboe 已提交
1458 1459
	if (bio_check_eod(bio, nr_sectors))
		goto end_io;
L
Linus Torvalds 已提交
1460 1461 1462 1463 1464 1465 1466 1467 1468

	/*
	 * Resolve the mapping until finished. (drivers are
	 * still free to implement/resolve their own stacking
	 * by explicitly returning 0)
	 *
	 * NOTE: we don't repeat the blk_size check for each new device.
	 * Stacking drivers are expected to know what they are doing.
	 */
1469
	old_sector = -1;
1470
	old_dev = 0;
L
Linus Torvalds 已提交
1471 1472 1473 1474
	do {
		char b[BDEVNAME_SIZE];

		q = bdev_get_queue(bio->bi_bdev);
1475
		if (unlikely(!q)) {
L
Linus Torvalds 已提交
1476 1477 1478 1479 1480
			printk(KERN_ERR
			       "generic_make_request: Trying to access "
				"nonexistent block-device %s (%Lu)\n",
				bdevname(bio->bi_bdev, b),
				(long long) bio->bi_sector);
1481
			goto end_io;
L
Linus Torvalds 已提交
1482 1483
		}

1484
		if (unlikely(!(bio->bi_rw & REQ_DISCARD) &&
1485
			     nr_sectors > queue_max_hw_sectors(q))) {
1486
			printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1487 1488 1489
			       bdevname(bio->bi_bdev, b),
			       bio_sectors(bio),
			       queue_max_hw_sectors(q));
L
Linus Torvalds 已提交
1490 1491 1492
			goto end_io;
		}

N
Nick Piggin 已提交
1493
		if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
L
Linus Torvalds 已提交
1494 1495
			goto end_io;

1496 1497 1498
		if (should_fail_request(bio))
			goto end_io;

L
Linus Torvalds 已提交
1499 1500 1501 1502 1503 1504
		/*
		 * If this device has partitions, remap block n
		 * of partition p to block n+start(p) of the disk.
		 */
		blk_partition_remap(bio);

1505 1506 1507
		if (bio_integrity_enabled(bio) && bio_integrity_prep(bio))
			goto end_io;

1508
		if (old_sector != -1)
1509
			trace_block_remap(q, bio, old_dev, old_sector);
1510

1511
		old_sector = bio->bi_sector;
1512 1513
		old_dev = bio->bi_bdev->bd_dev;

J
Jens Axboe 已提交
1514 1515
		if (bio_check_eod(bio, nr_sectors))
			goto end_io;
1516

1517
		if ((bio->bi_rw & REQ_DISCARD) && !blk_queue_discard(q)) {
1518 1519 1520
			err = -EOPNOTSUPP;
			goto end_io;
		}
1521

1522 1523
		trace_block_bio_queue(q, bio);

L
Linus Torvalds 已提交
1524 1525
		ret = q->make_request_fn(q, bio);
	} while (ret);
1526 1527 1528 1529 1530

	return;

end_io:
	bio_endio(bio, err);
L
Linus Torvalds 已提交
1531 1532
}

1533 1534 1535
/*
 * We only want one ->make_request_fn to be active at a time,
 * else stack usage with stacked devices could be a problem.
1536
 * So use current->bio_list to keep a list of requests
1537
 * submited by a make_request_fn function.
1538
 * current->bio_list is also used as a flag to say if
1539 1540 1541 1542 1543 1544 1545
 * generic_make_request is currently active in this task or not.
 * If it is NULL, then no make_request is active.  If it is non-NULL,
 * then a make_request is active, and new requests should be added
 * at the tail
 */
void generic_make_request(struct bio *bio)
{
1546 1547 1548
	struct bio_list bio_list_on_stack;

	if (current->bio_list) {
1549
		/* make_request is active */
1550
		bio_list_add(current->bio_list, bio);
1551 1552 1553 1554 1555 1556 1557
		return;
	}
	/* following loop may be a bit non-obvious, and so deserves some
	 * explanation.
	 * Before entering the loop, bio->bi_next is NULL (as all callers
	 * ensure that) so we have a list with a single bio.
	 * We pretend that we have just taken it off a longer list, so
1558 1559
	 * we assign bio_list to a pointer to the bio_list_on_stack,
	 * thus initialising the bio_list of new bios to be
1560 1561 1562 1563
	 * added.  __generic_make_request may indeed add some more bios
	 * through a recursive call to generic_make_request.  If it
	 * did, we find a non-NULL value in bio_list and re-enter the loop
	 * from the top.  In this case we really did just take the bio
1564 1565
	 * of the top of the list (no pretending) and so remove it from
	 * bio_list, and call into __generic_make_request again.
1566 1567 1568 1569 1570 1571
	 *
	 * The loop was structured like this to make only one call to
	 * __generic_make_request (which is important as it is large and
	 * inlined) and to keep the structure simple.
	 */
	BUG_ON(bio->bi_next);
1572 1573
	bio_list_init(&bio_list_on_stack);
	current->bio_list = &bio_list_on_stack;
1574 1575
	do {
		__generic_make_request(bio);
1576
		bio = bio_list_pop(current->bio_list);
1577
	} while (bio);
1578
	current->bio_list = NULL; /* deactivate */
1579
}
L
Linus Torvalds 已提交
1580 1581 1582
EXPORT_SYMBOL(generic_make_request);

/**
1583
 * submit_bio - submit a bio to the block device layer for I/O
L
Linus Torvalds 已提交
1584 1585 1586 1587 1588
 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is very similar in purpose to generic_make_request(), and
 * uses that function to do most of the work. Both are fairly rough
1589
 * interfaces; @bio must be presetup and ready for I/O.
L
Linus Torvalds 已提交
1590 1591 1592 1593 1594 1595
 *
 */
void submit_bio(int rw, struct bio *bio)
{
	int count = bio_sectors(bio);

1596
	bio->bi_rw |= rw;
L
Linus Torvalds 已提交
1597

1598 1599 1600 1601
	/*
	 * If it's a regular read/write or a barrier with data attached,
	 * go through the normal accounting stuff before submission.
	 */
1602
	if (bio_has_data(bio) && !(rw & REQ_DISCARD)) {
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
		if (rw & WRITE) {
			count_vm_events(PGPGOUT, count);
		} else {
			task_io_account_read(bio->bi_size);
			count_vm_events(PGPGIN, count);
		}

		if (unlikely(block_dump)) {
			char b[BDEVNAME_SIZE];
			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
1613
			current->comm, task_pid_nr(current),
1614 1615
				(rw & WRITE) ? "WRITE" : "READ",
				(unsigned long long)bio->bi_sector,
1616
				bdevname(bio->bi_bdev, b));
1617
		}
L
Linus Torvalds 已提交
1618 1619 1620 1621 1622 1623
	}

	generic_make_request(bio);
}
EXPORT_SYMBOL(submit_bio);

1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
/**
 * blk_rq_check_limits - Helper function to check a request for the queue limit
 * @q:  the queue
 * @rq: the request being checked
 *
 * Description:
 *    @rq may have been made based on weaker limitations of upper-level queues
 *    in request stacking drivers, and it may violate the limitation of @q.
 *    Since the block layer and the underlying device driver trust @rq
 *    after it is inserted to @q, it should be checked against @q before
 *    the insertion using this generic function.
 *
 *    This function should also be useful for request stacking drivers
 *    in some cases below, so export this fuction.
 *    Request stacking drivers like request-based dm may change the queue
 *    limits while requests are in the queue (e.g. dm's table swapping).
 *    Such request stacking drivers should check those requests agaist
 *    the new queue limits again when they dispatch those requests,
 *    although such checkings are also done against the old queue limits
 *    when submitting requests.
 */
int blk_rq_check_limits(struct request_queue *q, struct request *rq)
{
1647 1648 1649
	if (rq->cmd_flags & REQ_DISCARD)
		return 0;

1650 1651
	if (blk_rq_sectors(rq) > queue_max_sectors(q) ||
	    blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) {
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
		printk(KERN_ERR "%s: over max size limit.\n", __func__);
		return -EIO;
	}

	/*
	 * queue's settings related to segment counting like q->bounce_pfn
	 * may differ from that of other stacking queues.
	 * Recalculate it to check the request correctly on this queue's
	 * limitation.
	 */
	blk_recalc_rq_segments(rq);
1663
	if (rq->nr_phys_segments > queue_max_segments(q)) {
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
		return -EIO;
	}

	return 0;
}
EXPORT_SYMBOL_GPL(blk_rq_check_limits);

/**
 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
 * @q:  the queue to submit the request
 * @rq: the request being queued
 */
int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
{
	unsigned long flags;

	if (blk_rq_check_limits(q, rq))
		return -EIO;

#ifdef CONFIG_FAIL_MAKE_REQUEST
	if (rq->rq_disk && rq->rq_disk->part0.make_it_fail &&
	    should_fail(&fail_make_request, blk_rq_bytes(rq)))
		return -EIO;
#endif

	spin_lock_irqsave(q->queue_lock, flags);

	/*
	 * Submitting request must be dequeued before calling this function
	 * because it will be linked to another request_queue
	 */
	BUG_ON(blk_queued_rq(rq));

	drive_stat_acct(rq, 1);
	__elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0);

	spin_unlock_irqrestore(q->queue_lock, flags);

	return 0;
}
EXPORT_SYMBOL_GPL(blk_insert_cloned_request);

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
/**
 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
 * @rq: request to examine
 *
 * Description:
 *     A request could be merge of IOs which require different failure
 *     handling.  This function determines the number of bytes which
 *     can be failed from the beginning of the request without
 *     crossing into area which need to be retried further.
 *
 * Return:
 *     The number of bytes to fail.
 *
 * Context:
 *     queue_lock must be held.
 */
unsigned int blk_rq_err_bytes(const struct request *rq)
{
	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
	unsigned int bytes = 0;
	struct bio *bio;

	if (!(rq->cmd_flags & REQ_MIXED_MERGE))
		return blk_rq_bytes(rq);

	/*
	 * Currently the only 'mixing' which can happen is between
	 * different fastfail types.  We can safely fail portions
	 * which have all the failfast bits that the first one has -
	 * the ones which are at least as eager to fail as the first
	 * one.
	 */
	for (bio = rq->bio; bio; bio = bio->bi_next) {
		if ((bio->bi_rw & ff) != ff)
			break;
		bytes += bio->bi_size;
	}

	/* this could lead to infinite loop */
	BUG_ON(blk_rq_bytes(rq) && !bytes);
	return bytes;
}
EXPORT_SYMBOL_GPL(blk_rq_err_bytes);

1751 1752
static void blk_account_io_completion(struct request *req, unsigned int bytes)
{
1753
	if (blk_do_io_stat(req)) {
1754 1755 1756 1757 1758
		const int rw = rq_data_dir(req);
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
1759
		part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
		part_stat_add(cpu, part, sectors[rw], bytes >> 9);
		part_stat_unlock();
	}
}

static void blk_account_io_done(struct request *req)
{
	/*
	 * Account IO completion.  bar_rq isn't accounted as a normal
	 * IO on queueing nor completion.  Accounting the containing
	 * request is enough.
	 */
1772
	if (blk_do_io_stat(req) && req != &req->q->bar_rq) {
1773 1774 1775 1776 1777 1778
		unsigned long duration = jiffies - req->start_time;
		const int rw = rq_data_dir(req);
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
1779
		part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
1780 1781 1782 1783

		part_stat_inc(cpu, part, ios[rw]);
		part_stat_add(cpu, part, ticks[rw], duration);
		part_round_stats(cpu, part);
1784
		part_dec_in_flight(part, rw);
1785 1786 1787 1788 1789

		part_stat_unlock();
	}
}

1790
/**
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
 * blk_peek_request - peek at the top of a request queue
 * @q: request queue to peek at
 *
 * Description:
 *     Return the request at the top of @q.  The returned request
 *     should be started using blk_start_request() before LLD starts
 *     processing it.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 *
 * Context:
 *     queue_lock must be held.
 */
struct request *blk_peek_request(struct request_queue *q)
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
{
	struct request *rq;
	int ret;

	while ((rq = __elv_next_request(q)) != NULL) {
		if (!(rq->cmd_flags & REQ_STARTED)) {
			/*
			 * This is the first time the device driver
			 * sees this request (possibly after
			 * requeueing).  Notify IO scheduler.
			 */
1818
			if (rq->cmd_flags & REQ_SORTED)
1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
				elv_activate_rq(q, rq);

			/*
			 * just mark as started even if we don't start
			 * it, a request that has been delayed should
			 * not be passed by new incoming requests
			 */
			rq->cmd_flags |= REQ_STARTED;
			trace_block_rq_issue(q, rq);
		}

		if (!q->boundary_rq || q->boundary_rq == rq) {
			q->end_sector = rq_end_sector(rq);
			q->boundary_rq = NULL;
		}

		if (rq->cmd_flags & REQ_DONTPREP)
			break;

1838
		if (q->dma_drain_size && blk_rq_bytes(rq)) {
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
			/*
			 * make sure space for the drain appears we
			 * know we can do this because max_hw_segments
			 * has been adjusted to be one fewer than the
			 * device can handle
			 */
			rq->nr_phys_segments++;
		}

		if (!q->prep_rq_fn)
			break;

		ret = q->prep_rq_fn(q, rq);
		if (ret == BLKPREP_OK) {
			break;
		} else if (ret == BLKPREP_DEFER) {
			/*
			 * the request may have been (partially) prepped.
			 * we need to keep this request in the front to
			 * avoid resource deadlock.  REQ_STARTED will
			 * prevent other fs requests from passing this one.
			 */
1861
			if (q->dma_drain_size && blk_rq_bytes(rq) &&
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
			    !(rq->cmd_flags & REQ_DONTPREP)) {
				/*
				 * remove the space for the drain we added
				 * so that we don't add it again
				 */
				--rq->nr_phys_segments;
			}

			rq = NULL;
			break;
		} else if (ret == BLKPREP_KILL) {
			rq->cmd_flags |= REQ_QUIET;
1874 1875 1876 1877 1878
			/*
			 * Mark this request as started so we don't trigger
			 * any debug logic in the end I/O path.
			 */
			blk_start_request(rq);
1879
			__blk_end_request_all(rq, -EIO);
1880 1881 1882 1883 1884 1885 1886 1887
		} else {
			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
			break;
		}
	}

	return rq;
}
1888
EXPORT_SYMBOL(blk_peek_request);
1889

1890
void blk_dequeue_request(struct request *rq)
1891
{
1892 1893
	struct request_queue *q = rq->q;

1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
	BUG_ON(list_empty(&rq->queuelist));
	BUG_ON(ELV_ON_HASH(rq));

	list_del_init(&rq->queuelist);

	/*
	 * the time frame between a request being removed from the lists
	 * and to it is freed is accounted as io that is in progress at
	 * the driver side.
	 */
1904
	if (blk_account_rq(rq)) {
1905
		q->in_flight[rq_is_sync(rq)]++;
1906 1907
		set_io_start_time_ns(rq);
	}
1908 1909
}

1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
/**
 * blk_start_request - start request processing on the driver
 * @req: request to dequeue
 *
 * Description:
 *     Dequeue @req and start timeout timer on it.  This hands off the
 *     request to the driver.
 *
 *     Block internal functions which don't want to start timer should
 *     call blk_dequeue_request().
 *
 * Context:
 *     queue_lock must be held.
 */
void blk_start_request(struct request *req)
{
	blk_dequeue_request(req);

	/*
1929 1930
	 * We are now handing the request to the hardware, initialize
	 * resid_len to full count and add the timeout handler.
1931
	 */
1932
	req->resid_len = blk_rq_bytes(req);
1933 1934 1935
	if (unlikely(blk_bidi_rq(req)))
		req->next_rq->resid_len = blk_rq_bytes(req->next_rq);

1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965
	blk_add_timer(req);
}
EXPORT_SYMBOL(blk_start_request);

/**
 * blk_fetch_request - fetch a request from a request queue
 * @q: request queue to fetch a request from
 *
 * Description:
 *     Return the request at the top of @q.  The request is started on
 *     return and LLD can start processing it immediately.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 *
 * Context:
 *     queue_lock must be held.
 */
struct request *blk_fetch_request(struct request_queue *q)
{
	struct request *rq;

	rq = blk_peek_request(q);
	if (rq)
		blk_start_request(rq);
	return rq;
}
EXPORT_SYMBOL(blk_fetch_request);

1966
/**
1967
 * blk_update_request - Special helper function for request stacking drivers
1968
 * @req:      the request being processed
1969
 * @error:    %0 for success, < %0 for error
1970
 * @nr_bytes: number of bytes to complete @req
1971 1972
 *
 * Description:
1973 1974 1975
 *     Ends I/O on a number of bytes attached to @req, but doesn't complete
 *     the request structure even if @req doesn't have leftover.
 *     If @req has leftover, sets it up for the next range of segments.
1976 1977 1978 1979 1980 1981 1982
 *
 *     This special helper function is only for request stacking drivers
 *     (e.g. request-based dm) so that they can handle partial completion.
 *     Actual device drivers should use blk_end_request instead.
 *
 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
 *     %false return from this function.
1983 1984
 *
 * Return:
1985 1986
 *     %false - this request doesn't have any more data
 *     %true  - this request has more data
1987
 **/
1988
bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
L
Linus Torvalds 已提交
1989
{
1990
	int total_bytes, bio_nbytes, next_idx = 0;
L
Linus Torvalds 已提交
1991 1992
	struct bio *bio;

1993 1994 1995
	if (!req->bio)
		return false;

1996
	trace_block_rq_complete(req->q, req);
1997

L
Linus Torvalds 已提交
1998
	/*
1999 2000 2001 2002 2003 2004
	 * For fs requests, rq is just carrier of independent bio's
	 * and each partial completion should be handled separately.
	 * Reset per-request error on each partial completion.
	 *
	 * TODO: tj: This is too subtle.  It would be better to let
	 * low level drivers do what they see fit.
L
Linus Torvalds 已提交
2005
	 */
2006
	if (req->cmd_type == REQ_TYPE_FS)
L
Linus Torvalds 已提交
2007 2008
		req->errors = 0;

2009 2010
	if (error && req->cmd_type == REQ_TYPE_FS &&
	    !(req->cmd_flags & REQ_QUIET)) {
2011
		printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
L
Linus Torvalds 已提交
2012
				req->rq_disk ? req->rq_disk->disk_name : "?",
2013
				(unsigned long long)blk_rq_pos(req));
L
Linus Torvalds 已提交
2014 2015
	}

2016
	blk_account_io_completion(req, nr_bytes);
2017

L
Linus Torvalds 已提交
2018 2019 2020 2021 2022 2023 2024
	total_bytes = bio_nbytes = 0;
	while ((bio = req->bio) != NULL) {
		int nbytes;

		if (nr_bytes >= bio->bi_size) {
			req->bio = bio->bi_next;
			nbytes = bio->bi_size;
N
NeilBrown 已提交
2025
			req_bio_endio(req, bio, nbytes, error);
L
Linus Torvalds 已提交
2026 2027 2028 2029 2030
			next_idx = 0;
			bio_nbytes = 0;
		} else {
			int idx = bio->bi_idx + next_idx;

2031
			if (unlikely(idx >= bio->bi_vcnt)) {
L
Linus Torvalds 已提交
2032
				blk_dump_rq_flags(req, "__end_that");
2033
				printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
2034
				       __func__, idx, bio->bi_vcnt);
L
Linus Torvalds 已提交
2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
				break;
			}

			nbytes = bio_iovec_idx(bio, idx)->bv_len;
			BIO_BUG_ON(nbytes > bio->bi_size);

			/*
			 * not a complete bvec done
			 */
			if (unlikely(nbytes > nr_bytes)) {
				bio_nbytes += nr_bytes;
				total_bytes += nr_bytes;
				break;
			}

			/*
			 * advance to the next vector
			 */
			next_idx++;
			bio_nbytes += nbytes;
		}

		total_bytes += nbytes;
		nr_bytes -= nbytes;

2060 2061
		bio = req->bio;
		if (bio) {
L
Linus Torvalds 已提交
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072
			/*
			 * end more in this run, or just return 'not-done'
			 */
			if (unlikely(nr_bytes <= 0))
				break;
		}
	}

	/*
	 * completely done
	 */
2073 2074 2075 2076 2077 2078
	if (!req->bio) {
		/*
		 * Reset counters so that the request stacking driver
		 * can find how many bytes remain in the request
		 * later.
		 */
2079
		req->__data_len = 0;
2080 2081
		return false;
	}
L
Linus Torvalds 已提交
2082 2083 2084 2085 2086

	/*
	 * if the request wasn't completed, update state
	 */
	if (bio_nbytes) {
N
NeilBrown 已提交
2087
		req_bio_endio(req, bio, bio_nbytes, error);
L
Linus Torvalds 已提交
2088 2089 2090 2091 2092
		bio->bi_idx += next_idx;
		bio_iovec(bio)->bv_offset += nr_bytes;
		bio_iovec(bio)->bv_len -= nr_bytes;
	}

2093
	req->__data_len -= total_bytes;
2094 2095 2096
	req->buffer = bio_data(req->bio);

	/* update sector only for requests with clear definition of sector */
2097
	if (req->cmd_type == REQ_TYPE_FS || (req->cmd_flags & REQ_DISCARD))
2098
		req->__sector += total_bytes >> 9;
2099

2100 2101 2102 2103 2104 2105
	/* mixed attributes always follow the first bio */
	if (req->cmd_flags & REQ_MIXED_MERGE) {
		req->cmd_flags &= ~REQ_FAILFAST_MASK;
		req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
	}

2106 2107 2108 2109 2110 2111
	/*
	 * If total number of sectors is less than the first segment
	 * size, something has gone terribly wrong.
	 */
	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
		printk(KERN_ERR "blk: request botched\n");
2112
		req->__data_len = blk_rq_cur_bytes(req);
2113 2114 2115
	}

	/* recalculate the number of segments */
L
Linus Torvalds 已提交
2116
	blk_recalc_rq_segments(req);
2117

2118
	return true;
L
Linus Torvalds 已提交
2119
}
2120
EXPORT_SYMBOL_GPL(blk_update_request);
L
Linus Torvalds 已提交
2121

2122 2123 2124
static bool blk_update_bidi_request(struct request *rq, int error,
				    unsigned int nr_bytes,
				    unsigned int bidi_bytes)
2125
{
2126 2127
	if (blk_update_request(rq, error, nr_bytes))
		return true;
2128

2129 2130 2131 2132
	/* Bidi request must be completed as a whole */
	if (unlikely(blk_bidi_rq(rq)) &&
	    blk_update_request(rq->next_rq, error, bidi_bytes))
		return true;
2133

2134 2135
	if (blk_queue_add_random(rq->q))
		add_disk_randomness(rq->rq_disk);
2136 2137

	return false;
L
Linus Torvalds 已提交
2138 2139
}

2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
/**
 * blk_unprep_request - unprepare a request
 * @req:	the request
 *
 * This function makes a request ready for complete resubmission (or
 * completion).  It happens only after all error handling is complete,
 * so represents the appropriate moment to deallocate any resources
 * that were allocated to the request in the prep_rq_fn.  The queue
 * lock is held when calling this.
 */
void blk_unprep_request(struct request *req)
{
	struct request_queue *q = req->q;

	req->cmd_flags &= ~REQ_DONTPREP;
	if (q->unprep_rq_fn)
		q->unprep_rq_fn(q, req);
}
EXPORT_SYMBOL_GPL(blk_unprep_request);

L
Linus Torvalds 已提交
2160 2161 2162
/*
 * queue lock must be held
 */
2163
static void blk_finish_request(struct request *req, int error)
L
Linus Torvalds 已提交
2164
{
2165 2166 2167
	if (blk_rq_tagged(req))
		blk_queue_end_tag(req->q, req);

2168
	BUG_ON(blk_queued_rq(req));
L
Linus Torvalds 已提交
2169

2170
	if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
2171
		laptop_io_completion(&req->q->backing_dev_info);
L
Linus Torvalds 已提交
2172

2173 2174
	blk_delete_timer(req);

2175 2176 2177 2178
	if (req->cmd_flags & REQ_DONTPREP)
		blk_unprep_request(req);


2179
	blk_account_io_done(req);
2180

L
Linus Torvalds 已提交
2181
	if (req->end_io)
2182
		req->end_io(req, error);
2183 2184 2185 2186
	else {
		if (blk_bidi_rq(req))
			__blk_put_request(req->next_rq->q, req->next_rq);

L
Linus Torvalds 已提交
2187
		__blk_put_request(req->q, req);
2188
	}
L
Linus Torvalds 已提交
2189 2190
}

2191
/**
2192 2193 2194 2195 2196
 * blk_end_bidi_request - Complete a bidi request
 * @rq:         the request to complete
 * @error:      %0 for success, < %0 for error
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
2197 2198
 *
 * Description:
2199
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
2200 2201 2202
 *     Drivers that supports bidi can safely call this member for any
 *     type of request, bidi or uni.  In the later case @bidi_bytes is
 *     just ignored.
2203 2204
 *
 * Return:
2205 2206
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
2207
 **/
2208
static bool blk_end_bidi_request(struct request *rq, int error,
K
Kiyoshi Ueda 已提交
2209 2210
				 unsigned int nr_bytes, unsigned int bidi_bytes)
{
2211
	struct request_queue *q = rq->q;
2212
	unsigned long flags;
K
Kiyoshi Ueda 已提交
2213

2214 2215
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
K
Kiyoshi Ueda 已提交
2216

2217
	spin_lock_irqsave(q->queue_lock, flags);
2218
	blk_finish_request(rq, error);
2219 2220
	spin_unlock_irqrestore(q->queue_lock, flags);

2221
	return false;
K
Kiyoshi Ueda 已提交
2222 2223
}

2224
/**
2225 2226
 * __blk_end_bidi_request - Complete a bidi request with queue lock held
 * @rq:         the request to complete
2227
 * @error:      %0 for success, < %0 for error
2228 2229
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
2230 2231
 *
 * Description:
2232 2233
 *     Identical to blk_end_bidi_request() except that queue lock is
 *     assumed to be locked on entry and remains so on return.
2234 2235
 *
 * Return:
2236 2237
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
2238
 **/
2239 2240
static bool __blk_end_bidi_request(struct request *rq, int error,
				   unsigned int nr_bytes, unsigned int bidi_bytes)
2241
{
2242 2243
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
2244

2245
	blk_finish_request(rq, error);
2246

2247
	return false;
2248
}
2249 2250 2251 2252

/**
 * blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
2253
 * @error:    %0 for success, < %0 for error
2254 2255 2256 2257 2258 2259 2260
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
2261 2262
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
2263
 **/
2264
bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2265
{
2266
	return blk_end_bidi_request(rq, error, nr_bytes, 0);
2267
}
2268
EXPORT_SYMBOL(blk_end_request);
2269 2270

/**
2271 2272
 * blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
2273
 * @error: %0 for success, < %0 for error
2274 2275
 *
 * Description:
2276 2277 2278
 *     Completely finish @rq.
 */
void blk_end_request_all(struct request *rq, int error)
2279
{
2280 2281
	bool pending;
	unsigned int bidi_bytes = 0;
2282

2283 2284
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);
2285

2286 2287 2288
	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
}
2289
EXPORT_SYMBOL(blk_end_request_all);
2290

2291 2292 2293
/**
 * blk_end_request_cur - Helper function to finish the current request chunk.
 * @rq: the request to finish the current chunk for
2294
 * @error: %0 for success, < %0 for error
2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305
 *
 * Description:
 *     Complete the current consecutively mapped chunk from @rq.
 *
 * Return:
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
bool blk_end_request_cur(struct request *rq, int error)
{
	return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2306
}
2307
EXPORT_SYMBOL(blk_end_request_cur);
2308

2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327
/**
 * blk_end_request_err - Finish a request till the next failure boundary.
 * @rq: the request to finish till the next failure boundary for
 * @error: must be negative errno
 *
 * Description:
 *     Complete @rq till the next failure boundary.
 *
 * Return:
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
bool blk_end_request_err(struct request *rq, int error)
{
	WARN_ON(error >= 0);
	return blk_end_request(rq, error, blk_rq_err_bytes(rq));
}
EXPORT_SYMBOL_GPL(blk_end_request_err);

2328
/**
2329 2330 2331 2332
 * __blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
 * @error:    %0 for success, < %0 for error
 * @nr_bytes: number of bytes to complete
2333 2334
 *
 * Description:
2335
 *     Must be called with queue lock held unlike blk_end_request().
2336 2337
 *
 * Return:
2338 2339
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
2340
 **/
2341
bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2342
{
2343
	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2344
}
2345
EXPORT_SYMBOL(__blk_end_request);
2346

K
Kiyoshi Ueda 已提交
2347
/**
2348 2349
 * __blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
2350
 * @error: %0 for success, < %0 for error
K
Kiyoshi Ueda 已提交
2351 2352
 *
 * Description:
2353
 *     Completely finish @rq.  Must be called with queue lock held.
K
Kiyoshi Ueda 已提交
2354
 */
2355
void __blk_end_request_all(struct request *rq, int error)
K
Kiyoshi Ueda 已提交
2356
{
2357 2358 2359 2360 2361 2362 2363 2364
	bool pending;
	unsigned int bidi_bytes = 0;

	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);

	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
K
Kiyoshi Ueda 已提交
2365
}
2366
EXPORT_SYMBOL(__blk_end_request_all);
K
Kiyoshi Ueda 已提交
2367

2368
/**
2369 2370
 * __blk_end_request_cur - Helper function to finish the current request chunk.
 * @rq: the request to finish the current chunk for
2371
 * @error: %0 for success, < %0 for error
2372 2373
 *
 * Description:
2374 2375
 *     Complete the current consecutively mapped chunk from @rq.  Must
 *     be called with queue lock held.
2376 2377
 *
 * Return:
2378 2379 2380 2381
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
bool __blk_end_request_cur(struct request *rq, int error)
2382
{
2383
	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2384
}
2385
EXPORT_SYMBOL(__blk_end_request_cur);
2386

2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406
/**
 * __blk_end_request_err - Finish a request till the next failure boundary.
 * @rq: the request to finish till the next failure boundary for
 * @error: must be negative errno
 *
 * Description:
 *     Complete @rq till the next failure boundary.  Must be called
 *     with queue lock held.
 *
 * Return:
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
bool __blk_end_request_err(struct request *rq, int error)
{
	WARN_ON(error >= 0);
	return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
}
EXPORT_SYMBOL_GPL(__blk_end_request_err);

J
Jens Axboe 已提交
2407 2408
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
		     struct bio *bio)
L
Linus Torvalds 已提交
2409
{
2410
	/* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
2411
	rq->cmd_flags |= bio->bi_rw & REQ_WRITE;
L
Linus Torvalds 已提交
2412

D
David Woodhouse 已提交
2413 2414 2415 2416
	if (bio_has_data(bio)) {
		rq->nr_phys_segments = bio_phys_segments(q, bio);
		rq->buffer = bio_data(bio);
	}
2417
	rq->__data_len = bio->bi_size;
L
Linus Torvalds 已提交
2418 2419
	rq->bio = rq->biotail = bio;

N
NeilBrown 已提交
2420 2421 2422
	if (bio->bi_bdev)
		rq->rq_disk = bio->bi_bdev->bd_disk;
}
L
Linus Torvalds 已提交
2423

2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
/**
 * rq_flush_dcache_pages - Helper function to flush all pages in a request
 * @rq: the request to be flushed
 *
 * Description:
 *     Flush all pages in @rq.
 */
void rq_flush_dcache_pages(struct request *rq)
{
	struct req_iterator iter;
	struct bio_vec *bvec;

	rq_for_each_segment(bvec, rq, iter)
		flush_dcache_page(bvec->bv_page);
}
EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
#endif

2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
/**
 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
 * @q : the queue of the device being checked
 *
 * Description:
 *    Check if underlying low-level drivers of a device are busy.
 *    If the drivers want to export their busy state, they must set own
 *    exporting function using blk_queue_lld_busy() first.
 *
 *    Basically, this function is used only by request stacking drivers
 *    to stop dispatching requests to underlying devices when underlying
 *    devices are busy.  This behavior helps more I/O merging on the queue
 *    of the request stacking driver and prevents I/O throughput regression
 *    on burst I/O load.
 *
 * Return:
 *    0 - Not busy (The request stacking driver should dispatch request)
 *    1 - Busy (The request stacking driver should stop dispatching request)
 */
int blk_lld_busy(struct request_queue *q)
{
	if (q->lld_busy_fn)
		return q->lld_busy_fn(q);

	return 0;
}
EXPORT_SYMBOL_GPL(blk_lld_busy);

2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497
/**
 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
 * @rq: the clone request to be cleaned up
 *
 * Description:
 *     Free all bios in @rq for a cloned request.
 */
void blk_rq_unprep_clone(struct request *rq)
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;

		bio_put(bio);
	}
}
EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);

/*
 * Copy attributes of the original request to the clone request.
 * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
 */
static void __blk_rq_prep_clone(struct request *dst, struct request *src)
{
	dst->cpu = src->cpu;
	dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE);
2498 2499
	if (src->cmd_flags & REQ_DISCARD)
		dst->cmd_flags |= REQ_DISCARD;
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546
	dst->cmd_type = src->cmd_type;
	dst->__sector = blk_rq_pos(src);
	dst->__data_len = blk_rq_bytes(src);
	dst->nr_phys_segments = src->nr_phys_segments;
	dst->ioprio = src->ioprio;
	dst->extra_len = src->extra_len;
}

/**
 * blk_rq_prep_clone - Helper function to setup clone request
 * @rq: the request to be setup
 * @rq_src: original request to be cloned
 * @bs: bio_set that bios for clone are allocated from
 * @gfp_mask: memory allocation mask for bio
 * @bio_ctr: setup function to be called for each clone bio.
 *           Returns %0 for success, non %0 for failure.
 * @data: private data to be passed to @bio_ctr
 *
 * Description:
 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
 *     The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
 *     are not copied, and copying such parts is the caller's responsibility.
 *     Also, pages which the original bios are pointing to are not copied
 *     and the cloned bios just point same pages.
 *     So cloned bios must be completed before original bios, which means
 *     the caller must complete @rq before @rq_src.
 */
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
		      struct bio_set *bs, gfp_t gfp_mask,
		      int (*bio_ctr)(struct bio *, struct bio *, void *),
		      void *data)
{
	struct bio *bio, *bio_src;

	if (!bs)
		bs = fs_bio_set;

	blk_rq_init(NULL, rq);

	__rq_for_each_bio(bio_src, rq_src) {
		bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs);
		if (!bio)
			goto free_and_out;

		__bio_clone(bio, bio_src);

		if (bio_integrity(bio_src) &&
2547
		    bio_integrity_clone(bio, bio_src, gfp_mask, bs))
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572
			goto free_and_out;

		if (bio_ctr && bio_ctr(bio, bio_src, data))
			goto free_and_out;

		if (rq->bio) {
			rq->biotail->bi_next = bio;
			rq->biotail = bio;
		} else
			rq->bio = rq->biotail = bio;
	}

	__blk_rq_prep_clone(rq, rq_src);

	return 0;

free_and_out:
	if (bio)
		bio_free(bio, bs);
	blk_rq_unprep_clone(rq);

	return -ENOMEM;
}
EXPORT_SYMBOL_GPL(blk_rq_prep_clone);

2573
int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
L
Linus Torvalds 已提交
2574 2575 2576 2577 2578 2579 2580
{
	return queue_work(kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work);

int __init blk_dev_init(void)
{
2581 2582 2583
	BUILD_BUG_ON(__REQ_NR_BITS > 8 *
			sizeof(((struct request *)0)->cmd_flags));

L
Linus Torvalds 已提交
2584 2585 2586 2587 2588
	kblockd_workqueue = create_workqueue("kblockd");
	if (!kblockd_workqueue)
		panic("Failed to create kblockd\n");

	request_cachep = kmem_cache_create("blkdev_requests",
2589
			sizeof(struct request), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
2590

2591
	blk_requestq_cachep = kmem_cache_create("blkdev_queue",
2592
			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
2593

2594
	return 0;
L
Linus Torvalds 已提交
2595
}