blk-core.c 114.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> -  July2000
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>	/* for max_pfn/max_low_pfn */
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
28
#include <linux/task_io_accounting_ops.h>
29 30
#include <linux/interrupt.h>
#include <linux/cpu.h>
31
#include <linux/blktrace_api.h>
32
#include <linux/fault-inject.h>
J
Jens Axboe 已提交
33
#include <linux/scatterlist.h>
L
Linus Torvalds 已提交
34 35 36 37 38 39

/*
 * for max sense size
 */
#include <scsi/scsi_cmnd.h>

40
static void blk_unplug_work(struct work_struct *work);
L
Linus Torvalds 已提交
41
static void blk_unplug_timeout(unsigned long data);
42
static void drive_stat_acct(struct request *rq, int new_io);
43
static void init_request_from_bio(struct request *req, struct bio *bio);
44
static int __make_request(struct request_queue *q, struct bio *bio);
45
static struct io_context *current_io_context(gfp_t gfp_flags, int node);
46
static void blk_recalc_rq_segments(struct request *rq);
N
NeilBrown 已提交
47 48
static void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
			    struct bio *bio);
L
Linus Torvalds 已提交
49 50 51 52

/*
 * For the allocated request tables
 */
53
static struct kmem_cache *request_cachep;
L
Linus Torvalds 已提交
54 55 56 57

/*
 * For queue allocation
 */
58
static struct kmem_cache *requestq_cachep;
L
Linus Torvalds 已提交
59 60 61 62

/*
 * For io context allocations
 */
63
static struct kmem_cache *iocontext_cachep;
L
Linus Torvalds 已提交
64 65 66 67

/*
 * Controlling structure to kblockd
 */
68
static struct workqueue_struct *kblockd_workqueue;
L
Linus Torvalds 已提交
69 70 71 72 73 74

unsigned long blk_max_low_pfn, blk_max_pfn;

EXPORT_SYMBOL(blk_max_low_pfn);
EXPORT_SYMBOL(blk_max_pfn);

75 76
static DEFINE_PER_CPU(struct list_head, blk_cpu_done);

L
Linus Torvalds 已提交
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
/* Amount of time in which a process may batch requests */
#define BLK_BATCH_TIME	(HZ/50UL)

/* Number of requests a "batching" process may submit */
#define BLK_BATCH_REQ	32

/*
 * Return the threshold (number of used requests) at which the queue is
 * considered to be congested.  It include a little hysteresis to keep the
 * context switch rate down.
 */
static inline int queue_congestion_on_threshold(struct request_queue *q)
{
	return q->nr_congestion_on;
}

/*
 * The threshold at which a queue is considered to be uncongested
 */
static inline int queue_congestion_off_threshold(struct request_queue *q)
{
	return q->nr_congestion_off;
}

static void blk_queue_congestion_threshold(struct request_queue *q)
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

/**
 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
 * @bdev:	device
 *
 * Locates the passed device's request queue and returns the address of its
 * backing_dev_info
 *
 * Will return NULL if the request queue cannot be located.
 */
struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
{
	struct backing_dev_info *ret = NULL;
128
	struct request_queue *q = bdev_get_queue(bdev);
L
Linus Torvalds 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146

	if (q)
		ret = &q->backing_dev_info;
	return ret;
}
EXPORT_SYMBOL(blk_get_backing_dev_info);

/**
 * blk_queue_prep_rq - set a prepare_request function for queue
 * @q:		queue
 * @pfn:	prepare_request function
 *
 * It's possible for a queue to register a prepare_request callback which
 * is invoked before the request is handed to the request_fn. The goal of
 * the function is to prepare a request for I/O, it can be used to build a
 * cdb from the request data for instance.
 *
 */
147
void blk_queue_prep_rq(struct request_queue *q, prep_rq_fn *pfn)
L
Linus Torvalds 已提交
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
{
	q->prep_rq_fn = pfn;
}

EXPORT_SYMBOL(blk_queue_prep_rq);

/**
 * blk_queue_merge_bvec - set a merge_bvec function for queue
 * @q:		queue
 * @mbfn:	merge_bvec_fn
 *
 * Usually queues have static limitations on the max sectors or segments that
 * we can put in a request. Stacking drivers may have some settings that
 * are dynamic, and thus we have to query the queue whether it is ok to
 * add a new bio_vec to a bio at a given offset or not. If the block device
 * has such limitations, it needs to register a merge_bvec_fn to control
 * the size of bio's sent to it. Note that a block device *must* allow a
 * single page to be added to an empty bio. The block device driver may want
 * to use the bio_split() function to deal with these bio's. By default
 * no merge_bvec_fn is defined for a queue, and only the fixed limits are
 * honored.
 */
170
void blk_queue_merge_bvec(struct request_queue *q, merge_bvec_fn *mbfn)
L
Linus Torvalds 已提交
171 172 173 174 175 176
{
	q->merge_bvec_fn = mbfn;
}

EXPORT_SYMBOL(blk_queue_merge_bvec);

177
void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn)
178 179 180 181 182 183
{
	q->softirq_done_fn = fn;
}

EXPORT_SYMBOL(blk_queue_softirq_done);

L
Linus Torvalds 已提交
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/**
 * blk_queue_make_request - define an alternate make_request function for a device
 * @q:  the request queue for the device to be affected
 * @mfn: the alternate make_request function
 *
 * Description:
 *    The normal way for &struct bios to be passed to a device
 *    driver is for them to be collected into requests on a request
 *    queue, and then to allow the device driver to select requests
 *    off that queue when it is ready.  This works well for many block
 *    devices. However some block devices (typically virtual devices
 *    such as md or lvm) do not benefit from the processing on the
 *    request queue, and are served best by having the requests passed
 *    directly to them.  This can be achieved by providing a function
 *    to blk_queue_make_request().
 *
 * Caveat:
 *    The driver that does this *must* be able to deal appropriately
 *    with buffers in "highmemory". This can be accomplished by either calling
 *    __bio_kmap_atomic() to get a temporary kernel mapping, or by calling
 *    blk_queue_bounce() to create a buffer in normal memory.
 **/
206
void blk_queue_make_request(struct request_queue * q, make_request_fn * mfn)
L
Linus Torvalds 已提交
207 208 209 210 211
{
	/*
	 * set defaults
	 */
	q->nr_requests = BLKDEV_MAX_RQ;
212 213
	blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);
	blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
L
Linus Torvalds 已提交
214 215 216 217
	q->make_request_fn = mfn;
	q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
	q->backing_dev_info.state = 0;
	q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
218
	blk_queue_max_sectors(q, SAFE_MAX_SECTORS);
L
Linus Torvalds 已提交
219 220 221 222 223 224 225 226 227 228
	blk_queue_hardsect_size(q, 512);
	blk_queue_dma_alignment(q, 511);
	blk_queue_congestion_threshold(q);
	q->nr_batching = BLK_BATCH_REQ;

	q->unplug_thresh = 4;		/* hmm */
	q->unplug_delay = (3 * HZ) / 1000;	/* 3 milliseconds */
	if (q->unplug_delay == 0)
		q->unplug_delay = 1;

229
	INIT_WORK(&q->unplug_work, blk_unplug_work);
L
Linus Torvalds 已提交
230 231 232 233 234 235 236 237 238 239 240 241

	q->unplug_timer.function = blk_unplug_timeout;
	q->unplug_timer.data = (unsigned long)q;

	/*
	 * by default assume old behaviour and bounce for any highmem page
	 */
	blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
}

EXPORT_SYMBOL(blk_queue_make_request);

242
static void rq_init(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
243 244
{
	INIT_LIST_HEAD(&rq->queuelist);
245
	INIT_LIST_HEAD(&rq->donelist);
L
Linus Torvalds 已提交
246 247 248

	rq->errors = 0;
	rq->bio = rq->biotail = NULL;
249 250
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
251
	rq->ioprio = 0;
L
Linus Torvalds 已提交
252 253 254 255 256 257
	rq->buffer = NULL;
	rq->ref_count = 1;
	rq->q = q;
	rq->special = NULL;
	rq->data_len = 0;
	rq->data = NULL;
M
Mike Christie 已提交
258
	rq->nr_phys_segments = 0;
L
Linus Torvalds 已提交
259 260 261
	rq->sense = NULL;
	rq->end_io = NULL;
	rq->end_io_data = NULL;
262
	rq->completion_data = NULL;
263
	rq->next_rq = NULL;
L
Linus Torvalds 已提交
264 265 266 267
}

/**
 * blk_queue_ordered - does this queue support ordered writes
268 269
 * @q:        the request queue
 * @ordered:  one of QUEUE_ORDERED_*
J
Jens Axboe 已提交
270
 * @prepare_flush_fn: rq setup helper for cache flush ordered writes
L
Linus Torvalds 已提交
271 272 273 274 275 276 277 278
 *
 * Description:
 *   For journalled file systems, doing ordered writes on a commit
 *   block instead of explicitly doing wait_on_buffer (which is bad
 *   for performance) can be a big win. Block drivers supporting this
 *   feature should call this function and indicate so.
 *
 **/
279
int blk_queue_ordered(struct request_queue *q, unsigned ordered,
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
		      prepare_flush_fn *prepare_flush_fn)
{
	if (ordered & (QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_POSTFLUSH) &&
	    prepare_flush_fn == NULL) {
		printk(KERN_ERR "blk_queue_ordered: prepare_flush_fn required\n");
		return -EINVAL;
	}

	if (ordered != QUEUE_ORDERED_NONE &&
	    ordered != QUEUE_ORDERED_DRAIN &&
	    ordered != QUEUE_ORDERED_DRAIN_FLUSH &&
	    ordered != QUEUE_ORDERED_DRAIN_FUA &&
	    ordered != QUEUE_ORDERED_TAG &&
	    ordered != QUEUE_ORDERED_TAG_FLUSH &&
	    ordered != QUEUE_ORDERED_TAG_FUA) {
		printk(KERN_ERR "blk_queue_ordered: bad value %d\n", ordered);
		return -EINVAL;
L
Linus Torvalds 已提交
297
	}
298

299
	q->ordered = ordered;
300 301 302 303
	q->next_ordered = ordered;
	q->prepare_flush_fn = prepare_flush_fn;

	return 0;
L
Linus Torvalds 已提交
304 305 306 307 308 309 310
}

EXPORT_SYMBOL(blk_queue_ordered);

/*
 * Cache flushing for ordered writes handling
 */
311
inline unsigned blk_ordered_cur_seq(struct request_queue *q)
L
Linus Torvalds 已提交
312
{
313 314 315
	if (!q->ordseq)
		return 0;
	return 1 << ffz(q->ordseq);
L
Linus Torvalds 已提交
316 317
}

318
unsigned blk_ordered_req_seq(struct request *rq)
L
Linus Torvalds 已提交
319
{
320
	struct request_queue *q = rq->q;
L
Linus Torvalds 已提交
321

322
	BUG_ON(q->ordseq == 0);
323

324 325 326 327 328 329
	if (rq == &q->pre_flush_rq)
		return QUEUE_ORDSEQ_PREFLUSH;
	if (rq == &q->bar_rq)
		return QUEUE_ORDSEQ_BAR;
	if (rq == &q->post_flush_rq)
		return QUEUE_ORDSEQ_POSTFLUSH;
L
Linus Torvalds 已提交
330

331 332 333 334 335 336 337 338 339
	/*
	 * !fs requests don't need to follow barrier ordering.  Always
	 * put them at the front.  This fixes the following deadlock.
	 *
	 * http://thread.gmane.org/gmane.linux.kernel/537473
	 */
	if (!blk_fs_request(rq))
		return QUEUE_ORDSEQ_DRAIN;

340 341
	if ((rq->cmd_flags & REQ_ORDERED_COLOR) ==
	    (q->orig_bar_rq->cmd_flags & REQ_ORDERED_COLOR))
342 343 344
		return QUEUE_ORDSEQ_DRAIN;
	else
		return QUEUE_ORDSEQ_DONE;
L
Linus Torvalds 已提交
345 346
}

347
void blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error)
L
Linus Torvalds 已提交
348
{
349
	struct request *rq;
L
Linus Torvalds 已提交
350

351 352
	if (error && !q->orderr)
		q->orderr = error;
L
Linus Torvalds 已提交
353

354 355
	BUG_ON(q->ordseq & seq);
	q->ordseq |= seq;
L
Linus Torvalds 已提交
356

357 358
	if (blk_ordered_cur_seq(q) != QUEUE_ORDSEQ_DONE)
		return;
L
Linus Torvalds 已提交
359 360

	/*
361
	 * Okay, sequence complete.
L
Linus Torvalds 已提交
362
	 */
363
	q->ordseq = 0;
J
Jens Axboe 已提交
364
	rq = q->orig_bar_rq;
L
Linus Torvalds 已提交
365

366 367
	if (__blk_end_request(rq, q->orderr, blk_rq_bytes(rq)))
		BUG();
L
Linus Torvalds 已提交
368 369
}

370
static void pre_flush_end_io(struct request *rq, int error)
L
Linus Torvalds 已提交
371
{
372 373 374
	elv_completed_request(rq->q, rq);
	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_PREFLUSH, error);
}
L
Linus Torvalds 已提交
375

376 377 378 379 380
static void bar_end_io(struct request *rq, int error)
{
	elv_completed_request(rq->q, rq);
	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_BAR, error);
}
L
Linus Torvalds 已提交
381

382 383 384 385 386
static void post_flush_end_io(struct request *rq, int error)
{
	elv_completed_request(rq->q, rq);
	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_POSTFLUSH, error);
}
L
Linus Torvalds 已提交
387

388
static void queue_flush(struct request_queue *q, unsigned which)
389 390 391
{
	struct request *rq;
	rq_end_io_fn *end_io;
L
Linus Torvalds 已提交
392

393 394 395 396 397 398
	if (which == QUEUE_ORDERED_PREFLUSH) {
		rq = &q->pre_flush_rq;
		end_io = pre_flush_end_io;
	} else {
		rq = &q->post_flush_rq;
		end_io = post_flush_end_io;
L
Linus Torvalds 已提交
399
	}
400

401
	rq->cmd_flags = REQ_HARDBARRIER;
402 403
	rq_init(q, rq);
	rq->elevator_private = NULL;
404
	rq->elevator_private2 = NULL;
405 406 407 408
	rq->rq_disk = q->bar_rq.rq_disk;
	rq->end_io = end_io;
	q->prepare_flush_fn(q, rq);

409
	elv_insert(q, rq, ELEVATOR_INSERT_FRONT);
L
Linus Torvalds 已提交
410 411
}

412
static inline struct request *start_ordered(struct request_queue *q,
413
					    struct request *rq)
L
Linus Torvalds 已提交
414
{
415 416 417 418 419 420 421 422 423 424
	q->orderr = 0;
	q->ordered = q->next_ordered;
	q->ordseq |= QUEUE_ORDSEQ_STARTED;

	/*
	 * Prep proxy barrier request.
	 */
	blkdev_dequeue_request(rq);
	q->orig_bar_rq = rq;
	rq = &q->bar_rq;
425
	rq->cmd_flags = 0;
426
	rq_init(q, rq);
427 428
	if (bio_data_dir(q->orig_bar_rq->bio) == WRITE)
		rq->cmd_flags |= REQ_RW;
J
Jens Axboe 已提交
429 430
	if (q->ordered & QUEUE_ORDERED_FUA)
		rq->cmd_flags |= REQ_FUA;
431
	rq->elevator_private = NULL;
432
	rq->elevator_private2 = NULL;
433 434 435 436 437 438 439
	init_request_from_bio(rq, q->orig_bar_rq->bio);
	rq->end_io = bar_end_io;

	/*
	 * Queue ordered sequence.  As we stack them at the head, we
	 * need to queue in reverse order.  Note that we rely on that
	 * no fs request uses ELEVATOR_INSERT_FRONT and thus no fs
440 441 442 443
	 * request gets inbetween ordered sequence. If this request is
	 * an empty barrier, we don't need to do a postflush ever since
	 * there will be no data written between the pre and post flush.
	 * Hence a single flush will suffice.
444
	 */
445
	if ((q->ordered & QUEUE_ORDERED_POSTFLUSH) && !blk_empty_barrier(rq))
446 447 448 449
		queue_flush(q, QUEUE_ORDERED_POSTFLUSH);
	else
		q->ordseq |= QUEUE_ORDSEQ_POSTFLUSH;

450
	elv_insert(q, rq, ELEVATOR_INSERT_FRONT);
451 452 453 454 455 456

	if (q->ordered & QUEUE_ORDERED_PREFLUSH) {
		queue_flush(q, QUEUE_ORDERED_PREFLUSH);
		rq = &q->pre_flush_rq;
	} else
		q->ordseq |= QUEUE_ORDSEQ_PREFLUSH;
L
Linus Torvalds 已提交
457

458 459 460 461 462 463
	if ((q->ordered & QUEUE_ORDERED_TAG) || q->in_flight == 0)
		q->ordseq |= QUEUE_ORDSEQ_DRAIN;
	else
		rq = NULL;

	return rq;
L
Linus Torvalds 已提交
464 465
}

466
int blk_do_ordered(struct request_queue *q, struct request **rqp)
L
Linus Torvalds 已提交
467
{
468
	struct request *rq = *rqp;
469
	const int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq);
L
Linus Torvalds 已提交
470

471 472 473
	if (!q->ordseq) {
		if (!is_barrier)
			return 1;
L
Linus Torvalds 已提交
474

475 476 477 478 479 480 481 482 483
		if (q->next_ordered != QUEUE_ORDERED_NONE) {
			*rqp = start_ordered(q, rq);
			return 1;
		} else {
			/*
			 * This can happen when the queue switches to
			 * ORDERED_NONE while this request is on it.
			 */
			blkdev_dequeue_request(rq);
484 485 486
			if (__blk_end_request(rq, -EOPNOTSUPP,
					      blk_rq_bytes(rq)))
				BUG();
487 488 489 490
			*rqp = NULL;
			return 0;
		}
	}
L
Linus Torvalds 已提交
491

492 493 494 495 496 497 498 499 500
	/*
	 * Ordered sequence in progress
	 */

	/* Special requests are not subject to ordering rules. */
	if (!blk_fs_request(rq) &&
	    rq != &q->pre_flush_rq && rq != &q->post_flush_rq)
		return 1;

501
	if (q->ordered & QUEUE_ORDERED_TAG) {
502
		/* Ordered by tag.  Blocking the next barrier is enough. */
503 504
		if (is_barrier && rq != &q->bar_rq)
			*rqp = NULL;
505 506 507 508 509
	} else {
		/* Ordered by draining.  Wait for turn. */
		WARN_ON(blk_ordered_req_seq(rq) < blk_ordered_cur_seq(q));
		if (blk_ordered_req_seq(rq) > blk_ordered_cur_seq(q))
			*rqp = NULL;
L
Linus Torvalds 已提交
510 511 512 513 514
	}

	return 1;
}

N
NeilBrown 已提交
515 516
static void req_bio_endio(struct request *rq, struct bio *bio,
			  unsigned int nbytes, int error)
L
Linus Torvalds 已提交
517
{
518
	struct request_queue *q = rq->q;
519

N
NeilBrown 已提交
520 521 522 523 524
	if (&q->bar_rq != rq) {
		if (error)
			clear_bit(BIO_UPTODATE, &bio->bi_flags);
		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
			error = -EIO;
525

N
NeilBrown 已提交
526 527 528 529 530
		if (unlikely(nbytes > bio->bi_size)) {
			printk("%s: want %u bytes done, only %u left\n",
			       __FUNCTION__, nbytes, bio->bi_size);
			nbytes = bio->bi_size;
		}
531

N
NeilBrown 已提交
532 533 534
		bio->bi_size -= nbytes;
		bio->bi_sector += (nbytes >> 9);
		if (bio->bi_size == 0)
535
			bio_endio(bio, error);
N
NeilBrown 已提交
536 537 538 539 540 541 542 543 544
	} else {

		/*
		 * Okay, this is the barrier request in progress, just
		 * record the error;
		 */
		if (error && !q->orderr)
			q->orderr = error;
	}
L
Linus Torvalds 已提交
545 546 547 548 549 550 551 552 553 554 555
}

/**
 * blk_queue_bounce_limit - set bounce buffer limit for queue
 * @q:  the request queue for the device
 * @dma_addr:   bus address limit
 *
 * Description:
 *    Different hardware can have different requirements as to what pages
 *    it can do I/O directly to. A low level driver can call
 *    blk_queue_bounce_limit to have lower memory pages allocated as bounce
556
 *    buffers for doing I/O to pages residing above @page.
L
Linus Torvalds 已提交
557
 **/
558
void blk_queue_bounce_limit(struct request_queue *q, u64 dma_addr)
L
Linus Torvalds 已提交
559 560
{
	unsigned long bounce_pfn = dma_addr >> PAGE_SHIFT;
561 562 563 564 565 566 567
	int dma = 0;

	q->bounce_gfp = GFP_NOIO;
#if BITS_PER_LONG == 64
	/* Assume anything <= 4GB can be handled by IOMMU.
	   Actually some IOMMUs can handle everything, but I don't
	   know of a way to test this here. */
568
	if (bounce_pfn < (min_t(u64,0xffffffff,BLK_BOUNCE_HIGH) >> PAGE_SHIFT))
569 570 571 572 573 574 575 576
		dma = 1;
	q->bounce_pfn = max_low_pfn;
#else
	if (bounce_pfn < blk_max_low_pfn)
		dma = 1;
	q->bounce_pfn = bounce_pfn;
#endif
	if (dma) {
L
Linus Torvalds 已提交
577 578
		init_emergency_isa_pool();
		q->bounce_gfp = GFP_NOIO | GFP_DMA;
579 580
		q->bounce_pfn = bounce_pfn;
	}
L
Linus Torvalds 已提交
581 582 583 584 585 586 587 588 589 590 591 592 593
}

EXPORT_SYMBOL(blk_queue_bounce_limit);

/**
 * blk_queue_max_sectors - set max sectors for a request for this queue
 * @q:  the request queue for the device
 * @max_sectors:  max sectors in the usual 512b unit
 *
 * Description:
 *    Enables a low level driver to set an upper limit on the size of
 *    received requests.
 **/
594
void blk_queue_max_sectors(struct request_queue *q, unsigned int max_sectors)
L
Linus Torvalds 已提交
595 596 597 598 599 600
{
	if ((max_sectors << 9) < PAGE_CACHE_SIZE) {
		max_sectors = 1 << (PAGE_CACHE_SHIFT - 9);
		printk("%s: set to minimum %d\n", __FUNCTION__, max_sectors);
	}

601 602 603 604 605 606
	if (BLK_DEF_MAX_SECTORS > max_sectors)
		q->max_hw_sectors = q->max_sectors = max_sectors;
 	else {
		q->max_sectors = BLK_DEF_MAX_SECTORS;
		q->max_hw_sectors = max_sectors;
	}
L
Linus Torvalds 已提交
607 608 609 610 611 612 613 614 615 616 617 618 619 620
}

EXPORT_SYMBOL(blk_queue_max_sectors);

/**
 * blk_queue_max_phys_segments - set max phys segments for a request for this queue
 * @q:  the request queue for the device
 * @max_segments:  max number of segments
 *
 * Description:
 *    Enables a low level driver to set an upper limit on the number of
 *    physical data segments in a request.  This would be the largest sized
 *    scatter list the driver could handle.
 **/
621 622
void blk_queue_max_phys_segments(struct request_queue *q,
				 unsigned short max_segments)
L
Linus Torvalds 已提交
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
{
	if (!max_segments) {
		max_segments = 1;
		printk("%s: set to minimum %d\n", __FUNCTION__, max_segments);
	}

	q->max_phys_segments = max_segments;
}

EXPORT_SYMBOL(blk_queue_max_phys_segments);

/**
 * blk_queue_max_hw_segments - set max hw segments for a request for this queue
 * @q:  the request queue for the device
 * @max_segments:  max number of segments
 *
 * Description:
 *    Enables a low level driver to set an upper limit on the number of
 *    hw data segments in a request.  This would be the largest number of
 *    address/length pairs the host adapter can actually give as once
 *    to the device.
 **/
645 646
void blk_queue_max_hw_segments(struct request_queue *q,
			       unsigned short max_segments)
L
Linus Torvalds 已提交
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
{
	if (!max_segments) {
		max_segments = 1;
		printk("%s: set to minimum %d\n", __FUNCTION__, max_segments);
	}

	q->max_hw_segments = max_segments;
}

EXPORT_SYMBOL(blk_queue_max_hw_segments);

/**
 * blk_queue_max_segment_size - set max segment size for blk_rq_map_sg
 * @q:  the request queue for the device
 * @max_size:  max size of segment in bytes
 *
 * Description:
 *    Enables a low level driver to set an upper limit on the size of a
 *    coalesced segment
 **/
667
void blk_queue_max_segment_size(struct request_queue *q, unsigned int max_size)
L
Linus Torvalds 已提交
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
{
	if (max_size < PAGE_CACHE_SIZE) {
		max_size = PAGE_CACHE_SIZE;
		printk("%s: set to minimum %d\n", __FUNCTION__, max_size);
	}

	q->max_segment_size = max_size;
}

EXPORT_SYMBOL(blk_queue_max_segment_size);

/**
 * blk_queue_hardsect_size - set hardware sector size for the queue
 * @q:  the request queue for the device
 * @size:  the hardware sector size, in bytes
 *
 * Description:
 *   This should typically be set to the lowest possible sector size
 *   that the hardware can operate on (possible without reverting to
 *   even internal read-modify-write operations). Usually the default
 *   of 512 covers most hardware.
 **/
690
void blk_queue_hardsect_size(struct request_queue *q, unsigned short size)
L
Linus Torvalds 已提交
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
{
	q->hardsect_size = size;
}

EXPORT_SYMBOL(blk_queue_hardsect_size);

/*
 * Returns the minimum that is _not_ zero, unless both are zero.
 */
#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))

/**
 * blk_queue_stack_limits - inherit underlying queue limits for stacked drivers
 * @t:	the stacking driver (top)
 * @b:  the underlying device (bottom)
 **/
707
void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b)
L
Linus Torvalds 已提交
708 709
{
	/* zero is "infinity" */
710 711
	t->max_sectors = min_not_zero(t->max_sectors,b->max_sectors);
	t->max_hw_sectors = min_not_zero(t->max_hw_sectors,b->max_hw_sectors);
L
Linus Torvalds 已提交
712 713 714 715 716

	t->max_phys_segments = min(t->max_phys_segments,b->max_phys_segments);
	t->max_hw_segments = min(t->max_hw_segments,b->max_hw_segments);
	t->max_segment_size = min(t->max_segment_size,b->max_segment_size);
	t->hardsect_size = max(t->hardsect_size,b->hardsect_size);
717 718
	if (!test_bit(QUEUE_FLAG_CLUSTER, &b->queue_flags))
		clear_bit(QUEUE_FLAG_CLUSTER, &t->queue_flags);
L
Linus Torvalds 已提交
719 720 721 722
}

EXPORT_SYMBOL(blk_queue_stack_limits);

J
James Bottomley 已提交
723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
/**
 * blk_queue_dma_drain - Set up a drain buffer for excess dma.
 *
 * @q:  the request queue for the device
 * @buf:	physically contiguous buffer
 * @size:	size of the buffer in bytes
 *
 * Some devices have excess DMA problems and can't simply discard (or
 * zero fill) the unwanted piece of the transfer.  They have to have a
 * real area of memory to transfer it into.  The use case for this is
 * ATAPI devices in DMA mode.  If the packet command causes a transfer
 * bigger than the transfer size some HBAs will lock up if there
 * aren't DMA elements to contain the excess transfer.  What this API
 * does is adjust the queue so that the buf is always appended
 * silently to the scatterlist.
 *
 * Note: This routine adjusts max_hw_segments to make room for
 * appending the drain buffer.  If you call
 * blk_queue_max_hw_segments() or blk_queue_max_phys_segments() after
 * calling this routine, you must set the limit to one fewer than your
 * device can support otherwise there won't be room for the drain
 * buffer.
 */
int blk_queue_dma_drain(struct request_queue *q, void *buf,
				unsigned int size)
{
	if (q->max_hw_segments < 2 || q->max_phys_segments < 2)
		return -EINVAL;
	/* make room for appending the drain */
	--q->max_hw_segments;
	--q->max_phys_segments;
	q->dma_drain_buffer = buf;
	q->dma_drain_size = size;

	return 0;
}

EXPORT_SYMBOL_GPL(blk_queue_dma_drain);

L
Linus Torvalds 已提交
762 763 764 765 766
/**
 * blk_queue_segment_boundary - set boundary rules for segment merging
 * @q:  the request queue for the device
 * @mask:  the memory boundary mask
 **/
767
void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask)
L
Linus Torvalds 已提交
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
{
	if (mask < PAGE_CACHE_SIZE - 1) {
		mask = PAGE_CACHE_SIZE - 1;
		printk("%s: set to minimum %lx\n", __FUNCTION__, mask);
	}

	q->seg_boundary_mask = mask;
}

EXPORT_SYMBOL(blk_queue_segment_boundary);

/**
 * blk_queue_dma_alignment - set dma length and memory alignment
 * @q:     the request queue for the device
 * @mask:  alignment mask
 *
 * description:
 *    set required memory and length aligment for direct dma transactions.
 *    this is used when buiding direct io requests for the queue.
 *
 **/
789
void blk_queue_dma_alignment(struct request_queue *q, int mask)
L
Linus Torvalds 已提交
790 791 792 793 794 795
{
	q->dma_alignment = mask;
}

EXPORT_SYMBOL(blk_queue_dma_alignment);

796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
/**
 * blk_queue_update_dma_alignment - update dma length and memory alignment
 * @q:     the request queue for the device
 * @mask:  alignment mask
 *
 * description:
 *    update required memory and length aligment for direct dma transactions.
 *    If the requested alignment is larger than the current alignment, then
 *    the current queue alignment is updated to the new value, otherwise it
 *    is left alone.  The design of this is to allow multiple objects
 *    (driver, device, transport etc) to set their respective
 *    alignments without having them interfere.
 *
 **/
void blk_queue_update_dma_alignment(struct request_queue *q, int mask)
{
	BUG_ON(mask > PAGE_SIZE);

	if (mask > q->dma_alignment)
		q->dma_alignment = mask;
}

EXPORT_SYMBOL(blk_queue_update_dma_alignment);

L
Linus Torvalds 已提交
820 821 822 823 824 825 826 827 828 829 830
/**
 * blk_queue_find_tag - find a request by its tag and queue
 * @q:	 The request queue for the device
 * @tag: The tag of the request
 *
 * Notes:
 *    Should be used when a device returns a tag and you want to match
 *    it with a request.
 *
 *    no locks need be held.
 **/
831
struct request *blk_queue_find_tag(struct request_queue *q, int tag)
L
Linus Torvalds 已提交
832
{
833
	return blk_map_queue_find_tag(q->queue_tags, tag);
L
Linus Torvalds 已提交
834 835 836 837 838
}

EXPORT_SYMBOL(blk_queue_find_tag);

/**
839 840
 * __blk_free_tags - release a given set of tag maintenance info
 * @bqt:	the tag map to free
L
Linus Torvalds 已提交
841
 *
842 843 844 845
 * Tries to free the specified @bqt@.  Returns true if it was
 * actually freed and false if there are still references using it
 */
static int __blk_free_tags(struct blk_queue_tag *bqt)
L
Linus Torvalds 已提交
846
{
847
	int retval;
L
Linus Torvalds 已提交
848

849 850
	retval = atomic_dec_and_test(&bqt->refcnt);
	if (retval) {
L
Linus Torvalds 已提交
851 852 853 854 855 856 857 858 859
		BUG_ON(bqt->busy);

		kfree(bqt->tag_index);
		bqt->tag_index = NULL;

		kfree(bqt->tag_map);
		bqt->tag_map = NULL;

		kfree(bqt);
860

L
Linus Torvalds 已提交
861 862
	}

863 864 865 866 867 868 869 870 871 872 873
	return retval;
}

/**
 * __blk_queue_free_tags - release tag maintenance info
 * @q:  the request queue for the device
 *
 *  Notes:
 *    blk_cleanup_queue() will take care of calling this function, if tagging
 *    has been used. So there's no need to call this directly.
 **/
874
static void __blk_queue_free_tags(struct request_queue *q)
875 876 877 878 879 880 881 882
{
	struct blk_queue_tag *bqt = q->queue_tags;

	if (!bqt)
		return;

	__blk_free_tags(bqt);

L
Linus Torvalds 已提交
883 884 885 886
	q->queue_tags = NULL;
	q->queue_flags &= ~(1 << QUEUE_FLAG_QUEUED);
}

887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902

/**
 * blk_free_tags - release a given set of tag maintenance info
 * @bqt:	the tag map to free
 *
 * For externally managed @bqt@ frees the map.  Callers of this
 * function must guarantee to have released all the queues that
 * might have been using this tag map.
 */
void blk_free_tags(struct blk_queue_tag *bqt)
{
	if (unlikely(!__blk_free_tags(bqt)))
		BUG();
}
EXPORT_SYMBOL(blk_free_tags);

L
Linus Torvalds 已提交
903 904 905 906 907 908 909 910
/**
 * blk_queue_free_tags - release tag maintenance info
 * @q:  the request queue for the device
 *
 *  Notes:
 *	This is used to disabled tagged queuing to a device, yet leave
 *	queue in function.
 **/
911
void blk_queue_free_tags(struct request_queue *q)
L
Linus Torvalds 已提交
912 913 914 915 916 917 918
{
	clear_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
}

EXPORT_SYMBOL(blk_queue_free_tags);

static int
919
init_tag_map(struct request_queue *q, struct blk_queue_tag *tags, int depth)
L
Linus Torvalds 已提交
920 921 922
{
	struct request **tag_index;
	unsigned long *tag_map;
923
	int nr_ulongs;
L
Linus Torvalds 已提交
924

925
	if (q && depth > q->nr_requests * 2) {
L
Linus Torvalds 已提交
926 927 928 929 930
		depth = q->nr_requests * 2;
		printk(KERN_ERR "%s: adjusted depth to %d\n",
				__FUNCTION__, depth);
	}

931
	tag_index = kzalloc(depth * sizeof(struct request *), GFP_ATOMIC);
L
Linus Torvalds 已提交
932 933 934
	if (!tag_index)
		goto fail;

935
	nr_ulongs = ALIGN(depth, BITS_PER_LONG) / BITS_PER_LONG;
936
	tag_map = kzalloc(nr_ulongs * sizeof(unsigned long), GFP_ATOMIC);
L
Linus Torvalds 已提交
937 938 939
	if (!tag_map)
		goto fail;

940
	tags->real_max_depth = depth;
L
Linus Torvalds 已提交
941 942 943 944 945 946 947 948 949 950
	tags->max_depth = depth;
	tags->tag_index = tag_index;
	tags->tag_map = tag_map;

	return 0;
fail:
	kfree(tag_index);
	return -ENOMEM;
}

951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
						   int depth)
{
	struct blk_queue_tag *tags;

	tags = kmalloc(sizeof(struct blk_queue_tag), GFP_ATOMIC);
	if (!tags)
		goto fail;

	if (init_tag_map(q, tags, depth))
		goto fail;

	tags->busy = 0;
	atomic_set(&tags->refcnt, 1);
	return tags;
fail:
	kfree(tags);
	return NULL;
}

/**
 * blk_init_tags - initialize the tag info for an external tag map
 * @depth:	the maximum queue depth supported
 * @tags: the tag to use
 **/
struct blk_queue_tag *blk_init_tags(int depth)
{
	return __blk_queue_init_tags(NULL, depth);
}
EXPORT_SYMBOL(blk_init_tags);

L
Linus Torvalds 已提交
982 983 984 985 986 987
/**
 * blk_queue_init_tags - initialize the queue tag info
 * @q:  the request queue for the device
 * @depth:  the maximum queue depth supported
 * @tags: the tag to use
 **/
988
int blk_queue_init_tags(struct request_queue *q, int depth,
L
Linus Torvalds 已提交
989 990 991 992 993 994 995
			struct blk_queue_tag *tags)
{
	int rc;

	BUG_ON(tags && q->queue_tags && tags != q->queue_tags);

	if (!tags && !q->queue_tags) {
996
		tags = __blk_queue_init_tags(q, depth);
L
Linus Torvalds 已提交
997

998
		if (!tags)
L
Linus Torvalds 已提交
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
			goto fail;
	} else if (q->queue_tags) {
		if ((rc = blk_queue_resize_tags(q, depth)))
			return rc;
		set_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
		return 0;
	} else
		atomic_inc(&tags->refcnt);

	/*
	 * assign it, all done
	 */
	q->queue_tags = tags;
	q->queue_flags |= (1 << QUEUE_FLAG_QUEUED);
1013
	INIT_LIST_HEAD(&q->tag_busy_list);
L
Linus Torvalds 已提交
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
	return 0;
fail:
	kfree(tags);
	return -ENOMEM;
}

EXPORT_SYMBOL(blk_queue_init_tags);

/**
 * blk_queue_resize_tags - change the queueing depth
 * @q:  the request queue for the device
 * @new_depth: the new max command queueing depth
 *
 *  Notes:
 *    Must be called with the queue lock held.
 **/
1030
int blk_queue_resize_tags(struct request_queue *q, int new_depth)
L
Linus Torvalds 已提交
1031 1032 1033 1034
{
	struct blk_queue_tag *bqt = q->queue_tags;
	struct request **tag_index;
	unsigned long *tag_map;
1035
	int max_depth, nr_ulongs;
L
Linus Torvalds 已提交
1036 1037 1038 1039

	if (!bqt)
		return -ENXIO;

1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
	/*
	 * if we already have large enough real_max_depth.  just
	 * adjust max_depth.  *NOTE* as requests with tag value
	 * between new_depth and real_max_depth can be in-flight, tag
	 * map can not be shrunk blindly here.
	 */
	if (new_depth <= bqt->real_max_depth) {
		bqt->max_depth = new_depth;
		return 0;
	}

1051 1052 1053 1054 1055 1056 1057
	/*
	 * Currently cannot replace a shared tag map with a new
	 * one, so error out if this is the case
	 */
	if (atomic_read(&bqt->refcnt) != 1)
		return -EBUSY;

L
Linus Torvalds 已提交
1058 1059 1060 1061 1062
	/*
	 * save the old state info, so we can copy it back
	 */
	tag_index = bqt->tag_index;
	tag_map = bqt->tag_map;
1063
	max_depth = bqt->real_max_depth;
L
Linus Torvalds 已提交
1064 1065 1066 1067 1068

	if (init_tag_map(q, bqt, new_depth))
		return -ENOMEM;

	memcpy(bqt->tag_index, tag_index, max_depth * sizeof(struct request *));
1069
	nr_ulongs = ALIGN(max_depth, BITS_PER_LONG) / BITS_PER_LONG;
1070
	memcpy(bqt->tag_map, tag_map, nr_ulongs * sizeof(unsigned long));
L
Linus Torvalds 已提交
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092

	kfree(tag_index);
	kfree(tag_map);
	return 0;
}

EXPORT_SYMBOL(blk_queue_resize_tags);

/**
 * blk_queue_end_tag - end tag operations for a request
 * @q:  the request queue for the device
 * @rq: the request that has completed
 *
 *  Description:
 *    Typically called when end_that_request_first() returns 0, meaning
 *    all transfers have been done for a request. It's important to call
 *    this function before end_that_request_last(), as that will put the
 *    request back on the free list thus corrupting the internal tag list.
 *
 *  Notes:
 *   queue lock must be held.
 **/
1093
void blk_queue_end_tag(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
1094 1095 1096 1097 1098 1099
{
	struct blk_queue_tag *bqt = q->queue_tags;
	int tag = rq->tag;

	BUG_ON(tag == -1);

1100
	if (unlikely(tag >= bqt->real_max_depth))
1101 1102 1103 1104
		/*
		 * This can happen after tag depth has been reduced.
		 * FIXME: how about a warning or info message here?
		 */
L
Linus Torvalds 已提交
1105 1106 1107
		return;

	list_del_init(&rq->queuelist);
1108
	rq->cmd_flags &= ~REQ_QUEUED;
L
Linus Torvalds 已提交
1109 1110 1111
	rq->tag = -1;

	if (unlikely(bqt->tag_index[tag] == NULL))
1112 1113
		printk(KERN_ERR "%s: tag %d is missing\n",
		       __FUNCTION__, tag);
L
Linus Torvalds 已提交
1114 1115

	bqt->tag_index[tag] = NULL;
J
Jens Axboe 已提交
1116

1117
	if (unlikely(!test_bit(tag, bqt->tag_map))) {
J
Jens Axboe 已提交
1118 1119 1120 1121
		printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
		       __FUNCTION__, tag);
		return;
	}
1122 1123 1124 1125 1126
	/*
	 * The tag_map bit acts as a lock for tag_index[bit], so we need
	 * unlock memory barrier semantics.
	 */
	clear_bit_unlock(tag, bqt->tag_map);
L
Linus Torvalds 已提交
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
	bqt->busy--;
}

EXPORT_SYMBOL(blk_queue_end_tag);

/**
 * blk_queue_start_tag - find a free tag and assign it
 * @q:  the request queue for the device
 * @rq:  the block request that needs tagging
 *
 *  Description:
 *    This can either be used as a stand-alone helper, or possibly be
 *    assigned as the queue &prep_rq_fn (in which case &struct request
 *    automagically gets a tag assigned). Note that this function
 *    assumes that any type of request can be queued! if this is not
 *    true for your device, you must check the request type before
 *    calling this function.  The request will also be removed from
 *    the request queue, so it's the drivers responsibility to readd
 *    it if it should need to be restarted for some reason.
 *
 *  Notes:
 *   queue lock must be held.
 **/
1150
int blk_queue_start_tag(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
1151 1152
{
	struct blk_queue_tag *bqt = q->queue_tags;
1153
	int tag;
L
Linus Torvalds 已提交
1154

1155
	if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
L
Linus Torvalds 已提交
1156
		printk(KERN_ERR 
1157 1158 1159
		       "%s: request %p for device [%s] already tagged %d",
		       __FUNCTION__, rq,
		       rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
L
Linus Torvalds 已提交
1160 1161 1162
		BUG();
	}

1163 1164 1165 1166 1167 1168 1169 1170
	/*
	 * Protect against shared tag maps, as we may not have exclusive
	 * access to the tag map.
	 */
	do {
		tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
		if (tag >= bqt->max_depth)
			return 1;
L
Linus Torvalds 已提交
1171

1172
	} while (test_and_set_bit_lock(tag, bqt->tag_map));
N
Nick Piggin 已提交
1173
	/*
1174 1175
	 * We need lock ordering semantics given by test_and_set_bit_lock.
	 * See blk_queue_end_tag for details.
N
Nick Piggin 已提交
1176
	 */
L
Linus Torvalds 已提交
1177

1178
	rq->cmd_flags |= REQ_QUEUED;
L
Linus Torvalds 已提交
1179 1180 1181
	rq->tag = tag;
	bqt->tag_index[tag] = rq;
	blkdev_dequeue_request(rq);
1182
	list_add(&rq->queuelist, &q->tag_busy_list);
L
Linus Torvalds 已提交
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
	bqt->busy++;
	return 0;
}

EXPORT_SYMBOL(blk_queue_start_tag);

/**
 * blk_queue_invalidate_tags - invalidate all pending tags
 * @q:  the request queue for the device
 *
 *  Description:
 *   Hardware conditions may dictate a need to stop all pending requests.
 *   In this case, we will safely clear the block side of the tag queue and
 *   readd all requests to the request queue in the right order.
 *
 *  Notes:
 *   queue lock must be held.
 **/
1201
void blk_queue_invalidate_tags(struct request_queue *q)
L
Linus Torvalds 已提交
1202 1203 1204
{
	struct list_head *tmp, *n;

1205 1206
	list_for_each_safe(tmp, n, &q->tag_busy_list)
		blk_requeue_request(q, list_entry_rq(tmp));
L
Linus Torvalds 已提交
1207 1208 1209 1210 1211 1212 1213 1214
}

EXPORT_SYMBOL(blk_queue_invalidate_tags);

void blk_dump_rq_flags(struct request *rq, char *msg)
{
	int bit;

1215 1216 1217
	printk("%s: dev %s: type=%x, flags=%x\n", msg,
		rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
		rq->cmd_flags);
L
Linus Torvalds 已提交
1218 1219 1220 1221 1222 1223

	printk("\nsector %llu, nr/cnr %lu/%u\n", (unsigned long long)rq->sector,
						       rq->nr_sectors,
						       rq->current_nr_sectors);
	printk("bio %p, biotail %p, buffer %p, data %p, len %u\n", rq->bio, rq->biotail, rq->buffer, rq->data, rq->data_len);

1224
	if (blk_pc_request(rq)) {
L
Linus Torvalds 已提交
1225 1226 1227 1228 1229 1230 1231 1232 1233
		printk("cdb: ");
		for (bit = 0; bit < sizeof(rq->cmd); bit++)
			printk("%02x ", rq->cmd[bit]);
		printk("\n");
	}
}

EXPORT_SYMBOL(blk_dump_rq_flags);

1234
void blk_recount_segments(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
1235
{
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
	struct request rq;
	struct bio *nxt = bio->bi_next;
	rq.q = q;
	rq.bio = rq.biotail = bio;
	bio->bi_next = NULL;
	blk_recalc_rq_segments(&rq);
	bio->bi_next = nxt;
	bio->bi_phys_segments = rq.nr_phys_segments;
	bio->bi_hw_segments = rq.nr_hw_segments;
	bio->bi_flags |= (1 << BIO_SEG_VALID);
}
EXPORT_SYMBOL(blk_recount_segments);

static void blk_recalc_rq_segments(struct request *rq)
{
	int nr_phys_segs;
	int nr_hw_segs;
	unsigned int phys_size;
	unsigned int hw_size;
L
Linus Torvalds 已提交
1255
	struct bio_vec *bv, *bvprv = NULL;
1256 1257 1258
	int seg_size;
	int hw_seg_size;
	int cluster;
1259
	struct req_iterator iter;
L
Linus Torvalds 已提交
1260
	int high, highprv = 1;
1261
	struct request_queue *q = rq->q;
L
Linus Torvalds 已提交
1262

1263
	if (!rq->bio)
L
Linus Torvalds 已提交
1264 1265 1266
		return;

	cluster = q->queue_flags & (1 << QUEUE_FLAG_CLUSTER);
1267 1268
	hw_seg_size = seg_size = 0;
	phys_size = hw_size = nr_phys_segs = nr_hw_segs = 0;
1269
	rq_for_each_segment(bv, rq, iter) {
L
Linus Torvalds 已提交
1270 1271 1272 1273 1274
		/*
		 * the trick here is making sure that a high page is never
		 * considered part of another segment, since that might
		 * change with the bounce page.
		 */
1275
		high = page_to_pfn(bv->bv_page) > q->bounce_pfn;
L
Linus Torvalds 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
		if (high || highprv)
			goto new_hw_segment;
		if (cluster) {
			if (seg_size + bv->bv_len > q->max_segment_size)
				goto new_segment;
			if (!BIOVEC_PHYS_MERGEABLE(bvprv, bv))
				goto new_segment;
			if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bv))
				goto new_segment;
			if (BIOVEC_VIRT_OVERSIZE(hw_seg_size + bv->bv_len))
				goto new_hw_segment;

			seg_size += bv->bv_len;
			hw_seg_size += bv->bv_len;
			bvprv = bv;
			continue;
		}
new_segment:
		if (BIOVEC_VIRT_MERGEABLE(bvprv, bv) &&
1295
		    !BIOVEC_VIRT_OVERSIZE(hw_seg_size + bv->bv_len))
L
Linus Torvalds 已提交
1296
			hw_seg_size += bv->bv_len;
1297
		else {
L
Linus Torvalds 已提交
1298
new_hw_segment:
1299 1300 1301
			if (nr_hw_segs == 1 &&
			    hw_seg_size > rq->bio->bi_hw_front_size)
				rq->bio->bi_hw_front_size = hw_seg_size;
L
Linus Torvalds 已提交
1302 1303 1304 1305 1306 1307 1308 1309 1310
			hw_seg_size = BIOVEC_VIRT_START_SIZE(bv) + bv->bv_len;
			nr_hw_segs++;
		}

		nr_phys_segs++;
		bvprv = bv;
		seg_size = bv->bv_len;
		highprv = high;
	}
1311 1312 1313 1314 1315 1316 1317 1318

	if (nr_hw_segs == 1 &&
	    hw_seg_size > rq->bio->bi_hw_front_size)
		rq->bio->bi_hw_front_size = hw_seg_size;
	if (hw_seg_size > rq->biotail->bi_hw_back_size)
		rq->biotail->bi_hw_back_size = hw_seg_size;
	rq->nr_phys_segments = nr_phys_segs;
	rq->nr_hw_segments = nr_hw_segs;
L
Linus Torvalds 已提交
1319 1320
}

1321
static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio,
L
Linus Torvalds 已提交
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
				   struct bio *nxt)
{
	if (!(q->queue_flags & (1 << QUEUE_FLAG_CLUSTER)))
		return 0;

	if (!BIOVEC_PHYS_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)))
		return 0;
	if (bio->bi_size + nxt->bi_size > q->max_segment_size)
		return 0;

	/*
	 * bio and nxt are contigous in memory, check if the queue allows
	 * these two to be merged into one
	 */
	if (BIO_SEG_BOUNDARY(q, bio, nxt))
		return 1;

	return 0;
}

1342
static int blk_hw_contig_segment(struct request_queue *q, struct bio *bio,
L
Linus Torvalds 已提交
1343 1344 1345 1346 1347 1348 1349
				 struct bio *nxt)
{
	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
		blk_recount_segments(q, bio);
	if (unlikely(!bio_flagged(nxt, BIO_SEG_VALID)))
		blk_recount_segments(q, nxt);
	if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)) ||
1350
	    BIOVEC_VIRT_OVERSIZE(bio->bi_hw_back_size + nxt->bi_hw_front_size))
L
Linus Torvalds 已提交
1351
		return 0;
1352
	if (bio->bi_hw_back_size + nxt->bi_hw_front_size > q->max_segment_size)
L
Linus Torvalds 已提交
1353 1354 1355 1356 1357 1358 1359 1360 1361
		return 0;

	return 1;
}

/*
 * map a request to scatterlist, return number of sg entries setup. Caller
 * must make sure sg can hold rq->nr_phys_segments entries
 */
1362
int blk_rq_map_sg(struct request_queue *q, struct request *rq,
J
Jens Axboe 已提交
1363
		  struct scatterlist *sglist)
L
Linus Torvalds 已提交
1364 1365
{
	struct bio_vec *bvec, *bvprv;
1366
	struct req_iterator iter;
J
Jens Axboe 已提交
1367
	struct scatterlist *sg;
1368
	int nsegs, cluster;
L
Linus Torvalds 已提交
1369 1370 1371 1372 1373 1374 1375 1376

	nsegs = 0;
	cluster = q->queue_flags & (1 << QUEUE_FLAG_CLUSTER);

	/*
	 * for each bio in rq
	 */
	bvprv = NULL;
J
Jens Axboe 已提交
1377
	sg = NULL;
1378
	rq_for_each_segment(bvec, rq, iter) {
1379
		int nbytes = bvec->bv_len;
L
Linus Torvalds 已提交
1380

1381
		if (bvprv && cluster) {
J
Jens Axboe 已提交
1382
			if (sg->length + nbytes > q->max_segment_size)
1383
				goto new_segment;
L
Linus Torvalds 已提交
1384

1385 1386 1387 1388
			if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec))
				goto new_segment;
			if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec))
				goto new_segment;
L
Linus Torvalds 已提交
1389

J
Jens Axboe 已提交
1390
			sg->length += nbytes;
1391
		} else {
L
Linus Torvalds 已提交
1392
new_segment:
J
Jens Axboe 已提交
1393 1394
			if (!sg)
				sg = sglist;
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
			else {
				/*
				 * If the driver previously mapped a shorter
				 * list, we could see a termination bit
				 * prematurely unless it fully inits the sg
				 * table on each mapping. We KNOW that there
				 * must be more entries here or the driver
				 * would be buggy, so force clear the
				 * termination bit to avoid doing a full
				 * sg_init_table() in drivers for each command.
				 */
				sg->page_link &= ~0x02;
J
Jens Axboe 已提交
1407
				sg = sg_next(sg);
1408
			}
1409

1410
			sg_set_page(sg, bvec->bv_page, nbytes, bvec->bv_offset);
1411 1412 1413
			nsegs++;
		}
		bvprv = bvec;
1414
	} /* segments in rq */
L
Linus Torvalds 已提交
1415

J
James Bottomley 已提交
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
	if (q->dma_drain_size) {
		sg->page_link &= ~0x02;
		sg = sg_next(sg);
		sg_set_page(sg, virt_to_page(q->dma_drain_buffer),
			    q->dma_drain_size,
			    ((unsigned long)q->dma_drain_buffer) &
			    (PAGE_SIZE - 1));
		nsegs++;
	}

1426
	if (sg)
J
Jens Axboe 已提交
1427
		sg_mark_end(sg);
1428

L
Linus Torvalds 已提交
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	return nsegs;
}

EXPORT_SYMBOL(blk_rq_map_sg);

/*
 * the standard queue merge functions, can be overridden with device
 * specific ones if so desired
 */

1439
static inline int ll_new_mergeable(struct request_queue *q,
L
Linus Torvalds 已提交
1440 1441 1442 1443 1444 1445
				   struct request *req,
				   struct bio *bio)
{
	int nr_phys_segs = bio_phys_segments(q, bio);

	if (req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) {
1446
		req->cmd_flags |= REQ_NOMERGE;
L
Linus Torvalds 已提交
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
		if (req == q->last_merge)
			q->last_merge = NULL;
		return 0;
	}

	/*
	 * A hw segment is just getting larger, bump just the phys
	 * counter.
	 */
	req->nr_phys_segments += nr_phys_segs;
	return 1;
}

1460
static inline int ll_new_hw_segment(struct request_queue *q,
L
Linus Torvalds 已提交
1461 1462 1463 1464 1465 1466 1467 1468
				    struct request *req,
				    struct bio *bio)
{
	int nr_hw_segs = bio_hw_segments(q, bio);
	int nr_phys_segs = bio_phys_segments(q, bio);

	if (req->nr_hw_segments + nr_hw_segs > q->max_hw_segments
	    || req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) {
1469
		req->cmd_flags |= REQ_NOMERGE;
L
Linus Torvalds 已提交
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
		if (req == q->last_merge)
			q->last_merge = NULL;
		return 0;
	}

	/*
	 * This will form the start of a new hw segment.  Bump both
	 * counters.
	 */
	req->nr_hw_segments += nr_hw_segs;
	req->nr_phys_segments += nr_phys_segs;
	return 1;
}

N
NeilBrown 已提交
1484 1485
static int ll_back_merge_fn(struct request_queue *q, struct request *req,
			    struct bio *bio)
L
Linus Torvalds 已提交
1486
{
1487
	unsigned short max_sectors;
L
Linus Torvalds 已提交
1488 1489
	int len;

1490 1491 1492 1493 1494 1495
	if (unlikely(blk_pc_request(req)))
		max_sectors = q->max_hw_sectors;
	else
		max_sectors = q->max_sectors;

	if (req->nr_sectors + bio_sectors(bio) > max_sectors) {
1496
		req->cmd_flags |= REQ_NOMERGE;
L
Linus Torvalds 已提交
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
		if (req == q->last_merge)
			q->last_merge = NULL;
		return 0;
	}
	if (unlikely(!bio_flagged(req->biotail, BIO_SEG_VALID)))
		blk_recount_segments(q, req->biotail);
	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
		blk_recount_segments(q, bio);
	len = req->biotail->bi_hw_back_size + bio->bi_hw_front_size;
	if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(req->biotail), __BVEC_START(bio)) &&
	    !BIOVEC_VIRT_OVERSIZE(len)) {
		int mergeable =  ll_new_mergeable(q, req, bio);

		if (mergeable) {
			if (req->nr_hw_segments == 1)
				req->bio->bi_hw_front_size = len;
			if (bio->bi_hw_segments == 1)
				bio->bi_hw_back_size = len;
		}
		return mergeable;
	}

	return ll_new_hw_segment(q, req, bio);
}

1522
static int ll_front_merge_fn(struct request_queue *q, struct request *req, 
L
Linus Torvalds 已提交
1523 1524
			     struct bio *bio)
{
1525
	unsigned short max_sectors;
L
Linus Torvalds 已提交
1526 1527
	int len;

1528 1529 1530 1531 1532 1533 1534
	if (unlikely(blk_pc_request(req)))
		max_sectors = q->max_hw_sectors;
	else
		max_sectors = q->max_sectors;


	if (req->nr_sectors + bio_sectors(bio) > max_sectors) {
1535
		req->cmd_flags |= REQ_NOMERGE;
L
Linus Torvalds 已提交
1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
		if (req == q->last_merge)
			q->last_merge = NULL;
		return 0;
	}
	len = bio->bi_hw_back_size + req->bio->bi_hw_front_size;
	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
		blk_recount_segments(q, bio);
	if (unlikely(!bio_flagged(req->bio, BIO_SEG_VALID)))
		blk_recount_segments(q, req->bio);
	if (BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(req->bio)) &&
	    !BIOVEC_VIRT_OVERSIZE(len)) {
		int mergeable =  ll_new_mergeable(q, req, bio);

		if (mergeable) {
			if (bio->bi_hw_segments == 1)
				bio->bi_hw_front_size = len;
			if (req->nr_hw_segments == 1)
				req->biotail->bi_hw_back_size = len;
		}
		return mergeable;
	}

	return ll_new_hw_segment(q, req, bio);
}

1561
static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
L
Linus Torvalds 已提交
1562 1563
				struct request *next)
{
1564 1565
	int total_phys_segments;
	int total_hw_segments;
L
Linus Torvalds 已提交
1566 1567 1568 1569 1570 1571 1572 1573 1574

	/*
	 * First check if the either of the requests are re-queued
	 * requests.  Can't merge them if they are.
	 */
	if (req->special || next->special)
		return 0;

	/*
1575
	 * Will it become too large?
L
Linus Torvalds 已提交
1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
	 */
	if ((req->nr_sectors + next->nr_sectors) > q->max_sectors)
		return 0;

	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
	if (blk_phys_contig_segment(q, req->biotail, next->bio))
		total_phys_segments--;

	if (total_phys_segments > q->max_phys_segments)
		return 0;

	total_hw_segments = req->nr_hw_segments + next->nr_hw_segments;
	if (blk_hw_contig_segment(q, req->biotail, next->bio)) {
		int len = req->biotail->bi_hw_back_size + next->bio->bi_hw_front_size;
		/*
		 * propagate the combined length to the end of the requests
		 */
		if (req->nr_hw_segments == 1)
			req->bio->bi_hw_front_size = len;
		if (next->nr_hw_segments == 1)
			next->biotail->bi_hw_back_size = len;
		total_hw_segments--;
	}

	if (total_hw_segments > q->max_hw_segments)
		return 0;

	/* Merge is OK... */
	req->nr_phys_segments = total_phys_segments;
	req->nr_hw_segments = total_hw_segments;
	return 1;
}

/*
 * "plug" the device if there are no outstanding requests: this will
 * force the transfer to start only after we have put all the requests
 * on the list.
 *
 * This is called with interrupts off and no requests on the queue and
 * with the queue lock held.
 */
1617
void blk_plug_device(struct request_queue *q)
L
Linus Torvalds 已提交
1618 1619 1620 1621 1622 1623 1624
{
	WARN_ON(!irqs_disabled());

	/*
	 * don't plug a stopped queue, it must be paired with blk_start_queue()
	 * which will restart the queueing
	 */
1625
	if (blk_queue_stopped(q))
L
Linus Torvalds 已提交
1626 1627
		return;

1628
	if (!test_and_set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
L
Linus Torvalds 已提交
1629
		mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
1630 1631
		blk_add_trace_generic(q, NULL, 0, BLK_TA_PLUG);
	}
L
Linus Torvalds 已提交
1632 1633 1634 1635 1636 1637 1638 1639
}

EXPORT_SYMBOL(blk_plug_device);

/*
 * remove the queue from the plugged list, if present. called with
 * queue lock held and interrupts disabled.
 */
1640
int blk_remove_plug(struct request_queue *q)
L
Linus Torvalds 已提交
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655
{
	WARN_ON(!irqs_disabled());

	if (!test_and_clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
		return 0;

	del_timer(&q->unplug_timer);
	return 1;
}

EXPORT_SYMBOL(blk_remove_plug);

/*
 * remove the plug and let it rip..
 */
1656
void __generic_unplug_device(struct request_queue *q)
L
Linus Torvalds 已提交
1657
{
1658
	if (unlikely(blk_queue_stopped(q)))
L
Linus Torvalds 已提交
1659 1660 1661 1662 1663
		return;

	if (!blk_remove_plug(q))
		return;

1664
	q->request_fn(q);
L
Linus Torvalds 已提交
1665 1666 1667 1668 1669
}
EXPORT_SYMBOL(__generic_unplug_device);

/**
 * generic_unplug_device - fire a request queue
1670
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
1671 1672 1673 1674 1675 1676 1677 1678
 *
 * Description:
 *   Linux uses plugging to build bigger requests queues before letting
 *   the device have at them. If a queue is plugged, the I/O scheduler
 *   is still adding and merging requests on the queue. Once the queue
 *   gets unplugged, the request_fn defined for the queue is invoked and
 *   transfers started.
 **/
1679
void generic_unplug_device(struct request_queue *q)
L
Linus Torvalds 已提交
1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
{
	spin_lock_irq(q->queue_lock);
	__generic_unplug_device(q);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(generic_unplug_device);

static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
				   struct page *page)
{
1690
	struct request_queue *q = bdi->unplug_io_data;
L
Linus Torvalds 已提交
1691

1692
	blk_unplug(q);
L
Linus Torvalds 已提交
1693 1694
}

1695
static void blk_unplug_work(struct work_struct *work)
L
Linus Torvalds 已提交
1696
{
1697 1698
	struct request_queue *q =
		container_of(work, struct request_queue, unplug_work);
L
Linus Torvalds 已提交
1699

1700 1701 1702
	blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
				q->rq.count[READ] + q->rq.count[WRITE]);

L
Linus Torvalds 已提交
1703 1704 1705 1706 1707
	q->unplug_fn(q);
}

static void blk_unplug_timeout(unsigned long data)
{
1708
	struct request_queue *q = (struct request_queue *)data;
L
Linus Torvalds 已提交
1709

1710 1711 1712
	blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_TIMER, NULL,
				q->rq.count[READ] + q->rq.count[WRITE]);

L
Linus Torvalds 已提交
1713 1714 1715
	kblockd_schedule_work(&q->unplug_work);
}

1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729
void blk_unplug(struct request_queue *q)
{
	/*
	 * devices don't necessarily have an ->unplug_fn defined
	 */
	if (q->unplug_fn) {
		blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
					q->rq.count[READ] + q->rq.count[WRITE]);

		q->unplug_fn(q);
	}
}
EXPORT_SYMBOL(blk_unplug);

L
Linus Torvalds 已提交
1730 1731
/**
 * blk_start_queue - restart a previously stopped queue
1732
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
1733 1734 1735 1736 1737 1738
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
 *   entered. Also see blk_stop_queue(). Queue lock must be held.
 **/
1739
void blk_start_queue(struct request_queue *q)
L
Linus Torvalds 已提交
1740
{
1741 1742
	WARN_ON(!irqs_disabled());

L
Linus Torvalds 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
	clear_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);

	/*
	 * one level of recursion is ok and is much faster than kicking
	 * the unplug handling
	 */
	if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
		q->request_fn(q);
		clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
	} else {
		blk_plug_device(q);
		kblockd_schedule_work(&q->unplug_work);
	}
}

EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
1762
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
 *   blk_start_queue() to restart queue operations. Queue lock must be held.
 **/
1774
void blk_stop_queue(struct request_queue *q)
L
Linus Torvalds 已提交
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789
{
	blk_remove_plug(q);
	set_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
1790
 *     that the callbacks might use. The caller must already have made sure
L
Linus Torvalds 已提交
1791 1792 1793 1794 1795 1796 1797
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
 */
void blk_sync_queue(struct request_queue *q)
{
	del_timer_sync(&q->unplug_timer);
1798
	kblockd_flush_work(&q->unplug_work);
L
Linus Torvalds 已提交
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
}
EXPORT_SYMBOL(blk_sync_queue);

/**
 * blk_run_queue - run a single device queue
 * @q:	The queue to run
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	blk_remove_plug(q);
J
Jens Axboe 已提交
1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826

	/*
	 * Only recurse once to avoid overrunning the stack, let the unplug
	 * handling reinvoke the handler shortly if we already got there.
	 */
	if (!elv_queue_empty(q)) {
		if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
			q->request_fn(q);
			clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
		} else {
			blk_plug_device(q);
			kblockd_schedule_work(&q->unplug_work);
		}
	}

L
Linus Torvalds 已提交
1827 1828 1829 1830 1831
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

/**
1832
 * blk_cleanup_queue: - release a &struct request_queue when it is no longer needed
1833
 * @kobj:    the kobj belonging of the request queue to be released
L
Linus Torvalds 已提交
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
 *
 * Description:
 *     blk_cleanup_queue is the pair to blk_init_queue() or
 *     blk_queue_make_request().  It should be called when a request queue is
 *     being released; typically when a block device is being de-registered.
 *     Currently, its primary task it to free all the &struct request
 *     structures that were allocated to the queue and the queue itself.
 *
 * Caveat:
 *     Hopefully the low level driver will have finished any
 *     outstanding requests first...
 **/
1846
static void blk_release_queue(struct kobject *kobj)
L
Linus Torvalds 已提交
1847
{
1848 1849
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);
L
Linus Torvalds 已提交
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
	struct request_list *rl = &q->rq;

	blk_sync_queue(q);

	if (rl->rq_pool)
		mempool_destroy(rl->rq_pool);

	if (q->queue_tags)
		__blk_queue_free_tags(q);

1860
	blk_trace_shutdown(q);
1861

P
Peter Zijlstra 已提交
1862
	bdi_destroy(&q->backing_dev_info);
L
Linus Torvalds 已提交
1863 1864 1865
	kmem_cache_free(requestq_cachep, q);
}

1866
void blk_put_queue(struct request_queue *q)
1867 1868 1869 1870 1871
{
	kobject_put(&q->kobj);
}
EXPORT_SYMBOL(blk_put_queue);

1872
void blk_cleanup_queue(struct request_queue * q)
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
{
	mutex_lock(&q->sysfs_lock);
	set_bit(QUEUE_FLAG_DEAD, &q->queue_flags);
	mutex_unlock(&q->sysfs_lock);

	if (q->elevator)
		elevator_exit(q->elevator);

	blk_put_queue(q);
}

L
Linus Torvalds 已提交
1884 1885
EXPORT_SYMBOL(blk_cleanup_queue);

1886
static int blk_init_free_list(struct request_queue *q)
L
Linus Torvalds 已提交
1887 1888 1889 1890 1891
{
	struct request_list *rl = &q->rq;

	rl->count[READ] = rl->count[WRITE] = 0;
	rl->starved[READ] = rl->starved[WRITE] = 0;
T
Tejun Heo 已提交
1892
	rl->elvpriv = 0;
L
Linus Torvalds 已提交
1893 1894 1895
	init_waitqueue_head(&rl->wait[READ]);
	init_waitqueue_head(&rl->wait[WRITE]);

1896 1897
	rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
				mempool_free_slab, request_cachep, q->node);
L
Linus Torvalds 已提交
1898 1899 1900 1901 1902 1903 1904

	if (!rl->rq_pool)
		return -ENOMEM;

	return 0;
}

1905
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
L
Linus Torvalds 已提交
1906
{
1907 1908 1909
	return blk_alloc_queue_node(gfp_mask, -1);
}
EXPORT_SYMBOL(blk_alloc_queue);
L
Linus Torvalds 已提交
1910

1911 1912
static struct kobj_type queue_ktype;

1913
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
1914
{
1915
	struct request_queue *q;
P
Peter Zijlstra 已提交
1916
	int err;
1917

1918 1919
	q = kmem_cache_alloc_node(requestq_cachep,
				gfp_mask | __GFP_ZERO, node_id);
L
Linus Torvalds 已提交
1920 1921 1922
	if (!q)
		return NULL;

P
Peter Zijlstra 已提交
1923 1924 1925 1926 1927 1928 1929 1930
	q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
	q->backing_dev_info.unplug_io_data = q;
	err = bdi_init(&q->backing_dev_info);
	if (err) {
		kmem_cache_free(requestq_cachep, q);
		return NULL;
	}

L
Linus Torvalds 已提交
1931
	init_timer(&q->unplug_timer);
1932

1933
	kobject_init(&q->kobj, &queue_ktype);
L
Linus Torvalds 已提交
1934

1935 1936
	mutex_init(&q->sysfs_lock);

L
Linus Torvalds 已提交
1937 1938
	return q;
}
1939
EXPORT_SYMBOL(blk_alloc_queue_node);
L
Linus Torvalds 已提交
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
1963 1964
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
L
Linus Torvalds 已提交
1965 1966 1967 1968 1969 1970 1971 1972
 *
 *    Function returns a pointer to the initialized request queue, or NULL if
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
1973

1974
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
L
Linus Torvalds 已提交
1975
{
1976 1977 1978 1979
	return blk_init_queue_node(rfn, lock, -1);
}
EXPORT_SYMBOL(blk_init_queue);

1980
struct request_queue *
1981 1982
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
1983
	struct request_queue *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
L
Linus Torvalds 已提交
1984 1985 1986 1987

	if (!q)
		return NULL;

1988
	q->node = node_id;
1989 1990 1991 1992
	if (blk_init_free_list(q)) {
		kmem_cache_free(requestq_cachep, q);
		return NULL;
	}
L
Linus Torvalds 已提交
1993

已提交
1994 1995 1996 1997 1998 1999 2000 2001 2002
	/*
	 * if caller didn't supply a lock, they get per-queue locking with
	 * our embedded lock
	 */
	if (!lock) {
		spin_lock_init(&q->__queue_lock);
		lock = &q->__queue_lock;
	}

L
Linus Torvalds 已提交
2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
	q->request_fn		= rfn;
	q->prep_rq_fn		= NULL;
	q->unplug_fn		= generic_unplug_device;
	q->queue_flags		= (1 << QUEUE_FLAG_CLUSTER);
	q->queue_lock		= lock;

	blk_queue_segment_boundary(q, 0xffffffff);

	blk_queue_make_request(q, __make_request);
	blk_queue_max_segment_size(q, MAX_SEGMENT_SIZE);

	blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
	blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);

2017 2018
	q->sg_reserved_size = INT_MAX;

L
Linus Torvalds 已提交
2019 2020 2021 2022 2023 2024 2025 2026
	/*
	 * all done
	 */
	if (!elevator_init(q, NULL)) {
		blk_queue_congestion_threshold(q);
		return q;
	}

2027
	blk_put_queue(q);
L
Linus Torvalds 已提交
2028 2029
	return NULL;
}
2030
EXPORT_SYMBOL(blk_init_queue_node);
L
Linus Torvalds 已提交
2031

2032
int blk_get_queue(struct request_queue *q)
L
Linus Torvalds 已提交
2033
{
N
Nick Piggin 已提交
2034
	if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
2035
		kobject_get(&q->kobj);
L
Linus Torvalds 已提交
2036 2037 2038 2039 2040 2041 2042 2043
		return 0;
	}

	return 1;
}

EXPORT_SYMBOL(blk_get_queue);

2044
static inline void blk_free_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
2045
{
2046
	if (rq->cmd_flags & REQ_ELVPRIV)
T
Tejun Heo 已提交
2047
		elv_put_request(q, rq);
L
Linus Torvalds 已提交
2048 2049 2050
	mempool_free(rq, q->rq.rq_pool);
}

J
Jens Axboe 已提交
2051
static struct request *
2052
blk_alloc_request(struct request_queue *q, int rw, int priv, gfp_t gfp_mask)
L
Linus Torvalds 已提交
2053 2054 2055 2056 2057 2058 2059
{
	struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);

	if (!rq)
		return NULL;

	/*
2060
	 * first three bits are identical in rq->cmd_flags and bio->bi_rw,
L
Linus Torvalds 已提交
2061 2062
	 * see bio.h and blkdev.h
	 */
2063
	rq->cmd_flags = rw | REQ_ALLOCED;
L
Linus Torvalds 已提交
2064

T
Tejun Heo 已提交
2065
	if (priv) {
2066
		if (unlikely(elv_set_request(q, rq, gfp_mask))) {
T
Tejun Heo 已提交
2067 2068 2069
			mempool_free(rq, q->rq.rq_pool);
			return NULL;
		}
2070
		rq->cmd_flags |= REQ_ELVPRIV;
T
Tejun Heo 已提交
2071
	}
L
Linus Torvalds 已提交
2072

T
Tejun Heo 已提交
2073
	return rq;
L
Linus Torvalds 已提交
2074 2075 2076 2077 2078 2079
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
2080
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
2101
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
2102 2103 2104 2105 2106 2107 2108 2109
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

2110
static void __freed_request(struct request_queue *q, int rw)
L
Linus Torvalds 已提交
2111 2112 2113 2114
{
	struct request_list *rl = &q->rq;

	if (rl->count[rw] < queue_congestion_off_threshold(q))
2115
		blk_clear_queue_congested(q, rw);
L
Linus Torvalds 已提交
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128

	if (rl->count[rw] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[rw]))
			wake_up(&rl->wait[rw]);

		blk_clear_queue_full(q, rw);
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
2129
static void freed_request(struct request_queue *q, int rw, int priv)
L
Linus Torvalds 已提交
2130 2131 2132 2133
{
	struct request_list *rl = &q->rq;

	rl->count[rw]--;
T
Tejun Heo 已提交
2134 2135
	if (priv)
		rl->elvpriv--;
L
Linus Torvalds 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144

	__freed_request(q, rw);

	if (unlikely(rl->starved[rw ^ 1]))
		__freed_request(q, rw ^ 1);
}

#define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist)
/*
N
Nick Piggin 已提交
2145 2146 2147
 * Get a free request, queue_lock must be held.
 * Returns NULL on failure, with queue_lock held.
 * Returns !NULL on success, with queue_lock *not held*.
L
Linus Torvalds 已提交
2148
 */
2149
static struct request *get_request(struct request_queue *q, int rw_flags,
2150
				   struct bio *bio, gfp_t gfp_mask)
L
Linus Torvalds 已提交
2151 2152 2153
{
	struct request *rq = NULL;
	struct request_list *rl = &q->rq;
2154
	struct io_context *ioc = NULL;
2155
	const int rw = rw_flags & 0x01;
2156 2157
	int may_queue, priv;

2158
	may_queue = elv_may_queue(q, rw_flags);
2159 2160 2161 2162 2163
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

	if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[rw]+1 >= q->nr_requests) {
2164
			ioc = current_io_context(GFP_ATOMIC, q->node);
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
			if (!blk_queue_full(q, rw)) {
				ioc_set_batching(q, ioc);
				blk_set_queue_full(q, rw);
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
					goto out;
				}
			}
L
Linus Torvalds 已提交
2185
		}
2186
		blk_set_queue_congested(q, rw);
L
Linus Torvalds 已提交
2187 2188
	}

2189 2190 2191 2192 2193
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
H
Hugh Dickins 已提交
2194
	if (rl->count[rw] >= (3 * q->nr_requests / 2))
2195
		goto out;
H
Hugh Dickins 已提交
2196

L
Linus Torvalds 已提交
2197 2198
	rl->count[rw]++;
	rl->starved[rw] = 0;
T
Tejun Heo 已提交
2199

J
Jens Axboe 已提交
2200
	priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
T
Tejun Heo 已提交
2201 2202 2203
	if (priv)
		rl->elvpriv++;

L
Linus Torvalds 已提交
2204 2205
	spin_unlock_irq(q->queue_lock);

2206
	rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
2207
	if (unlikely(!rq)) {
L
Linus Torvalds 已提交
2208 2209 2210 2211 2212 2213 2214 2215
		/*
		 * Allocation failed presumably due to memory. Undo anything
		 * we might have messed up.
		 *
		 * Allocating task should really be put onto the front of the
		 * wait queue, but this is pretty rare.
		 */
		spin_lock_irq(q->queue_lock);
T
Tejun Heo 已提交
2216
		freed_request(q, rw, priv);
L
Linus Torvalds 已提交
2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231

		/*
		 * in the very unlikely event that allocation failed and no
		 * requests for this direction was pending, mark us starved
		 * so that freeing of a request in the other direction will
		 * notice us. another possible fix would be to split the
		 * rq mempool into READ and WRITE
		 */
rq_starved:
		if (unlikely(rl->count[rw] == 0))
			rl->starved[rw] = 1;

		goto out;
	}

2232 2233 2234 2235 2236 2237
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
L
Linus Torvalds 已提交
2238 2239 2240 2241
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
	
	rq_init(q, rq);
2242 2243

	blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);
L
Linus Torvalds 已提交
2244 2245 2246 2247 2248 2249 2250
out:
	return rq;
}

/*
 * No available requests for this queue, unplug the device and wait for some
 * requests to become available.
N
Nick Piggin 已提交
2251 2252
 *
 * Called with q->queue_lock held, and returns with it unlocked.
L
Linus Torvalds 已提交
2253
 */
2254
static struct request *get_request_wait(struct request_queue *q, int rw_flags,
2255
					struct bio *bio)
L
Linus Torvalds 已提交
2256
{
2257
	const int rw = rw_flags & 0x01;
L
Linus Torvalds 已提交
2258 2259
	struct request *rq;

2260
	rq = get_request(q, rw_flags, bio, GFP_NOIO);
2261 2262
	while (!rq) {
		DEFINE_WAIT(wait);
L
Linus Torvalds 已提交
2263 2264 2265 2266 2267
		struct request_list *rl = &q->rq;

		prepare_to_wait_exclusive(&rl->wait[rw], &wait,
				TASK_UNINTERRUPTIBLE);

2268
		rq = get_request(q, rw_flags, bio, GFP_NOIO);
L
Linus Torvalds 已提交
2269 2270 2271 2272

		if (!rq) {
			struct io_context *ioc;

2273 2274
			blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ);

N
Nick Piggin 已提交
2275 2276
			__generic_unplug_device(q);
			spin_unlock_irq(q->queue_lock);
L
Linus Torvalds 已提交
2277 2278 2279 2280 2281 2282 2283 2284
			io_schedule();

			/*
			 * After sleeping, we become a "batching" process and
			 * will be able to allocate at least one request, and
			 * up to a big batch of them for a small period time.
			 * See ioc_batching, ioc_set_batching
			 */
2285
			ioc = current_io_context(GFP_NOIO, q->node);
L
Linus Torvalds 已提交
2286
			ioc_set_batching(q, ioc);
N
Nick Piggin 已提交
2287 2288

			spin_lock_irq(q->queue_lock);
L
Linus Torvalds 已提交
2289 2290
		}
		finish_wait(&rl->wait[rw], &wait);
2291
	}
L
Linus Torvalds 已提交
2292 2293 2294 2295

	return rq;
}

2296
struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
L
Linus Torvalds 已提交
2297 2298 2299 2300 2301
{
	struct request *rq;

	BUG_ON(rw != READ && rw != WRITE);

N
Nick Piggin 已提交
2302 2303
	spin_lock_irq(q->queue_lock);
	if (gfp_mask & __GFP_WAIT) {
2304
		rq = get_request_wait(q, rw, NULL);
N
Nick Piggin 已提交
2305
	} else {
2306
		rq = get_request(q, rw, NULL, gfp_mask);
N
Nick Piggin 已提交
2307 2308 2309 2310
		if (!rq)
			spin_unlock_irq(q->queue_lock);
	}
	/* q->queue_lock is unlocked at this point */
L
Linus Torvalds 已提交
2311 2312 2313 2314 2315

	return rq;
}
EXPORT_SYMBOL(blk_get_request);

2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
/**
 * blk_start_queueing - initiate dispatch of requests to device
 * @q:		request queue to kick into gear
 *
 * This is basically a helper to remove the need to know whether a queue
 * is plugged or not if someone just wants to initiate dispatch of requests
 * for this queue.
 *
 * The queue lock must be held with interrupts disabled.
 */
2326
void blk_start_queueing(struct request_queue *q)
2327 2328 2329 2330 2331 2332 2333 2334
{
	if (!blk_queue_plugged(q))
		q->request_fn(q);
	else
		__generic_unplug_device(q);
}
EXPORT_SYMBOL(blk_start_queueing);

L
Linus Torvalds 已提交
2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
2345
void blk_requeue_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
2346
{
2347 2348
	blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);

L
Linus Torvalds 已提交
2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);

	elv_requeue_request(q, rq);
}

EXPORT_SYMBOL(blk_requeue_request);

/**
 * blk_insert_request - insert a special request in to a request queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 * @at_head:	insert request at head or tail of queue
 * @data:	private data
 *
 * Description:
 *    Many block devices need to execute commands asynchronously, so they don't
 *    block the whole kernel from preemption during request execution.  This is
 *    accomplished normally by inserting aritficial requests tagged as
 *    REQ_SPECIAL in to the corresponding request queue, and letting them be
 *    scheduled for actual execution by the request queue.
 *
 *    We have the option of inserting the head or the tail of the queue.
 *    Typically we use the tail for new ioctls and so forth.  We use the head
 *    of the queue for things like a QUEUE_FULL message from a device, or a
 *    host that is unable to accept a particular command.
 */
2376
void blk_insert_request(struct request_queue *q, struct request *rq,
2377
			int at_head, void *data)
L
Linus Torvalds 已提交
2378
{
2379
	int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
L
Linus Torvalds 已提交
2380 2381 2382 2383 2384 2385 2386
	unsigned long flags;

	/*
	 * tell I/O scheduler that this isn't a regular read/write (ie it
	 * must not attempt merges on this) and that it acts as a soft
	 * barrier
	 */
2387 2388
	rq->cmd_type = REQ_TYPE_SPECIAL;
	rq->cmd_flags |= REQ_SOFTBARRIER;
L
Linus Torvalds 已提交
2389 2390 2391 2392 2393 2394 2395 2396

	rq->special = data;

	spin_lock_irqsave(q->queue_lock, flags);

	/*
	 * If command is tagged, release the tag
	 */
2397 2398
	if (blk_rq_tagged(rq))
		blk_queue_end_tag(q, rq);
L
Linus Torvalds 已提交
2399

2400
	drive_stat_acct(rq, 1);
2401
	__elv_add_request(q, rq, where, 0);
2402
	blk_start_queueing(q);
L
Linus Torvalds 已提交
2403 2404 2405 2406 2407
	spin_unlock_irqrestore(q->queue_lock, flags);
}

EXPORT_SYMBOL(blk_insert_request);

2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
static int __blk_rq_unmap_user(struct bio *bio)
{
	int ret = 0;

	if (bio) {
		if (bio_flagged(bio, BIO_USER_MAPPED))
			bio_unmap_user(bio);
		else
			ret = bio_uncopy_user(bio);
	}

	return ret;
}

N
NeilBrown 已提交
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
int blk_rq_append_bio(struct request_queue *q, struct request *rq,
		      struct bio *bio)
{
	if (!rq->bio)
		blk_rq_bio_prep(q, rq, bio);
	else if (!ll_back_merge_fn(q, rq, bio))
		return -EINVAL;
	else {
		rq->biotail->bi_next = bio;
		rq->biotail = bio;

		rq->data_len += bio->bi_size;
	}
	return 0;
}
EXPORT_SYMBOL(blk_rq_append_bio);

2439
static int __blk_rq_map_user(struct request_queue *q, struct request *rq,
2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
			     void __user *ubuf, unsigned int len)
{
	unsigned long uaddr;
	struct bio *bio, *orig_bio;
	int reading, ret;

	reading = rq_data_dir(rq) == READ;

	/*
	 * if alignment requirement is satisfied, map in user pages for
	 * direct dma. else, set up kernel bounce buffers
	 */
	uaddr = (unsigned long) ubuf;
	if (!(uaddr & queue_dma_alignment(q)) && !(len & queue_dma_alignment(q)))
		bio = bio_map_user(q, NULL, uaddr, len, reading);
	else
		bio = bio_copy_user(q, uaddr, len, reading);

2458
	if (IS_ERR(bio))
2459 2460 2461 2462
		return PTR_ERR(bio);

	orig_bio = bio;
	blk_queue_bounce(q, &bio);
2463

2464 2465 2466 2467 2468 2469
	/*
	 * We link the bounce buffer in and could have to traverse it
	 * later so we have to get a ref to prevent it from being freed
	 */
	bio_get(bio);

N
NeilBrown 已提交
2470 2471 2472
	ret = blk_rq_append_bio(q, rq, bio);
	if (!ret)
		return bio->bi_size;
2473 2474

	/* if it was boucned we must call the end io function */
2475
	bio_endio(bio, 0);
2476 2477 2478 2479 2480
	__blk_rq_unmap_user(orig_bio);
	bio_put(bio);
	return ret;
}

L
Linus Torvalds 已提交
2481 2482 2483
/**
 * blk_rq_map_user - map user data to a request, for REQ_BLOCK_PC usage
 * @q:		request queue where request should be inserted
2484
 * @rq:		request structure to fill
L
Linus Torvalds 已提交
2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
 * @ubuf:	the user buffer
 * @len:	length of user data
 *
 * Description:
 *    Data will be mapped directly for zero copy io, if possible. Otherwise
 *    a kernel bounce buffer is used.
 *
 *    A matching blk_rq_unmap_user() must be issued at the end of io, while
 *    still in process context.
 *
 *    Note: The mapped bio may need to be bounced through blk_queue_bounce()
 *    before being submitted to the device, as pages mapped may be out of
 *    reach. It's the callers responsibility to make sure this happens. The
 *    original bio must be passed back in to blk_rq_unmap_user() for proper
 *    unmapping.
 */
2501 2502
int blk_rq_map_user(struct request_queue *q, struct request *rq,
		    void __user *ubuf, unsigned long len)
L
Linus Torvalds 已提交
2503
{
2504
	unsigned long bytes_read = 0;
J
Jens Axboe 已提交
2505
	struct bio *bio = NULL;
2506
	int ret;
L
Linus Torvalds 已提交
2507

2508
	if (len > (q->max_hw_sectors << 9))
2509 2510 2511
		return -EINVAL;
	if (!len || !ubuf)
		return -EINVAL;
L
Linus Torvalds 已提交
2512

2513 2514
	while (bytes_read != len) {
		unsigned long map_len, end, start;
L
Linus Torvalds 已提交
2515

2516 2517 2518 2519
		map_len = min_t(unsigned long, len - bytes_read, BIO_MAX_SIZE);
		end = ((unsigned long)ubuf + map_len + PAGE_SIZE - 1)
								>> PAGE_SHIFT;
		start = (unsigned long)ubuf >> PAGE_SHIFT;
L
Linus Torvalds 已提交
2520

2521 2522 2523 2524 2525 2526 2527
		/*
		 * A bad offset could cause us to require BIO_MAX_PAGES + 1
		 * pages. If this happens we just lower the requested
		 * mapping len by a page so that we can fit
		 */
		if (end - start > BIO_MAX_PAGES)
			map_len -= PAGE_SIZE;
L
Linus Torvalds 已提交
2528

2529 2530 2531
		ret = __blk_rq_map_user(q, rq, ubuf, map_len);
		if (ret < 0)
			goto unmap_rq;
J
Jens Axboe 已提交
2532 2533
		if (!bio)
			bio = rq->bio;
2534 2535
		bytes_read += ret;
		ubuf += ret;
L
Linus Torvalds 已提交
2536 2537
	}

2538 2539 2540
	rq->buffer = rq->data = NULL;
	return 0;
unmap_rq:
J
Jens Axboe 已提交
2541
	blk_rq_unmap_user(bio);
2542
	return ret;
L
Linus Torvalds 已提交
2543 2544 2545 2546
}

EXPORT_SYMBOL(blk_rq_map_user);

2547 2548 2549 2550 2551 2552
/**
 * blk_rq_map_user_iov - map user data to a request, for REQ_BLOCK_PC usage
 * @q:		request queue where request should be inserted
 * @rq:		request to map data to
 * @iov:	pointer to the iovec
 * @iov_count:	number of elements in the iovec
2553
 * @len:	I/O byte count
2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
 *
 * Description:
 *    Data will be mapped directly for zero copy io, if possible. Otherwise
 *    a kernel bounce buffer is used.
 *
 *    A matching blk_rq_unmap_user() must be issued at the end of io, while
 *    still in process context.
 *
 *    Note: The mapped bio may need to be bounced through blk_queue_bounce()
 *    before being submitted to the device, as pages mapped may be out of
 *    reach. It's the callers responsibility to make sure this happens. The
 *    original bio must be passed back in to blk_rq_unmap_user() for proper
 *    unmapping.
 */
2568
int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
2569
			struct sg_iovec *iov, int iov_count, unsigned int len)
2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582
{
	struct bio *bio;

	if (!iov || iov_count <= 0)
		return -EINVAL;

	/* we don't allow misaligned data like bio_map_user() does.  If the
	 * user is using sg, they're expected to know the alignment constraints
	 * and respect them accordingly */
	bio = bio_map_user_iov(q, NULL, iov, iov_count, rq_data_dir(rq)== READ);
	if (IS_ERR(bio))
		return PTR_ERR(bio);

2583
	if (bio->bi_size != len) {
2584
		bio_endio(bio, 0);
2585 2586 2587 2588 2589
		bio_unmap_user(bio);
		return -EINVAL;
	}

	bio_get(bio);
2590 2591 2592 2593 2594 2595 2596
	blk_rq_bio_prep(q, rq, bio);
	rq->buffer = rq->data = NULL;
	return 0;
}

EXPORT_SYMBOL(blk_rq_map_user_iov);

L
Linus Torvalds 已提交
2597 2598
/**
 * blk_rq_unmap_user - unmap a request with user data
J
Jens Axboe 已提交
2599
 * @bio:	       start of bio list
L
Linus Torvalds 已提交
2600 2601
 *
 * Description:
J
Jens Axboe 已提交
2602 2603 2604
 *    Unmap a rq previously mapped by blk_rq_map_user(). The caller must
 *    supply the original rq->bio from the blk_rq_map_user() return, since
 *    the io completion may have changed rq->bio.
L
Linus Torvalds 已提交
2605
 */
J
Jens Axboe 已提交
2606
int blk_rq_unmap_user(struct bio *bio)
L
Linus Torvalds 已提交
2607
{
J
Jens Axboe 已提交
2608
	struct bio *mapped_bio;
2609
	int ret = 0, ret2;
L
Linus Torvalds 已提交
2610

J
Jens Axboe 已提交
2611 2612 2613
	while (bio) {
		mapped_bio = bio;
		if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
2614
			mapped_bio = bio->bi_private;
L
Linus Torvalds 已提交
2615

2616 2617 2618 2619
		ret2 = __blk_rq_unmap_user(mapped_bio);
		if (ret2 && !ret)
			ret = ret2;

J
Jens Axboe 已提交
2620 2621 2622
		mapped_bio = bio;
		bio = bio->bi_next;
		bio_put(mapped_bio);
2623
	}
2624 2625

	return ret;
L
Linus Torvalds 已提交
2626 2627 2628 2629
}

EXPORT_SYMBOL(blk_rq_unmap_user);

M
Mike Christie 已提交
2630 2631 2632
/**
 * blk_rq_map_kern - map kernel data to a request, for REQ_BLOCK_PC usage
 * @q:		request queue where request should be inserted
2633
 * @rq:		request to fill
M
Mike Christie 已提交
2634 2635
 * @kbuf:	the kernel buffer
 * @len:	length of user data
2636
 * @gfp_mask:	memory allocation flags
M
Mike Christie 已提交
2637
 */
2638
int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
A
Al Viro 已提交
2639
		    unsigned int len, gfp_t gfp_mask)
M
Mike Christie 已提交
2640 2641 2642
{
	struct bio *bio;

2643
	if (len > (q->max_hw_sectors << 9))
2644 2645 2646
		return -EINVAL;
	if (!len || !kbuf)
		return -EINVAL;
M
Mike Christie 已提交
2647 2648

	bio = bio_map_kern(q, kbuf, len, gfp_mask);
2649 2650
	if (IS_ERR(bio))
		return PTR_ERR(bio);
M
Mike Christie 已提交
2651

2652 2653
	if (rq_data_dir(rq) == WRITE)
		bio->bi_rw |= (1 << BIO_RW);
M
Mike Christie 已提交
2654

2655
	blk_rq_bio_prep(q, rq, bio);
2656
	blk_queue_bounce(q, &rq->bio);
2657 2658
	rq->buffer = rq->data = NULL;
	return 0;
M
Mike Christie 已提交
2659 2660 2661 2662
}

EXPORT_SYMBOL(blk_rq_map_kern);

2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
/**
 * blk_execute_rq_nowait - insert a request into queue for execution
 * @q:		queue to insert the request in
 * @bd_disk:	matching gendisk
 * @rq:		request to insert
 * @at_head:    insert request at head or tail of queue
 * @done:	I/O completion handler
 *
 * Description:
 *    Insert a fully prepared request at the back of the io scheduler queue
 *    for execution.  Don't wait for completion.
 */
2675
void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
2676
			   struct request *rq, int at_head,
2677
			   rq_end_io_fn *done)
2678 2679 2680 2681
{
	int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;

	rq->rq_disk = bd_disk;
2682
	rq->cmd_flags |= REQ_NOMERGE;
2683
	rq->end_io = done;
2684 2685 2686 2687 2688
	WARN_ON(irqs_disabled());
	spin_lock_irq(q->queue_lock);
	__elv_add_request(q, rq, where, 1);
	__generic_unplug_device(q);
	spin_unlock_irq(q->queue_lock);
2689
}
2690 2691
EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);

L
Linus Torvalds 已提交
2692 2693 2694 2695 2696
/**
 * blk_execute_rq - insert a request into queue for execution
 * @q:		queue to insert the request in
 * @bd_disk:	matching gendisk
 * @rq:		request to insert
2697
 * @at_head:    insert request at head or tail of queue
L
Linus Torvalds 已提交
2698 2699 2700
 *
 * Description:
 *    Insert a fully prepared request at the back of the io scheduler queue
2701
 *    for execution and wait for completion.
L
Linus Torvalds 已提交
2702
 */
2703
int blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk,
2704
		   struct request *rq, int at_head)
L
Linus Torvalds 已提交
2705
{
2706
	DECLARE_COMPLETION_ONSTACK(wait);
L
Linus Torvalds 已提交
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721
	char sense[SCSI_SENSE_BUFFERSIZE];
	int err = 0;

	/*
	 * we need an extra reference to the request, so we can look at
	 * it after io completion
	 */
	rq->ref_count++;

	if (!rq->sense) {
		memset(sense, 0, sizeof(sense));
		rq->sense = sense;
		rq->sense_len = 0;
	}

2722
	rq->end_io_data = &wait;
2723
	blk_execute_rq_nowait(q, bd_disk, rq, at_head, blk_end_sync_rq);
L
Linus Torvalds 已提交
2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
	wait_for_completion(&wait);

	if (rq->errors)
		err = -EIO;

	return err;
}

EXPORT_SYMBOL(blk_execute_rq);

2734 2735 2736 2737 2738 2739 2740 2741
static void bio_end_empty_barrier(struct bio *bio, int err)
{
	if (err)
		clear_bit(BIO_UPTODATE, &bio->bi_flags);

	complete(bio->bi_private);
}

L
Linus Torvalds 已提交
2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753
/**
 * blkdev_issue_flush - queue a flush
 * @bdev:	blockdev to issue flush for
 * @error_sector:	error sector
 *
 * Description:
 *    Issue a flush for the block device in question. Caller can supply
 *    room for storing the error offset in case of a flush error, if they
 *    wish to.  Caller must run wait_for_completion() on its own.
 */
int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector)
{
2754
	DECLARE_COMPLETION_ONSTACK(wait);
2755
	struct request_queue *q;
2756 2757
	struct bio *bio;
	int ret;
L
Linus Torvalds 已提交
2758 2759 2760 2761 2762 2763 2764 2765

	if (bdev->bd_disk == NULL)
		return -ENXIO;

	q = bdev_get_queue(bdev);
	if (!q)
		return -ENXIO;

2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790
	bio = bio_alloc(GFP_KERNEL, 0);
	if (!bio)
		return -ENOMEM;

	bio->bi_end_io = bio_end_empty_barrier;
	bio->bi_private = &wait;
	bio->bi_bdev = bdev;
	submit_bio(1 << BIO_RW_BARRIER, bio);

	wait_for_completion(&wait);

	/*
	 * The driver must store the error location in ->bi_sector, if
	 * it supports it. For non-stacked drivers, this should be copied
	 * from rq->sector.
	 */
	if (error_sector)
		*error_sector = bio->bi_sector;

	ret = 0;
	if (!bio_flagged(bio, BIO_UPTODATE))
		ret = -EIO;

	bio_put(bio);
	return ret;
L
Linus Torvalds 已提交
2791 2792 2793 2794
}

EXPORT_SYMBOL(blkdev_issue_flush);

2795
static void drive_stat_acct(struct request *rq, int new_io)
L
Linus Torvalds 已提交
2796 2797 2798 2799 2800 2801
{
	int rw = rq_data_dir(rq);

	if (!blk_fs_request(rq) || !rq->rq_disk)
		return;

2802
	if (!new_io) {
2803
		__disk_stat_inc(rq->rq_disk, merges[rw]);
2804
	} else {
L
Linus Torvalds 已提交
2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
		disk_round_stats(rq->rq_disk);
		rq->rq_disk->in_flight++;
	}
}

/*
 * add-request adds a request to the linked list.
 * queue lock is held and interrupts disabled, as we muck with the
 * request queue list.
 */
2815
static inline void add_request(struct request_queue * q, struct request * req)
L
Linus Torvalds 已提交
2816
{
2817
	drive_stat_acct(req, 1);
L
Linus Torvalds 已提交
2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844

	/*
	 * elevator indicated where it wants this request to be
	 * inserted at elevator_merge time
	 */
	__elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
}
 
/*
 * disk_round_stats()	- Round off the performance stats on a struct
 * disk_stats.
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
void disk_round_stats(struct gendisk *disk)
{
	unsigned long now = jiffies;

2845 2846
	if (now == disk->stamp)
		return;
L
Linus Torvalds 已提交
2847

2848 2849 2850 2851 2852
	if (disk->in_flight) {
		__disk_stat_add(disk, time_in_queue,
				disk->in_flight * (now - disk->stamp));
		__disk_stat_add(disk, io_ticks, (now - disk->stamp));
	}
L
Linus Torvalds 已提交
2853 2854 2855
	disk->stamp = now;
}

2856 2857
EXPORT_SYMBOL_GPL(disk_round_stats);

L
Linus Torvalds 已提交
2858 2859 2860
/*
 * queue lock must be held
 */
2861
void __blk_put_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
2862 2863 2864 2865 2866 2867
{
	if (unlikely(!q))
		return;
	if (unlikely(--req->ref_count))
		return;

2868 2869
	elv_completed_request(q, req);

L
Linus Torvalds 已提交
2870 2871 2872 2873
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
2874
	if (req->cmd_flags & REQ_ALLOCED) {
L
Linus Torvalds 已提交
2875
		int rw = rq_data_dir(req);
2876
		int priv = req->cmd_flags & REQ_ELVPRIV;
L
Linus Torvalds 已提交
2877 2878

		BUG_ON(!list_empty(&req->queuelist));
2879
		BUG_ON(!hlist_unhashed(&req->hash));
L
Linus Torvalds 已提交
2880 2881

		blk_free_request(q, req);
T
Tejun Heo 已提交
2882
		freed_request(q, rw, priv);
L
Linus Torvalds 已提交
2883 2884 2885
	}
}

2886 2887
EXPORT_SYMBOL_GPL(__blk_put_request);

L
Linus Torvalds 已提交
2888 2889
void blk_put_request(struct request *req)
{
2890
	unsigned long flags;
2891
	struct request_queue *q = req->q;
2892

L
Linus Torvalds 已提交
2893
	/*
2894 2895
	 * Gee, IDE calls in w/ NULL q.  Fix IDE and remove the
	 * following if (q) test.
L
Linus Torvalds 已提交
2896
	 */
2897
	if (q) {
L
Linus Torvalds 已提交
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908
		spin_lock_irqsave(q->queue_lock, flags);
		__blk_put_request(q, req);
		spin_unlock_irqrestore(q->queue_lock, flags);
	}
}

EXPORT_SYMBOL(blk_put_request);

/**
 * blk_end_sync_rq - executes a completion event on a request
 * @rq: request to complete
J
Jens Axboe 已提交
2909
 * @error: end io status of the request
L
Linus Torvalds 已提交
2910
 */
2911
void blk_end_sync_rq(struct request *rq, int error)
L
Linus Torvalds 已提交
2912
{
2913
	struct completion *waiting = rq->end_io_data;
L
Linus Torvalds 已提交
2914

2915
	rq->end_io_data = NULL;
L
Linus Torvalds 已提交
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928
	__blk_put_request(rq->q, rq);

	/*
	 * complete last, if this is a stack request the process (and thus
	 * the rq pointer) could be invalid right after this complete()
	 */
	complete(waiting);
}
EXPORT_SYMBOL(blk_end_sync_rq);

/*
 * Has to be called with the request spinlock acquired
 */
2929
static int attempt_merge(struct request_queue *q, struct request *req,
L
Linus Torvalds 已提交
2930 2931 2932 2933 2934 2935
			  struct request *next)
{
	if (!rq_mergeable(req) || !rq_mergeable(next))
		return 0;

	/*
A
Andreas Mohr 已提交
2936
	 * not contiguous
L
Linus Torvalds 已提交
2937 2938 2939 2940 2941 2942
	 */
	if (req->sector + req->nr_sectors != next->sector)
		return 0;

	if (rq_data_dir(req) != rq_data_dir(next)
	    || req->rq_disk != next->rq_disk
2943
	    || next->special)
L
Linus Torvalds 已提交
2944 2945 2946 2947 2948 2949 2950 2951
		return 0;

	/*
	 * If we are allowed to merge, then append bio list
	 * from next to rq and release next. merge_requests_fn
	 * will have updated segment counts, update sector
	 * counts here.
	 */
J
Jens Axboe 已提交
2952
	if (!ll_merge_requests_fn(q, req, next))
L
Linus Torvalds 已提交
2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975
		return 0;

	/*
	 * At this point we have either done a back merge
	 * or front merge. We need the smaller start_time of
	 * the merged requests to be the current request
	 * for accounting purposes.
	 */
	if (time_after(req->start_time, next->start_time))
		req->start_time = next->start_time;

	req->biotail->bi_next = next->bio;
	req->biotail = next->biotail;

	req->nr_sectors = req->hard_nr_sectors += next->hard_nr_sectors;

	elv_merge_requests(q, req, next);

	if (req->rq_disk) {
		disk_round_stats(req->rq_disk);
		req->rq_disk->in_flight--;
	}

2976 2977
	req->ioprio = ioprio_best(req->ioprio, next->ioprio);

L
Linus Torvalds 已提交
2978 2979 2980 2981
	__blk_put_request(q, next);
	return 1;
}

2982 2983
static inline int attempt_back_merge(struct request_queue *q,
				     struct request *rq)
L
Linus Torvalds 已提交
2984 2985 2986 2987 2988 2989 2990 2991 2992
{
	struct request *next = elv_latter_request(q, rq);

	if (next)
		return attempt_merge(q, rq, next);

	return 0;
}

2993 2994
static inline int attempt_front_merge(struct request_queue *q,
				      struct request *rq)
L
Linus Torvalds 已提交
2995 2996 2997 2998 2999 3000 3001 3002 3003
{
	struct request *prev = elv_former_request(q, rq);

	if (prev)
		return attempt_merge(q, prev, rq);

	return 0;
}

3004 3005
static void init_request_from_bio(struct request *req, struct bio *bio)
{
3006
	req->cmd_type = REQ_TYPE_FS;
3007 3008 3009 3010 3011

	/*
	 * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
	 */
	if (bio_rw_ahead(bio) || bio_failfast(bio))
3012
		req->cmd_flags |= REQ_FAILFAST;
3013 3014 3015 3016 3017

	/*
	 * REQ_BARRIER implies no merging, but lets make it explicit
	 */
	if (unlikely(bio_barrier(bio)))
3018
		req->cmd_flags |= (REQ_HARDBARRIER | REQ_NOMERGE);
3019

J
Jens Axboe 已提交
3020
	if (bio_sync(bio))
3021
		req->cmd_flags |= REQ_RW_SYNC;
3022 3023
	if (bio_rw_meta(bio))
		req->cmd_flags |= REQ_RW_META;
J
Jens Axboe 已提交
3024

3025 3026 3027 3028
	req->errors = 0;
	req->hard_sector = req->sector = bio->bi_sector;
	req->ioprio = bio_prio(bio);
	req->start_time = jiffies;
3029
	blk_rq_bio_prep(req->q, req, bio);
3030 3031
}

3032
static int __make_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
3033
{
3034
	struct request *req;
3035 3036 3037
	int el_ret, nr_sectors, barrier, err;
	const unsigned short prio = bio_prio(bio);
	const int sync = bio_sync(bio);
3038
	int rw_flags;
L
Linus Torvalds 已提交
3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049

	nr_sectors = bio_sectors(bio);

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

	barrier = bio_barrier(bio);
3050
	if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
L
Linus Torvalds 已提交
3051 3052 3053 3054 3055 3056
		err = -EOPNOTSUPP;
		goto end_io;
	}

	spin_lock_irq(q->queue_lock);

3057
	if (unlikely(barrier) || elv_queue_empty(q))
L
Linus Torvalds 已提交
3058 3059 3060 3061 3062 3063 3064
		goto get_rq;

	el_ret = elv_merge(q, &req, bio);
	switch (el_ret) {
		case ELEVATOR_BACK_MERGE:
			BUG_ON(!rq_mergeable(req));

J
Jens Axboe 已提交
3065
			if (!ll_back_merge_fn(q, req, bio))
L
Linus Torvalds 已提交
3066 3067
				break;

3068 3069
			blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);

L
Linus Torvalds 已提交
3070 3071 3072
			req->biotail->bi_next = bio;
			req->biotail = bio;
			req->nr_sectors = req->hard_nr_sectors += nr_sectors;
3073
			req->ioprio = ioprio_best(req->ioprio, prio);
3074
			drive_stat_acct(req, 0);
L
Linus Torvalds 已提交
3075
			if (!attempt_back_merge(q, req))
3076
				elv_merged_request(q, req, el_ret);
L
Linus Torvalds 已提交
3077 3078 3079 3080 3081
			goto out;

		case ELEVATOR_FRONT_MERGE:
			BUG_ON(!rq_mergeable(req));

J
Jens Axboe 已提交
3082
			if (!ll_front_merge_fn(q, req, bio))
L
Linus Torvalds 已提交
3083 3084
				break;

3085 3086
			blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);

L
Linus Torvalds 已提交
3087 3088 3089 3090 3091 3092 3093 3094 3095
			bio->bi_next = req->bio;
			req->bio = bio;

			/*
			 * may not be valid. if the low level driver said
			 * it didn't need a bounce buffer then it better
			 * not touch req->buffer either...
			 */
			req->buffer = bio_data(bio);
3096 3097 3098
			req->current_nr_sectors = bio_cur_sectors(bio);
			req->hard_cur_sectors = req->current_nr_sectors;
			req->sector = req->hard_sector = bio->bi_sector;
L
Linus Torvalds 已提交
3099
			req->nr_sectors = req->hard_nr_sectors += nr_sectors;
3100
			req->ioprio = ioprio_best(req->ioprio, prio);
3101
			drive_stat_acct(req, 0);
L
Linus Torvalds 已提交
3102
			if (!attempt_front_merge(q, req))
3103
				elv_merged_request(q, req, el_ret);
L
Linus Torvalds 已提交
3104 3105
			goto out;

3106
		/* ELV_NO_MERGE: elevator says don't/can't merge. */
L
Linus Torvalds 已提交
3107
		default:
3108
			;
L
Linus Torvalds 已提交
3109 3110
	}

3111
get_rq:
3112 3113 3114 3115 3116 3117 3118 3119 3120
	/*
	 * This sync check and mask will be re-done in init_request_from_bio(),
	 * but we need to set it earlier to expose the sync flag to the
	 * rq allocator and io schedulers.
	 */
	rw_flags = bio_data_dir(bio);
	if (sync)
		rw_flags |= REQ_RW_SYNC;

L
Linus Torvalds 已提交
3121
	/*
3122
	 * Grab a free request. This is might sleep but can not fail.
N
Nick Piggin 已提交
3123
	 * Returns with the queue unlocked.
3124
	 */
3125
	req = get_request_wait(q, rw_flags, bio);
N
Nick Piggin 已提交
3126

3127 3128 3129 3130 3131
	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
L
Linus Torvalds 已提交
3132
	 */
3133
	init_request_from_bio(req, bio);
L
Linus Torvalds 已提交
3134

3135 3136 3137
	spin_lock_irq(q->queue_lock);
	if (elv_queue_empty(q))
		blk_plug_device(q);
L
Linus Torvalds 已提交
3138 3139
	add_request(q, req);
out:
3140
	if (sync)
L
Linus Torvalds 已提交
3141 3142 3143 3144 3145 3146
		__generic_unplug_device(q);

	spin_unlock_irq(q->queue_lock);
	return 0;

end_io:
3147
	bio_endio(bio, err);
L
Linus Torvalds 已提交
3148 3149 3150 3151 3152 3153 3154 3155 3156 3157
	return 0;
}

/*
 * If bio->bi_dev is a partition, remap the location
 */
static inline void blk_partition_remap(struct bio *bio)
{
	struct block_device *bdev = bio->bi_bdev;

3158
	if (bio_sectors(bio) && bdev != bdev->bd_contains) {
L
Linus Torvalds 已提交
3159
		struct hd_struct *p = bdev->bd_part;
3160 3161 3162 3163
		const int rw = bio_data_dir(bio);

		p->sectors[rw] += bio_sectors(bio);
		p->ios[rw]++;
L
Linus Torvalds 已提交
3164 3165 3166

		bio->bi_sector += p->start_sect;
		bio->bi_bdev = bdev->bd_contains;
3167 3168 3169 3170

		blk_add_trace_remap(bdev_get_queue(bio->bi_bdev), bio,
				    bdev->bd_dev, bio->bi_sector,
				    bio->bi_sector - p->start_sect);
L
Linus Torvalds 已提交
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187
	}
}

static void handle_bad_sector(struct bio *bio)
{
	char b[BDEVNAME_SIZE];

	printk(KERN_INFO "attempt to access beyond end of device\n");
	printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
			bdevname(bio->bi_bdev, b),
			bio->bi_rw,
			(unsigned long long)bio->bi_sector + bio_sectors(bio),
			(long long)(bio->bi_bdev->bd_inode->i_size >> 9));

	set_bit(BIO_EOF, &bio->bi_flags);
}

3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223
#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
	return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

static int should_fail_request(struct bio *bio)
{
	if ((bio->bi_bdev->bd_disk->flags & GENHD_FL_FAIL) ||
	    (bio->bi_bdev->bd_part && bio->bi_bdev->bd_part->make_it_fail))
		return should_fail(&fail_make_request, bio->bi_size);

	return 0;
}

static int __init fail_make_request_debugfs(void)
{
	return init_fault_attr_dentries(&fail_make_request,
					"fail_make_request");
}

late_initcall(fail_make_request_debugfs);

#else /* CONFIG_FAIL_MAKE_REQUEST */

static inline int should_fail_request(struct bio *bio)
{
	return 0;
}

#endif /* CONFIG_FAIL_MAKE_REQUEST */

J
Jens Axboe 已提交
3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252
/*
 * Check whether this bio extends beyond the end of the device.
 */
static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
{
	sector_t maxsector;

	if (!nr_sectors)
		return 0;

	/* Test device or partition size, when known. */
	maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
	if (maxsector) {
		sector_t sector = bio->bi_sector;

		if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
			/*
			 * This may well happen - the kernel calls bread()
			 * without checking the size of the device, e.g., when
			 * mounting a device.
			 */
			handle_bad_sector(bio);
			return 1;
		}
	}

	return 0;
}

L
Linus Torvalds 已提交
3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276
/**
 * generic_make_request: hand a buffer to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * generic_make_request() is used to make I/O requests of block
 * devices. It is passed a &struct bio, which describes the I/O that needs
 * to be done.
 *
 * generic_make_request() does not return any status.  The
 * success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the bio->bi_end_io
 * function described (one day) else where.
 *
 * The caller of generic_make_request must make sure that bi_io_vec
 * are set to describe the memory buffer, and that bi_dev and bi_sector are
 * set to describe the device address, and the
 * bi_end_io and optionally bi_private are set to describe how
 * completion notification should be signaled.
 *
 * generic_make_request and the drivers it calls may use bi_next if this
 * bio happens to be merged with someone else, and may change bi_dev and
 * bi_sector for remaps as it sees fit.  So the values of these fields
 * should NOT be depended on after the call to generic_make_request.
 */
3277
static inline void __generic_make_request(struct bio *bio)
L
Linus Torvalds 已提交
3278
{
3279
	struct request_queue *q;
3280
	sector_t old_sector;
L
Linus Torvalds 已提交
3281
	int ret, nr_sectors = bio_sectors(bio);
3282
	dev_t old_dev;
3283
	int err = -EIO;
L
Linus Torvalds 已提交
3284 3285 3286

	might_sleep();

J
Jens Axboe 已提交
3287 3288
	if (bio_check_eod(bio, nr_sectors))
		goto end_io;
L
Linus Torvalds 已提交
3289 3290 3291 3292 3293 3294 3295 3296 3297

	/*
	 * Resolve the mapping until finished. (drivers are
	 * still free to implement/resolve their own stacking
	 * by explicitly returning 0)
	 *
	 * NOTE: we don't repeat the blk_size check for each new device.
	 * Stacking drivers are expected to know what they are doing.
	 */
3298
	old_sector = -1;
3299
	old_dev = 0;
L
Linus Torvalds 已提交
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310
	do {
		char b[BDEVNAME_SIZE];

		q = bdev_get_queue(bio->bi_bdev);
		if (!q) {
			printk(KERN_ERR
			       "generic_make_request: Trying to access "
				"nonexistent block-device %s (%Lu)\n",
				bdevname(bio->bi_bdev, b),
				(long long) bio->bi_sector);
end_io:
3311
			bio_endio(bio, err);
L
Linus Torvalds 已提交
3312 3313 3314
			break;
		}

J
Jens Axboe 已提交
3315
		if (unlikely(nr_sectors > q->max_hw_sectors)) {
L
Linus Torvalds 已提交
3316 3317 3318 3319 3320 3321 3322
			printk("bio too big device %s (%u > %u)\n", 
				bdevname(bio->bi_bdev, b),
				bio_sectors(bio),
				q->max_hw_sectors);
			goto end_io;
		}

N
Nick Piggin 已提交
3323
		if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
L
Linus Torvalds 已提交
3324 3325
			goto end_io;

3326 3327 3328
		if (should_fail_request(bio))
			goto end_io;

L
Linus Torvalds 已提交
3329 3330 3331 3332 3333 3334
		/*
		 * If this device has partitions, remap block n
		 * of partition p to block n+start(p) of the disk.
		 */
		blk_partition_remap(bio);

3335
		if (old_sector != -1)
J
Jens Axboe 已提交
3336
			blk_add_trace_remap(q, bio, old_dev, bio->bi_sector,
3337
					    old_sector);
3338 3339 3340

		blk_add_trace_bio(q, bio, BLK_TA_QUEUE);

3341
		old_sector = bio->bi_sector;
3342 3343
		old_dev = bio->bi_bdev->bd_dev;

J
Jens Axboe 已提交
3344 3345
		if (bio_check_eod(bio, nr_sectors))
			goto end_io;
3346 3347 3348 3349
		if (bio_empty_barrier(bio) && !q->prepare_flush_fn) {
			err = -EOPNOTSUPP;
			goto end_io;
		}
3350

L
Linus Torvalds 已提交
3351 3352 3353 3354
		ret = q->make_request_fn(q, bio);
	} while (ret);
}

3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405
/*
 * We only want one ->make_request_fn to be active at a time,
 * else stack usage with stacked devices could be a problem.
 * So use current->bio_{list,tail} to keep a list of requests
 * submited by a make_request_fn function.
 * current->bio_tail is also used as a flag to say if
 * generic_make_request is currently active in this task or not.
 * If it is NULL, then no make_request is active.  If it is non-NULL,
 * then a make_request is active, and new requests should be added
 * at the tail
 */
void generic_make_request(struct bio *bio)
{
	if (current->bio_tail) {
		/* make_request is active */
		*(current->bio_tail) = bio;
		bio->bi_next = NULL;
		current->bio_tail = &bio->bi_next;
		return;
	}
	/* following loop may be a bit non-obvious, and so deserves some
	 * explanation.
	 * Before entering the loop, bio->bi_next is NULL (as all callers
	 * ensure that) so we have a list with a single bio.
	 * We pretend that we have just taken it off a longer list, so
	 * we assign bio_list to the next (which is NULL) and bio_tail
	 * to &bio_list, thus initialising the bio_list of new bios to be
	 * added.  __generic_make_request may indeed add some more bios
	 * through a recursive call to generic_make_request.  If it
	 * did, we find a non-NULL value in bio_list and re-enter the loop
	 * from the top.  In this case we really did just take the bio
	 * of the top of the list (no pretending) and so fixup bio_list and
	 * bio_tail or bi_next, and call into __generic_make_request again.
	 *
	 * The loop was structured like this to make only one call to
	 * __generic_make_request (which is important as it is large and
	 * inlined) and to keep the structure simple.
	 */
	BUG_ON(bio->bi_next);
	do {
		current->bio_list = bio->bi_next;
		if (bio->bi_next == NULL)
			current->bio_tail = &current->bio_list;
		else
			bio->bi_next = NULL;
		__generic_make_request(bio);
		bio = current->bio_list;
	} while (bio);
	current->bio_tail = NULL; /* deactivate */
}

L
Linus Torvalds 已提交
3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421
EXPORT_SYMBOL(generic_make_request);

/**
 * submit_bio: submit a bio to the block device layer for I/O
 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is very similar in purpose to generic_make_request(), and
 * uses that function to do most of the work. Both are fairly rough
 * interfaces, @bio must be presetup and ready for I/O.
 *
 */
void submit_bio(int rw, struct bio *bio)
{
	int count = bio_sectors(bio);

3422
	bio->bi_rw |= rw;
L
Linus Torvalds 已提交
3423

3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442
	/*
	 * If it's a regular read/write or a barrier with data attached,
	 * go through the normal accounting stuff before submission.
	 */
	if (!bio_empty_barrier(bio)) {

		BIO_BUG_ON(!bio->bi_size);
		BIO_BUG_ON(!bio->bi_io_vec);

		if (rw & WRITE) {
			count_vm_events(PGPGOUT, count);
		} else {
			task_io_account_read(bio->bi_size);
			count_vm_events(PGPGIN, count);
		}

		if (unlikely(block_dump)) {
			char b[BDEVNAME_SIZE];
			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
3443
			current->comm, task_pid_nr(current),
3444 3445 3446 3447
				(rw & WRITE) ? "WRITE" : "READ",
				(unsigned long long)bio->bi_sector,
				bdevname(bio->bi_bdev,b));
		}
L
Linus Torvalds 已提交
3448 3449 3450 3451 3452 3453 3454
	}

	generic_make_request(bio);
}

EXPORT_SYMBOL(submit_bio);

3455
static void blk_recalc_rq_sectors(struct request *rq, int nsect)
L
Linus Torvalds 已提交
3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483
{
	if (blk_fs_request(rq)) {
		rq->hard_sector += nsect;
		rq->hard_nr_sectors -= nsect;

		/*
		 * Move the I/O submission pointers ahead if required.
		 */
		if ((rq->nr_sectors >= rq->hard_nr_sectors) &&
		    (rq->sector <= rq->hard_sector)) {
			rq->sector = rq->hard_sector;
			rq->nr_sectors = rq->hard_nr_sectors;
			rq->hard_cur_sectors = bio_cur_sectors(rq->bio);
			rq->current_nr_sectors = rq->hard_cur_sectors;
			rq->buffer = bio_data(rq->bio);
		}

		/*
		 * if total number of sectors is less than the first segment
		 * size, something has gone terribly wrong
		 */
		if (rq->nr_sectors < rq->current_nr_sectors) {
			printk("blk: request botched\n");
			rq->nr_sectors = rq->current_nr_sectors;
		}
	}
}

3484 3485 3486
/**
 * __end_that_request_first - end I/O on a request
 * @req:      the request being processed
3487
 * @error:    0 for success, < 0 for error
3488 3489 3490 3491 3492 3493 3494 3495 3496 3497
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @req, and sets it up
 *     for the next range of segments (if any) in the cluster.
 *
 * Return:
 *     0 - we are done with this request, call end_that_request_last()
 *     1 - still buffers pending for this request
 **/
3498
static int __end_that_request_first(struct request *req, int error,
L
Linus Torvalds 已提交
3499 3500
				    int nr_bytes)
{
3501
	int total_bytes, bio_nbytes, next_idx = 0;
L
Linus Torvalds 已提交
3502 3503
	struct bio *bio;

3504 3505
	blk_add_trace_rq(req->q, req, BLK_TA_COMPLETE);

L
Linus Torvalds 已提交
3506 3507 3508 3509 3510 3511 3512
	/*
	 * for a REQ_BLOCK_PC request, we want to carry any eventual
	 * sense key with us all the way through
	 */
	if (!blk_pc_request(req))
		req->errors = 0;

3513
	if (error) {
3514
		if (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))
L
Linus Torvalds 已提交
3515 3516 3517 3518 3519
			printk("end_request: I/O error, dev %s, sector %llu\n",
				req->rq_disk ? req->rq_disk->disk_name : "?",
				(unsigned long long)req->sector);
	}

3520
	if (blk_fs_request(req) && req->rq_disk) {
3521 3522
		const int rw = rq_data_dir(req);

3523
		disk_stat_add(req->rq_disk, sectors[rw], nr_bytes >> 9);
3524 3525
	}

L
Linus Torvalds 已提交
3526 3527 3528 3529
	total_bytes = bio_nbytes = 0;
	while ((bio = req->bio) != NULL) {
		int nbytes;

3530 3531 3532 3533 3534 3535 3536 3537
		/*
		 * For an empty barrier request, the low level driver must
		 * store a potential error location in ->sector. We pass
		 * that back up in ->bi_sector.
		 */
		if (blk_empty_barrier(req))
			bio->bi_sector = req->sector;

L
Linus Torvalds 已提交
3538 3539 3540
		if (nr_bytes >= bio->bi_size) {
			req->bio = bio->bi_next;
			nbytes = bio->bi_size;
N
NeilBrown 已提交
3541
			req_bio_endio(req, bio, nbytes, error);
L
Linus Torvalds 已提交
3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595
			next_idx = 0;
			bio_nbytes = 0;
		} else {
			int idx = bio->bi_idx + next_idx;

			if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
				blk_dump_rq_flags(req, "__end_that");
				printk("%s: bio idx %d >= vcnt %d\n",
						__FUNCTION__,
						bio->bi_idx, bio->bi_vcnt);
				break;
			}

			nbytes = bio_iovec_idx(bio, idx)->bv_len;
			BIO_BUG_ON(nbytes > bio->bi_size);

			/*
			 * not a complete bvec done
			 */
			if (unlikely(nbytes > nr_bytes)) {
				bio_nbytes += nr_bytes;
				total_bytes += nr_bytes;
				break;
			}

			/*
			 * advance to the next vector
			 */
			next_idx++;
			bio_nbytes += nbytes;
		}

		total_bytes += nbytes;
		nr_bytes -= nbytes;

		if ((bio = req->bio)) {
			/*
			 * end more in this run, or just return 'not-done'
			 */
			if (unlikely(nr_bytes <= 0))
				break;
		}
	}

	/*
	 * completely done
	 */
	if (!req->bio)
		return 0;

	/*
	 * if the request wasn't completed, update state
	 */
	if (bio_nbytes) {
N
NeilBrown 已提交
3596
		req_bio_endio(req, bio, bio_nbytes, error);
L
Linus Torvalds 已提交
3597 3598 3599 3600 3601 3602 3603 3604 3605 3606
		bio->bi_idx += next_idx;
		bio_iovec(bio)->bv_offset += nr_bytes;
		bio_iovec(bio)->bv_len -= nr_bytes;
	}

	blk_recalc_rq_sectors(req, total_bytes >> 9);
	blk_recalc_rq_segments(req);
	return 1;
}

3607 3608 3609 3610 3611 3612
/*
 * splice the completion data to a local structure and hand off to
 * process_completion_queue() to complete the requests
 */
static void blk_done_softirq(struct softirq_action *h)
{
3613
	struct list_head *cpu_list, local_list;
3614 3615 3616

	local_irq_disable();
	cpu_list = &__get_cpu_var(blk_cpu_done);
3617
	list_replace_init(cpu_list, &local_list);
3618 3619 3620 3621 3622 3623 3624 3625 3626 3627
	local_irq_enable();

	while (!list_empty(&local_list)) {
		struct request *rq = list_entry(local_list.next, struct request, donelist);

		list_del_init(&rq->donelist);
		rq->q->softirq_done_fn(rq);
	}
}

3628
static int __cpuinit blk_cpu_notify(struct notifier_block *self, unsigned long action,
3629 3630 3631 3632 3633 3634
			  void *hcpu)
{
	/*
	 * If a CPU goes away, splice its entries to the current CPU
	 * and trigger a run of the softirq
	 */
3635
	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648
		int cpu = (unsigned long) hcpu;

		local_irq_disable();
		list_splice_init(&per_cpu(blk_cpu_done, cpu),
				 &__get_cpu_var(blk_cpu_done));
		raise_softirq_irqoff(BLOCK_SOFTIRQ);
		local_irq_enable();
	}

	return NOTIFY_OK;
}


3649
static struct notifier_block blk_cpu_notifier __cpuinitdata = {
3650 3651 3652 3653 3654 3655 3656 3657 3658
	.notifier_call	= blk_cpu_notify,
};

/**
 * blk_complete_request - end I/O on a request
 * @req:      the request being processed
 *
 * Description:
 *     Ends all I/O on a request. It does not handle partial completions,
A
Andreas Mohr 已提交
3659
 *     unless the driver actually implements this in its completion callback
J
Jens Axboe 已提交
3660
 *     through requeueing. The actual completion happens out-of-order,
3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682
 *     through a softirq handler. The user must have registered a completion
 *     callback through blk_queue_softirq_done().
 **/

void blk_complete_request(struct request *req)
{
	struct list_head *cpu_list;
	unsigned long flags;

	BUG_ON(!req->q->softirq_done_fn);
		
	local_irq_save(flags);

	cpu_list = &__get_cpu_var(blk_cpu_done);
	list_add_tail(&req->donelist, cpu_list);
	raise_softirq_irqoff(BLOCK_SOFTIRQ);

	local_irq_restore(flags);
}

EXPORT_SYMBOL(blk_complete_request);
	
L
Linus Torvalds 已提交
3683 3684 3685
/*
 * queue lock must be held
 */
3686
static void end_that_request_last(struct request *req, int error)
L
Linus Torvalds 已提交
3687 3688
{
	struct gendisk *disk = req->rq_disk;
3689

3690 3691 3692 3693 3694
	if (blk_rq_tagged(req))
		blk_queue_end_tag(req->q, req);

	if (blk_queued_rq(req))
		blkdev_dequeue_request(req);
L
Linus Torvalds 已提交
3695 3696 3697 3698

	if (unlikely(laptop_mode) && blk_fs_request(req))
		laptop_io_completion();

3699 3700 3701 3702 3703 3704
	/*
	 * Account IO completion.  bar_rq isn't accounted as a normal
	 * IO on queueing nor completion.  Accounting the containing
	 * request is enough.
	 */
	if (disk && blk_fs_request(req) && req != &req->q->bar_rq) {
L
Linus Torvalds 已提交
3705
		unsigned long duration = jiffies - req->start_time;
3706 3707 3708 3709
		const int rw = rq_data_dir(req);

		__disk_stat_inc(disk, ios[rw]);
		__disk_stat_add(disk, ticks[rw], duration);
L
Linus Torvalds 已提交
3710 3711 3712
		disk_round_stats(disk);
		disk->in_flight--;
	}
3713

L
Linus Torvalds 已提交
3714
	if (req->end_io)
3715
		req->end_io(req, error);
3716 3717 3718 3719
	else {
		if (blk_bidi_rq(req))
			__blk_put_request(req->next_rq->q, req->next_rq);

L
Linus Torvalds 已提交
3720
		__blk_put_request(req->q, req);
3721
	}
L
Linus Torvalds 已提交
3722 3723
}

3724
static inline void __end_request(struct request *rq, int uptodate,
3725
				 unsigned int nr_bytes)
L
Linus Torvalds 已提交
3726
{
3727 3728 3729 3730 3731 3732
	int error = 0;

	if (uptodate <= 0)
		error = uptodate ? uptodate : -EIO;

	__blk_end_request(rq, error, nr_bytes);
L
Linus Torvalds 已提交
3733 3734
}

3735 3736 3737 3738
/**
 * blk_rq_bytes - Returns bytes left to complete in the entire request
 **/
unsigned int blk_rq_bytes(struct request *rq)
3739 3740 3741 3742 3743 3744
{
	if (blk_fs_request(rq))
		return rq->hard_nr_sectors << 9;

	return rq->data_len;
}
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760
EXPORT_SYMBOL_GPL(blk_rq_bytes);

/**
 * blk_rq_cur_bytes - Returns bytes left to complete in the current segment
 **/
unsigned int blk_rq_cur_bytes(struct request *rq)
{
	if (blk_fs_request(rq))
		return rq->current_nr_sectors << 9;

	if (rq->bio)
		return rq->bio->bi_size;

	return rq->data_len;
}
EXPORT_SYMBOL_GPL(blk_rq_cur_bytes);
3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774

/**
 * end_queued_request - end all I/O on a queued request
 * @rq:		the request being processed
 * @uptodate:	error value or 0/1 uptodate flag
 *
 * Description:
 *     Ends all I/O on a request, and removes it from the block layer queues.
 *     Not suitable for normal IO completion, unless the driver still has
 *     the request attached to the block layer.
 *
 **/
void end_queued_request(struct request *rq, int uptodate)
{
3775
	__end_request(rq, uptodate, blk_rq_bytes(rq));
3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791
}
EXPORT_SYMBOL(end_queued_request);

/**
 * end_dequeued_request - end all I/O on a dequeued request
 * @rq:		the request being processed
 * @uptodate:	error value or 0/1 uptodate flag
 *
 * Description:
 *     Ends all I/O on a request. The request must already have been
 *     dequeued using blkdev_dequeue_request(), as is normally the case
 *     for most drivers.
 *
 **/
void end_dequeued_request(struct request *rq, int uptodate)
{
3792
	__end_request(rq, uptodate, blk_rq_bytes(rq));
3793 3794 3795 3796 3797 3798
}
EXPORT_SYMBOL(end_dequeued_request);


/**
 * end_request - end I/O on the current segment of the request
3799
 * @req:	the request being processed
3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817
 * @uptodate:	error value or 0/1 uptodate flag
 *
 * Description:
 *     Ends I/O on the current segment of a request. If that is the only
 *     remaining segment, the request is also completed and freed.
 *
 *     This is a remnant of how older block drivers handled IO completions.
 *     Modern drivers typically end IO on the full request in one go, unless
 *     they have a residual value to account for. For that case this function
 *     isn't really useful, unless the residual just happens to be the
 *     full current segment. In other words, don't use this function in new
 *     code. Either use end_request_completely(), or the
 *     end_that_request_chunk() (along with end_that_request_last()) for
 *     partial completions.
 *
 **/
void end_request(struct request *req, int uptodate)
{
3818
	__end_request(req, uptodate, req->hard_cur_sectors << 9);
3819
}
L
Linus Torvalds 已提交
3820 3821
EXPORT_SYMBOL(end_request);

3822
/**
3823 3824 3825
 * blk_end_io - Generic end_io function to complete a request.
 * @rq:           the request being processed
 * @error:        0 for success, < 0 for error
3826 3827
 * @nr_bytes:     number of bytes to complete @rq
 * @bidi_bytes:   number of bytes to complete @rq->next_rq
3828 3829 3830 3831
 * @drv_callback: function called between completion of bios in the request
 *                and completion of the request.
 *                If the callback returns non 0, this helper returns without
 *                completion of the request.
3832 3833
 *
 * Description:
3834
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3835 3836 3837 3838
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
 *     0 - we are done with this request
3839
 *     1 - this request is not freed yet, it still has pending buffers.
3840
 **/
3841
static int blk_end_io(struct request *rq, int error, int nr_bytes,
3842
		      int bidi_bytes, int (drv_callback)(struct request *))
3843 3844 3845 3846 3847
{
	struct request_queue *q = rq->q;
	unsigned long flags = 0UL;

	if (blk_fs_request(rq) || blk_pc_request(rq)) {
3848
		if (__end_that_request_first(rq, error, nr_bytes))
3849
			return 1;
3850 3851 3852

		/* Bidi request must be completed as a whole */
		if (blk_bidi_rq(rq) &&
3853
		    __end_that_request_first(rq->next_rq, error, bidi_bytes))
3854
			return 1;
3855 3856
	}

3857 3858 3859 3860
	/* Special feature for tricky drivers */
	if (drv_callback && drv_callback(rq))
		return 1;

3861 3862 3863
	add_disk_randomness(rq->rq_disk);

	spin_lock_irqsave(q->queue_lock, flags);
3864
	end_that_request_last(rq, error);
3865 3866 3867 3868
	spin_unlock_irqrestore(q->queue_lock, flags);

	return 0;
}
3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885

/**
 * blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
 * @error:    0 for success, < 0 for error
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
 *     0 - we are done with this request
 *     1 - still buffers pending for this request
 **/
int blk_end_request(struct request *rq, int error, int nr_bytes)
{
3886
	return blk_end_io(rq, error, nr_bytes, 0, NULL);
3887
}
3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905
EXPORT_SYMBOL_GPL(blk_end_request);

/**
 * __blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
 * @error:    0 for success, < 0 for error
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Must be called with queue lock held unlike blk_end_request().
 *
 * Return:
 *     0 - we are done with this request
 *     1 - still buffers pending for this request
 **/
int __blk_end_request(struct request *rq, int error, int nr_bytes)
{
	if (blk_fs_request(rq) || blk_pc_request(rq)) {
3906
		if (__end_that_request_first(rq, error, nr_bytes))
3907 3908 3909 3910 3911
			return 1;
	}

	add_disk_randomness(rq->rq_disk);

3912
	end_that_request_last(rq, error);
3913 3914 3915 3916 3917

	return 0;
}
EXPORT_SYMBOL_GPL(__blk_end_request);

3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938
/**
 * blk_end_bidi_request - Helper function for drivers to complete bidi request.
 * @rq:         the bidi request being processed
 * @error:      0 for success, < 0 for error
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
 *
 * Return:
 *     0 - we are done with this request
 *     1 - still buffers pending for this request
 **/
int blk_end_bidi_request(struct request *rq, int error, int nr_bytes,
			 int bidi_bytes)
{
	return blk_end_io(rq, error, nr_bytes, bidi_bytes, NULL);
}
EXPORT_SYMBOL_GPL(blk_end_bidi_request);

3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966
/**
 * blk_end_request_callback - Special helper function for tricky drivers
 * @rq:           the request being processed
 * @error:        0 for success, < 0 for error
 * @nr_bytes:     number of bytes to complete
 * @drv_callback: function called between completion of bios in the request
 *                and completion of the request.
 *                If the callback returns non 0, this helper returns without
 *                completion of the request.
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 *     This special helper function is used only for existing tricky drivers.
 *     (e.g. cdrom_newpc_intr() of ide-cd)
 *     This interface will be removed when such drivers are rewritten.
 *     Don't use this interface in other places anymore.
 *
 * Return:
 *     0 - we are done with this request
 *     1 - this request is not freed yet.
 *         this request still has pending buffers or
 *         the driver doesn't want to finish this request yet.
 **/
int blk_end_request_callback(struct request *rq, int error, int nr_bytes,
			     int (drv_callback)(struct request *))
{
3967
	return blk_end_io(rq, error, nr_bytes, 0, drv_callback);
3968 3969 3970
}
EXPORT_SYMBOL_GPL(blk_end_request_callback);

N
NeilBrown 已提交
3971 3972
static void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
			    struct bio *bio)
L
Linus Torvalds 已提交
3973
{
3974 3975
	/* first two bits are identical in rq->cmd_flags and bio->bi_rw */
	rq->cmd_flags |= (bio->bi_rw & 3);
L
Linus Torvalds 已提交
3976 3977 3978 3979 3980 3981 3982

	rq->nr_phys_segments = bio_phys_segments(q, bio);
	rq->nr_hw_segments = bio_hw_segments(q, bio);
	rq->current_nr_sectors = bio_cur_sectors(bio);
	rq->hard_cur_sectors = rq->current_nr_sectors;
	rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
	rq->buffer = bio_data(bio);
3983
	rq->data_len = bio->bi_size;
L
Linus Torvalds 已提交
3984 3985 3986

	rq->bio = rq->biotail = bio;

N
NeilBrown 已提交
3987 3988 3989
	if (bio->bi_bdev)
		rq->rq_disk = bio->bi_bdev->bd_disk;
}
L
Linus Torvalds 已提交
3990 3991 3992 3993 3994 3995 3996 3997

int kblockd_schedule_work(struct work_struct *work)
{
	return queue_work(kblockd_workqueue, work);
}

EXPORT_SYMBOL(kblockd_schedule_work);

A
Andrew Morton 已提交
3998
void kblockd_flush_work(struct work_struct *work)
L
Linus Torvalds 已提交
3999
{
4000
	cancel_work_sync(work);
L
Linus Torvalds 已提交
4001
}
A
Andrew Morton 已提交
4002
EXPORT_SYMBOL(kblockd_flush_work);
L
Linus Torvalds 已提交
4003 4004 4005

int __init blk_dev_init(void)
{
4006 4007
	int i;

L
Linus Torvalds 已提交
4008 4009 4010 4011 4012
	kblockd_workqueue = create_workqueue("kblockd");
	if (!kblockd_workqueue)
		panic("Failed to create kblockd\n");

	request_cachep = kmem_cache_create("blkdev_requests",
4013
			sizeof(struct request), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
4014 4015

	requestq_cachep = kmem_cache_create("blkdev_queue",
4016
			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
4017 4018

	iocontext_cachep = kmem_cache_create("blkdev_ioc",
4019
			sizeof(struct io_context), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
4020

4021
	for_each_possible_cpu(i)
4022 4023 4024
		INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));

	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL);
4025
	register_hotcpu_notifier(&blk_cpu_notifier);
4026

4027 4028
	blk_max_low_pfn = max_low_pfn - 1;
	blk_max_pfn = max_pfn - 1;
L
Linus Torvalds 已提交
4029 4030 4031 4032

	return 0;
}

4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047
static void cfq_dtor(struct io_context *ioc)
{
	struct cfq_io_context *cic[1];
	int r;

	/*
	 * We don't have a specific key to lookup with, so use the gang
	 * lookup to just retrieve the first item stored. The cfq exit
	 * function will iterate the full tree, so any member will do.
	 */
	r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
	if (r > 0)
		cic[0]->dtor(ioc);
}

L
Linus Torvalds 已提交
4048
/*
4049 4050
 * IO Context helper functions. put_io_context() returns 1 if there are no
 * more users of this io context, 0 otherwise.
L
Linus Torvalds 已提交
4051
 */
4052
int put_io_context(struct io_context *ioc)
L
Linus Torvalds 已提交
4053 4054
{
	if (ioc == NULL)
4055
		return 1;
L
Linus Torvalds 已提交
4056 4057 4058 4059

	BUG_ON(atomic_read(&ioc->refcount) == 0);

	if (atomic_dec_and_test(&ioc->refcount)) {
4060
		rcu_read_lock();
L
Linus Torvalds 已提交
4061 4062
		if (ioc->aic && ioc->aic->dtor)
			ioc->aic->dtor(ioc->aic);
4063
		rcu_read_unlock();
4064
		cfq_dtor(ioc);
L
Linus Torvalds 已提交
4065 4066

		kmem_cache_free(iocontext_cachep, ioc);
4067
		return 1;
L
Linus Torvalds 已提交
4068
	}
4069
	return 0;
L
Linus Torvalds 已提交
4070 4071 4072
}
EXPORT_SYMBOL(put_io_context);

4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088
static void cfq_exit(struct io_context *ioc)
{
	struct cfq_io_context *cic[1];
	int r;

	rcu_read_lock();
	/*
	 * See comment for cfq_dtor()
	 */
	r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
	rcu_read_unlock();

	if (r > 0)
		cic[0]->exit(ioc);
}

L
Linus Torvalds 已提交
4089 4090 4091 4092 4093
/* Called by the exitting task */
void exit_io_context(void)
{
	struct io_context *ioc;

4094
	task_lock(current);
L
Linus Torvalds 已提交
4095 4096
	ioc = current->io_context;
	current->io_context = NULL;
4097
	task_unlock(current);
L
Linus Torvalds 已提交
4098

4099 4100 4101
	if (atomic_dec_and_test(&ioc->nr_tasks)) {
		if (ioc->aic && ioc->aic->exit)
			ioc->aic->exit(ioc->aic);
4102
		cfq_exit(ioc);
4103

4104 4105
		put_io_context(ioc);
	}
L
Linus Torvalds 已提交
4106 4107
}

4108 4109 4110 4111 4112 4113 4114
struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
{
	struct io_context *ret;

	ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
	if (ret) {
		atomic_set(&ret->refcount, 1);
4115 4116
		atomic_set(&ret->nr_tasks, 1);
		spin_lock_init(&ret->lock);
4117 4118 4119 4120 4121
		ret->ioprio_changed = 0;
		ret->ioprio = 0;
		ret->last_waited = jiffies; /* doesn't matter... */
		ret->nr_batch_requests = 0; /* because this is 0 */
		ret->aic = NULL;
4122
		INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
4123 4124 4125 4126 4127 4128
		ret->ioc_data = NULL;
	}

	return ret;
}

L
Linus Torvalds 已提交
4129 4130
/*
 * If the current task has no IO context then create one and initialise it.
N
Nick Piggin 已提交
4131
 * Otherwise, return its existing IO context.
L
Linus Torvalds 已提交
4132
 *
N
Nick Piggin 已提交
4133 4134 4135
 * This returned IO context doesn't have a specifically elevated refcount,
 * but since the current task itself holds a reference, the context can be
 * used in general code, so long as it stays within `current` context.
L
Linus Torvalds 已提交
4136
 */
4137
static struct io_context *current_io_context(gfp_t gfp_flags, int node)
L
Linus Torvalds 已提交
4138 4139 4140 4141 4142
{
	struct task_struct *tsk = current;
	struct io_context *ret;

	ret = tsk->io_context;
N
Nick Piggin 已提交
4143 4144
	if (likely(ret))
		return ret;
L
Linus Torvalds 已提交
4145

4146
	ret = alloc_io_context(gfp_flags, node);
L
Linus Torvalds 已提交
4147
	if (ret) {
4148 4149
		/* make sure set_task_ioprio() sees the settings above */
		smp_wmb();
N
Nick Piggin 已提交
4150 4151
		tsk->io_context = ret;
	}
L
Linus Torvalds 已提交
4152

N
Nick Piggin 已提交
4153 4154
	return ret;
}
L
Linus Torvalds 已提交
4155

N
Nick Piggin 已提交
4156 4157 4158 4159 4160 4161
/*
 * If the current task has no IO context then create one and initialise it.
 * If it does have a context, take a ref on it.
 *
 * This is always called in the context of the task which submitted the I/O.
 */
4162
struct io_context *get_io_context(gfp_t gfp_flags, int node)
N
Nick Piggin 已提交
4163
{
4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175
	struct io_context *ret = NULL;

	/*
	 * Check for unlikely race with exiting task. ioc ref count is
	 * zero when ioc is being detached.
	 */
	do {
		ret = current_io_context(gfp_flags, node);
		if (unlikely(!ret))
			break;
	} while (!atomic_inc_not_zero(&ret->refcount));

L
Linus Torvalds 已提交
4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235
	return ret;
}
EXPORT_SYMBOL(get_io_context);

void copy_io_context(struct io_context **pdst, struct io_context **psrc)
{
	struct io_context *src = *psrc;
	struct io_context *dst = *pdst;

	if (src) {
		BUG_ON(atomic_read(&src->refcount) == 0);
		atomic_inc(&src->refcount);
		put_io_context(dst);
		*pdst = src;
	}
}
EXPORT_SYMBOL(copy_io_context);

void swap_io_context(struct io_context **ioc1, struct io_context **ioc2)
{
	struct io_context *temp;
	temp = *ioc1;
	*ioc1 = *ioc2;
	*ioc2 = temp;
}
EXPORT_SYMBOL(swap_io_context);

/*
 * sysfs parts below
 */
struct queue_sysfs_entry {
	struct attribute attr;
	ssize_t (*show)(struct request_queue *, char *);
	ssize_t (*store)(struct request_queue *, const char *, size_t);
};

static ssize_t
queue_var_show(unsigned int var, char *page)
{
	return sprintf(page, "%d\n", var);
}

static ssize_t
queue_var_store(unsigned long *var, const char *page, size_t count)
{
	char *p = (char *) page;

	*var = simple_strtoul(p, &p, 10);
	return count;
}

static ssize_t queue_requests_show(struct request_queue *q, char *page)
{
	return queue_var_show(q->nr_requests, (page));
}

static ssize_t
queue_requests_store(struct request_queue *q, const char *page, size_t count)
{
	struct request_list *rl = &q->rq;
4236 4237 4238 4239
	unsigned long nr;
	int ret = queue_var_store(&nr, page, count);
	if (nr < BLKDEV_MIN_RQ)
		nr = BLKDEV_MIN_RQ;
L
Linus Torvalds 已提交
4240

4241 4242
	spin_lock_irq(q->queue_lock);
	q->nr_requests = nr;
L
Linus Torvalds 已提交
4243 4244 4245
	blk_queue_congestion_threshold(q);

	if (rl->count[READ] >= queue_congestion_on_threshold(q))
4246
		blk_set_queue_congested(q, READ);
L
Linus Torvalds 已提交
4247
	else if (rl->count[READ] < queue_congestion_off_threshold(q))
4248
		blk_clear_queue_congested(q, READ);
L
Linus Torvalds 已提交
4249 4250

	if (rl->count[WRITE] >= queue_congestion_on_threshold(q))
4251
		blk_set_queue_congested(q, WRITE);
L
Linus Torvalds 已提交
4252
	else if (rl->count[WRITE] < queue_congestion_off_threshold(q))
4253
		blk_clear_queue_congested(q, WRITE);
L
Linus Torvalds 已提交
4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267

	if (rl->count[READ] >= q->nr_requests) {
		blk_set_queue_full(q, READ);
	} else if (rl->count[READ]+1 <= q->nr_requests) {
		blk_clear_queue_full(q, READ);
		wake_up(&rl->wait[READ]);
	}

	if (rl->count[WRITE] >= q->nr_requests) {
		blk_set_queue_full(q, WRITE);
	} else if (rl->count[WRITE]+1 <= q->nr_requests) {
		blk_clear_queue_full(q, WRITE);
		wake_up(&rl->wait[WRITE]);
	}
4268
	spin_unlock_irq(q->queue_lock);
L
Linus Torvalds 已提交
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371
	return ret;
}

static ssize_t queue_ra_show(struct request_queue *q, char *page)
{
	int ra_kb = q->backing_dev_info.ra_pages << (PAGE_CACHE_SHIFT - 10);

	return queue_var_show(ra_kb, (page));
}

static ssize_t
queue_ra_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long ra_kb;
	ssize_t ret = queue_var_store(&ra_kb, page, count);

	spin_lock_irq(q->queue_lock);
	q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
	spin_unlock_irq(q->queue_lock);

	return ret;
}

static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
{
	int max_sectors_kb = q->max_sectors >> 1;

	return queue_var_show(max_sectors_kb, (page));
}

static ssize_t
queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
{
	unsigned long max_sectors_kb,
			max_hw_sectors_kb = q->max_hw_sectors >> 1,
			page_kb = 1 << (PAGE_CACHE_SHIFT - 10);
	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);

	if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
		return -EINVAL;
	/*
	 * Take the queue lock to update the readahead and max_sectors
	 * values synchronously:
	 */
	spin_lock_irq(q->queue_lock);
	q->max_sectors = max_sectors_kb << 1;
	spin_unlock_irq(q->queue_lock);

	return ret;
}

static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
{
	int max_hw_sectors_kb = q->max_hw_sectors >> 1;

	return queue_var_show(max_hw_sectors_kb, (page));
}


static struct queue_sysfs_entry queue_requests_entry = {
	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
	.show = queue_requests_show,
	.store = queue_requests_store,
};

static struct queue_sysfs_entry queue_ra_entry = {
	.attr = {.name = "read_ahead_kb", .mode = S_IRUGO | S_IWUSR },
	.show = queue_ra_show,
	.store = queue_ra_store,
};

static struct queue_sysfs_entry queue_max_sectors_entry = {
	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
	.show = queue_max_sectors_show,
	.store = queue_max_sectors_store,
};

static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
	.attr = {.name = "max_hw_sectors_kb", .mode = S_IRUGO },
	.show = queue_max_hw_sectors_show,
};

static struct queue_sysfs_entry queue_iosched_entry = {
	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
	.show = elv_iosched_show,
	.store = elv_iosched_store,
};

static struct attribute *default_attrs[] = {
	&queue_requests_entry.attr,
	&queue_ra_entry.attr,
	&queue_max_hw_sectors_entry.attr,
	&queue_max_sectors_entry.attr,
	&queue_iosched_entry.attr,
	NULL,
};

#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)

static ssize_t
queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
4372 4373
	struct request_queue *q =
		container_of(kobj, struct request_queue, kobj);
4374
	ssize_t res;
L
Linus Torvalds 已提交
4375 4376

	if (!entry->show)
4377
		return -EIO;
4378 4379 4380 4381 4382 4383 4384 4385
	mutex_lock(&q->sysfs_lock);
	if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->show(q, page);
	mutex_unlock(&q->sysfs_lock);
	return res;
L
Linus Torvalds 已提交
4386 4387 4388 4389 4390 4391 4392
}

static ssize_t
queue_attr_store(struct kobject *kobj, struct attribute *attr,
		    const char *page, size_t length)
{
	struct queue_sysfs_entry *entry = to_queue(attr);
4393
	struct request_queue *q = container_of(kobj, struct request_queue, kobj);
4394 4395

	ssize_t res;
L
Linus Torvalds 已提交
4396 4397

	if (!entry->store)
4398
		return -EIO;
4399 4400 4401 4402 4403 4404 4405 4406
	mutex_lock(&q->sysfs_lock);
	if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
		mutex_unlock(&q->sysfs_lock);
		return -ENOENT;
	}
	res = entry->store(q, page, length);
	mutex_unlock(&q->sysfs_lock);
	return res;
L
Linus Torvalds 已提交
4407 4408 4409 4410 4411 4412 4413
}

static struct sysfs_ops queue_sysfs_ops = {
	.show	= queue_attr_show,
	.store	= queue_attr_store,
};

4414
static struct kobj_type queue_ktype = {
L
Linus Torvalds 已提交
4415 4416
	.sysfs_ops	= &queue_sysfs_ops,
	.default_attrs	= default_attrs,
4417
	.release	= blk_release_queue,
L
Linus Torvalds 已提交
4418 4419 4420 4421 4422 4423
};

int blk_register_queue(struct gendisk *disk)
{
	int ret;

4424
	struct request_queue *q = disk->queue;
L
Linus Torvalds 已提交
4425 4426 4427 4428

	if (!q || !q->request_fn)
		return -ENXIO;

4429 4430
	ret = kobject_add(&q->kobj, kobject_get(&disk->dev.kobj),
			  "%s", "queue");
L
Linus Torvalds 已提交
4431 4432 4433
	if (ret < 0)
		return ret;

4434 4435
	kobject_uevent(&q->kobj, KOBJ_ADD);

L
Linus Torvalds 已提交
4436 4437
	ret = elv_register_queue(q);
	if (ret) {
4438 4439
		kobject_uevent(&q->kobj, KOBJ_REMOVE);
		kobject_del(&q->kobj);
L
Linus Torvalds 已提交
4440 4441 4442 4443 4444 4445 4446 4447
		return ret;
	}

	return 0;
}

void blk_unregister_queue(struct gendisk *disk)
{
4448
	struct request_queue *q = disk->queue;
L
Linus Torvalds 已提交
4449 4450 4451 4452

	if (q && q->request_fn) {
		elv_unregister_queue(q);

4453 4454
		kobject_uevent(&q->kobj, KOBJ_REMOVE);
		kobject_del(&q->kobj);
4455
		kobject_put(&disk->dev.kobj);
L
Linus Torvalds 已提交
4456 4457
	}
}