blkdev.h 46.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
#ifndef _LINUX_BLKDEV_H
#define _LINUX_BLKDEV_H

4 5
#include <linux/sched.h>

J
Jens Axboe 已提交
6 7
#ifdef CONFIG_BLOCK

L
Linus Torvalds 已提交
8 9 10
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/list.h>
11
#include <linux/llist.h>
L
Linus Torvalds 已提交
12 13 14 15 16 17 18 19
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/pagemap.h>
#include <linux/backing-dev.h>
#include <linux/wait.h>
#include <linux/mempool.h>
#include <linux/bio.h>
#include <linux/stringify.h>
20
#include <linux/gfp.h>
21
#include <linux/bsg.h>
22
#include <linux/smp.h>
T
Tejun Heo 已提交
23
#include <linux/rcupdate.h>
L
Linus Torvalds 已提交
24 25 26

#include <asm/scatterlist.h>

27
struct module;
28 29
struct scsi_ioctl_command;

L
Linus Torvalds 已提交
30 31 32
struct request_queue;
struct elevator_queue;
struct request_pm_state;
33
struct blk_trace;
34 35
struct request;
struct sg_io_hdr;
M
Mike Christie 已提交
36
struct bsg_job;
T
Tejun Heo 已提交
37
struct blkcg_gq;
L
Linus Torvalds 已提交
38 39 40 41

#define BLKDEV_MIN_RQ	4
#define BLKDEV_MAX_RQ	128	/* Default maximum */

T
Tejun Heo 已提交
42 43 44 45 46 47
/*
 * Maximum number of blkcg policies allowed to be registered concurrently.
 * Defined here to simplify include dependency.
 */
#define BLKCG_MAX_POLS		2

L
Linus Torvalds 已提交
48
struct request;
49
typedef void (rq_end_io_fn)(struct request *, int);
L
Linus Torvalds 已提交
50

51 52 53
#define BLK_RL_SYNCFULL		(1U << 0)
#define BLK_RL_ASYNCFULL	(1U << 1)

L
Linus Torvalds 已提交
54
struct request_list {
55
	struct request_queue	*q;	/* the queue this rl belongs to */
56 57 58
#ifdef CONFIG_BLK_CGROUP
	struct blkcg_gq		*blkg;	/* blkg this request pool belongs to */
#endif
59 60 61 62
	/*
	 * count[], starved[], and wait[] are indexed by
	 * BLK_RW_SYNC/BLK_RW_ASYNC
	 */
63 64 65 66
	int			count[2];
	int			starved[2];
	mempool_t		*rq_pool;
	wait_queue_head_t	wait[2];
67
	unsigned int		flags;
L
Linus Torvalds 已提交
68 69
};

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * request command types
 */
enum rq_cmd_type_bits {
	REQ_TYPE_FS		= 1,	/* fs request */
	REQ_TYPE_BLOCK_PC,		/* scsi command */
	REQ_TYPE_SENSE,			/* sense request */
	REQ_TYPE_PM_SUSPEND,		/* suspend request */
	REQ_TYPE_PM_RESUME,		/* resume request */
	REQ_TYPE_PM_SHUTDOWN,		/* shutdown request */
	REQ_TYPE_SPECIAL,		/* driver defined type */
	/*
	 * for ATA/ATAPI devices. this really doesn't belong here, ide should
	 * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
	 * private REQ_LB opcodes to differentiate what type of request this is
	 */
	REQ_TYPE_ATA_TASKFILE,
87
	REQ_TYPE_ATA_PC,
88 89
};

L
Linus Torvalds 已提交
90 91 92
#define BLK_MAX_CDB	16

/*
J
Jens Axboe 已提交
93
 * try to put the fields that are referenced together in the same cacheline.
94
 * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init()
J
Jens Axboe 已提交
95
 * as well!
L
Linus Torvalds 已提交
96 97
 */
struct request {
98
	struct list_head queuelist;
99 100
	union {
		struct call_single_data csd;
101
		struct work_struct requeue_work;
102
		unsigned long fifo_time;
103
	};
104

105
	struct request_queue *q;
106
	struct blk_mq_ctx *mq_ctx;
107

J
Jens Axboe 已提交
108
	u64 cmd_flags;
109
	enum rq_cmd_type_bits cmd_type;
J
Jens Axboe 已提交
110
	unsigned long atomic_flags;
L
Linus Torvalds 已提交
111

112 113
	int cpu;

114 115
	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
116
	sector_t __sector;		/* sector cursor */
L
Linus Torvalds 已提交
117 118 119 120

	struct bio *bio;
	struct bio *biotail;

121 122 123 124 125 126 127 128 129 130 131 132
	/*
	 * The hash is used inside the scheduler, and killed once the
	 * request reaches the dispatch list. The ipi_list is only used
	 * to queue the request for softirq completion, which is long
	 * after the request has been unhashed (and even removed from
	 * the dispatch list).
	 */
	union {
		struct hlist_node hash;	/* merge hash */
		struct list_head ipi_list;
	};

133 134 135
	/*
	 * The rb_node is only used inside the io scheduler, requests
	 * are pruned when moved to the dispatch queue. So let the
136
	 * completion_data share space with the rb_node.
137 138 139
	 */
	union {
		struct rb_node rb_node;	/* sort/lookup */
140
		void *completion_data;
141
	};
142

143
	/*
144
	 * Three pointers are available for the IO schedulers, if they need
145 146
	 * more they have to dynamically allocate it.  Flush requests are
	 * never put on the IO scheduler. So let the flush fields share
147
	 * space with the elevator data.
148
	 */
149
	union {
150 151 152 153 154
		struct {
			struct io_cq		*icq;
			void			*priv[2];
		} elv;

155 156 157
		struct {
			unsigned int		seq;
			struct list_head	list;
158
			rq_end_io_fn		*saved_end_io;
159 160
		} flush;
	};
161

162
	struct gendisk *rq_disk;
163
	struct hd_struct *part;
L
Linus Torvalds 已提交
164
	unsigned long start_time;
165
#ifdef CONFIG_BLK_CGROUP
166
	struct request_list *rl;		/* rl this rq is alloced from */
167 168 169
	unsigned long long start_time_ns;
	unsigned long long io_start_time_ns;    /* when passed to hardware */
#endif
L
Linus Torvalds 已提交
170 171 172 173
	/* Number of scatter-gather DMA addr+len pairs after
	 * physical address coalescing is performed.
	 */
	unsigned short nr_phys_segments;
174 175 176
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	unsigned short nr_integrity_segments;
#endif
L
Linus Torvalds 已提交
177

178 179
	unsigned short ioprio;

T
Tejun Heo 已提交
180
	void *special;		/* opaque pointer available for LLD use */
L
Linus Torvalds 已提交
181

182 183 184
	int tag;
	int errors;

L
Linus Torvalds 已提交
185 186 187
	/*
	 * when request is used as a packet command carrier
	 */
188 189
	unsigned char __cmd[BLK_MAX_CDB];
	unsigned char *cmd;
190
	unsigned short cmd_len;
L
Linus Torvalds 已提交
191

192
	unsigned int extra_len;	/* length of alignment and padding */
L
Linus Torvalds 已提交
193
	unsigned int sense_len;
T
Tejun Heo 已提交
194
	unsigned int resid_len;	/* residual count */
L
Linus Torvalds 已提交
195 196
	void *sense;

J
Jens Axboe 已提交
197 198
	unsigned long deadline;
	struct list_head timeout_list;
L
Linus Torvalds 已提交
199
	unsigned int timeout;
200
	int retries;
L
Linus Torvalds 已提交
201 202

	/*
203
	 * completion callback.
L
Linus Torvalds 已提交
204 205 206
	 */
	rq_end_io_fn *end_io;
	void *end_io_data;
207 208 209

	/* for bidi */
	struct request *next_rq;
L
Linus Torvalds 已提交
210 211
};

212 213 214 215 216
static inline unsigned short req_get_ioprio(struct request *req)
{
	return req->ioprio;
}

L
Linus Torvalds 已提交
217
/*
218
 * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
L
Linus Torvalds 已提交
219 220 221 222 223 224 225 226 227 228 229 230 231
 * requests. Some step values could eventually be made generic.
 */
struct request_pm_state
{
	/* PM state machine step value, currently driver specific */
	int	pm_step;
	/* requested PM state value (S1, S2, S3, S4, ...) */
	u32	pm_state;
	void*	data;		/* for driver use */
};

#include <linux/elevator.h>

232 233
struct blk_queue_ctx;

234
typedef void (request_fn_proc) (struct request_queue *q);
235
typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
236
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
237
typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
L
Linus Torvalds 已提交
238 239

struct bio_vec;
240 241 242 243 244 245 246 247
struct bvec_merge_data {
	struct block_device *bi_bdev;
	sector_t bi_sector;
	unsigned bi_size;
	unsigned long bi_rw;
};
typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
			     struct bio_vec *);
248
typedef void (softirq_done_fn)(struct request *);
249
typedef int (dma_drain_needed_fn)(struct request *);
250
typedef int (lld_busy_fn) (struct request_queue *q);
M
Mike Christie 已提交
251
typedef int (bsg_job_fn) (struct bsg_job *);
L
Linus Torvalds 已提交
252

J
Jens Axboe 已提交
253 254 255 256 257 258 259 260
enum blk_eh_timer_return {
	BLK_EH_NOT_HANDLED,
	BLK_EH_HANDLED,
	BLK_EH_RESET_TIMER,
};

typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);

L
Linus Torvalds 已提交
261 262 263 264 265 266 267 268 269 270
enum blk_queue_state {
	Queue_down,
	Queue_up,
};

struct blk_queue_tag {
	struct request **tag_index;	/* map of busy tags */
	unsigned long *tag_map;		/* bit map of free/busy tags */
	int busy;			/* current depth */
	int max_depth;			/* what we will send to device */
271
	int real_max_depth;		/* what the array can hold */
L
Linus Torvalds 已提交
272 273 274
	atomic_t refcnt;		/* map can be shared */
};

275 276 277
#define BLK_SCSI_MAX_CMDS	(256)
#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))

278 279 280 281 282 283 284
struct queue_limits {
	unsigned long		bounce_pfn;
	unsigned long		seg_boundary_mask;

	unsigned int		max_hw_sectors;
	unsigned int		max_sectors;
	unsigned int		max_segment_size;
285 286 287 288
	unsigned int		physical_block_size;
	unsigned int		alignment_offset;
	unsigned int		io_min;
	unsigned int		io_opt;
289
	unsigned int		max_discard_sectors;
290
	unsigned int		max_write_same_sectors;
291 292
	unsigned int		discard_granularity;
	unsigned int		discard_alignment;
293 294

	unsigned short		logical_block_size;
295
	unsigned short		max_segments;
296
	unsigned short		max_integrity_segments;
297

298
	unsigned char		misaligned;
299
	unsigned char		discard_misaligned;
300
	unsigned char		cluster;
301
	unsigned char		discard_zeroes_data;
302
	unsigned char		raid_partial_stripes_expensive;
303 304
};

305
struct request_queue {
L
Linus Torvalds 已提交
306 307 308 309 310
	/*
	 * Together with queue_head for cacheline sharing
	 */
	struct list_head	queue_head;
	struct request		*last_merge;
J
Jens Axboe 已提交
311
	struct elevator_queue	*elevator;
312 313
	int			nr_rqs[2];	/* # allocated [a]sync rqs */
	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
L
Linus Torvalds 已提交
314 315

	/*
316 317 318 319
	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
	 * is used, root blkg allocates from @q->root_rl and all other
	 * blkgs from their own blkg->rl.  Which one to use should be
	 * determined using bio_request_list().
L
Linus Torvalds 已提交
320
	 */
321
	struct request_list	root_rl;
L
Linus Torvalds 已提交
322 323 324 325

	request_fn_proc		*request_fn;
	make_request_fn		*make_request_fn;
	prep_rq_fn		*prep_rq_fn;
326
	unprep_rq_fn		*unprep_rq_fn;
L
Linus Torvalds 已提交
327
	merge_bvec_fn		*merge_bvec_fn;
328
	softirq_done_fn		*softirq_done_fn;
J
Jens Axboe 已提交
329
	rq_timed_out_fn		*rq_timed_out_fn;
330
	dma_drain_needed_fn	*dma_drain_needed;
331
	lld_busy_fn		*lld_busy_fn;
L
Linus Torvalds 已提交
332

333 334 335 336 337 338 339 340 341 342 343 344
	struct blk_mq_ops	*mq_ops;

	unsigned int		*mq_map;

	/* sw queues */
	struct blk_mq_ctx	*queue_ctx;
	unsigned int		nr_queues;

	/* hw dispatch queues */
	struct blk_mq_hw_ctx	**queue_hw_ctx;
	unsigned int		nr_hw_queues;

345 346 347
	/*
	 * Dispatch queue sorting
	 */
J
Jens Axboe 已提交
348
	sector_t		end_sector;
349 350
	struct request		*boundary_rq;

L
Linus Torvalds 已提交
351
	/*
352
	 * Delayed queue handling
L
Linus Torvalds 已提交
353
	 */
354
	struct delayed_work	delay_work;
L
Linus Torvalds 已提交
355 356 357 358 359 360 361 362 363 364

	struct backing_dev_info	backing_dev_info;

	/*
	 * The queue owner gets to use this for whatever they like.
	 * ll_rw_blk doesn't touch it.
	 */
	void			*queuedata;

	/*
365
	 * various queue flags, see QUEUE_* below
L
Linus Torvalds 已提交
366
	 */
367
	unsigned long		queue_flags;
L
Linus Torvalds 已提交
368

369 370 371 372 373 374
	/*
	 * ida allocated id for this queue.  Used to index queues from
	 * ioctx.
	 */
	int			id;

L
Linus Torvalds 已提交
375
	/*
376
	 * queue needs bounce pages for pages above this limit
L
Linus Torvalds 已提交
377
	 */
378
	gfp_t			bounce_gfp;
L
Linus Torvalds 已提交
379 380

	/*
已提交
381 382 383
	 * protects queue structures from reentrancy. ->__queue_lock should
	 * _never_ be used directly, it is queue private. always use
	 * ->queue_lock.
L
Linus Torvalds 已提交
384
	 */
已提交
385
	spinlock_t		__queue_lock;
L
Linus Torvalds 已提交
386 387 388 389 390 391 392
	spinlock_t		*queue_lock;

	/*
	 * queue kobject
	 */
	struct kobject kobj;

393 394 395 396 397
	/*
	 * mq queue kobject
	 */
	struct kobject mq_kobj;

L
Lin Ming 已提交
398 399 400 401 402 403
#ifdef CONFIG_PM_RUNTIME
	struct device		*dev;
	int			rpm_status;
	unsigned int		nr_pending;
#endif

L
Linus Torvalds 已提交
404 405 406 407 408 409 410 411
	/*
	 * queue settings
	 */
	unsigned long		nr_requests;	/* Max # of requests */
	unsigned int		nr_congestion_on;
	unsigned int		nr_congestion_off;
	unsigned int		nr_batching;

J
James Bottomley 已提交
412
	unsigned int		dma_drain_size;
413
	void			*dma_drain_buffer;
414
	unsigned int		dma_pad_mask;
L
Linus Torvalds 已提交
415 416 417
	unsigned int		dma_alignment;

	struct blk_queue_tag	*queue_tags;
418
	struct list_head	tag_busy_list;
L
Linus Torvalds 已提交
419

420
	unsigned int		nr_sorted;
421
	unsigned int		in_flight[2];
422 423 424 425 426 427
	/*
	 * Number of active block driver functions for which blk_drain_queue()
	 * must wait. Must be incremented around functions that unlock the
	 * queue_lock internally, e.g. scsi_request_fn().
	 */
	unsigned int		request_fn_active;
L
Linus Torvalds 已提交
428

J
Jens Axboe 已提交
429 430 431 432
	unsigned int		rq_timeout;
	struct timer_list	timeout;
	struct list_head	timeout_list;

433
	struct list_head	icq_list;
434
#ifdef CONFIG_BLK_CGROUP
435
	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
T
Tejun Heo 已提交
436
	struct blkcg_gq		*root_blkg;
437
	struct list_head	blkg_list;
438
#endif
439

440 441
	struct queue_limits	limits;

L
Linus Torvalds 已提交
442 443 444 445 446
	/*
	 * sg stuff
	 */
	unsigned int		sg_timeout;
	unsigned int		sg_reserved_size;
447
	int			node;
448
#ifdef CONFIG_BLK_DEV_IO_TRACE
449
	struct blk_trace	*blk_trace;
450
#endif
L
Linus Torvalds 已提交
451
	/*
452
	 * for flush operations
L
Linus Torvalds 已提交
453
	 */
454
	unsigned int		flush_flags;
455
	unsigned int		flush_not_queueable:1;
456
	unsigned int		flush_queue_delayed:1;
457 458 459 460 461
	unsigned int		flush_pending_idx:1;
	unsigned int		flush_running_idx:1;
	unsigned long		flush_pending_since;
	struct list_head	flush_queue[2];
	struct list_head	flush_data_in_flight;
462 463
	struct request		*flush_rq;
	spinlock_t		mq_flush_lock;
464 465

	struct mutex		sysfs_lock;
466

467 468
	int			bypass_depth;

469
#if defined(CONFIG_BLK_DEV_BSG)
M
Mike Christie 已提交
470 471
	bsg_job_fn		*bsg_job_fn;
	int			bsg_job_size;
472 473
	struct bsg_class_device bsg_dev;
#endif
474 475 476 477 478

#ifdef CONFIG_BLK_DEV_THROTTLING
	/* Throttle data */
	struct throtl_data *td;
#endif
T
Tejun Heo 已提交
479
	struct rcu_head		rcu_head;
480 481 482
	wait_queue_head_t	mq_freeze_wq;
	struct percpu_counter	mq_usage_counter;
	struct list_head	all_q_node;
L
Linus Torvalds 已提交
483 484 485 486
};

#define QUEUE_FLAG_QUEUED	1	/* uses generic tag queueing */
#define QUEUE_FLAG_STOPPED	2	/* queue is stopped */
487 488
#define	QUEUE_FLAG_SYNCFULL	3	/* read queue has been filled */
#define QUEUE_FLAG_ASYNCFULL	4	/* write queue has been filled */
B
Bart Van Assche 已提交
489
#define QUEUE_FLAG_DYING	5	/* queue being torn down */
490
#define QUEUE_FLAG_BYPASS	6	/* act as dumb FIFO queue */
491 492
#define QUEUE_FLAG_BIDI		7	/* queue supports bidi requests */
#define QUEUE_FLAG_NOMERGES     8	/* disable merge attempts */
D
Dan Williams 已提交
493
#define QUEUE_FLAG_SAME_COMP	9	/* complete on same CPU-group */
494 495 496
#define QUEUE_FLAG_FAIL_IO     10	/* fake timeout */
#define QUEUE_FLAG_STACKABLE   11	/* supports request stacking */
#define QUEUE_FLAG_NONROT      12	/* non-rotational device (SSD) */
497
#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
498 499 500 501 502
#define QUEUE_FLAG_IO_STAT     13	/* do IO stats */
#define QUEUE_FLAG_DISCARD     14	/* supports DISCARD */
#define QUEUE_FLAG_NOXMERGES   15	/* No extended merges */
#define QUEUE_FLAG_ADD_RANDOM  16	/* Contributes to random pool */
#define QUEUE_FLAG_SECDISCARD  17	/* supports SECDISCARD */
D
Dan Williams 已提交
503
#define QUEUE_FLAG_SAME_FORCE  18	/* force complete on same CPU */
504
#define QUEUE_FLAG_DEAD        19	/* queue tear-down finished */
505
#define QUEUE_FLAG_INIT_DONE   20	/* queue is initialized */
506 507

#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
508
				 (1 << QUEUE_FLAG_STACKABLE)	|	\
509 510
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_ADD_RANDOM))
511

512 513 514
#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
				 (1 << QUEUE_FLAG_SAME_COMP))

515
static inline void queue_lockdep_assert_held(struct request_queue *q)
516
{
517 518
	if (q->queue_lock)
		lockdep_assert_held(q->queue_lock);
519 520
}

N
Nick Piggin 已提交
521 522 523 524 525 526
static inline void queue_flag_set_unlocked(unsigned int flag,
					   struct request_queue *q)
{
	__set_bit(flag, &q->queue_flags);
}

J
Jens Axboe 已提交
527 528 529
static inline int queue_flag_test_and_clear(unsigned int flag,
					    struct request_queue *q)
{
530
	queue_lockdep_assert_held(q);
J
Jens Axboe 已提交
531 532 533 534 535 536 537 538 539 540 541 542

	if (test_bit(flag, &q->queue_flags)) {
		__clear_bit(flag, &q->queue_flags);
		return 1;
	}

	return 0;
}

static inline int queue_flag_test_and_set(unsigned int flag,
					  struct request_queue *q)
{
543
	queue_lockdep_assert_held(q);
J
Jens Axboe 已提交
544 545 546 547 548 549 550 551 552

	if (!test_bit(flag, &q->queue_flags)) {
		__set_bit(flag, &q->queue_flags);
		return 0;
	}

	return 1;
}

N
Nick Piggin 已提交
553 554
static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
{
555
	queue_lockdep_assert_held(q);
N
Nick Piggin 已提交
556 557 558 559 560 561 562 563 564
	__set_bit(flag, &q->queue_flags);
}

static inline void queue_flag_clear_unlocked(unsigned int flag,
					     struct request_queue *q)
{
	__clear_bit(flag, &q->queue_flags);
}

565 566 567 568 569
static inline int queue_in_flight(struct request_queue *q)
{
	return q->in_flight[0] + q->in_flight[1];
}

N
Nick Piggin 已提交
570 571
static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
{
572
	queue_lockdep_assert_held(q);
N
Nick Piggin 已提交
573 574 575
	__clear_bit(flag, &q->queue_flags);
}

L
Linus Torvalds 已提交
576 577
#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
B
Bart Van Assche 已提交
578
#define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
579
#define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
580
#define blk_queue_bypass(q)	test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
581
#define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
582
#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
583 584
#define blk_queue_noxmerges(q)	\
	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
585
#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
586
#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
587
#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
588 589
#define blk_queue_stackable(q)	\
	test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
590
#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
A
Adrian Hunter 已提交
591 592
#define blk_queue_secdiscard(q)	(blk_queue_discard(q) && \
	test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags))
L
Linus Torvalds 已提交
593

594 595 596 597 598 599
#define blk_noretry_request(rq) \
	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
			     REQ_FAILFAST_DRIVER))

#define blk_account_rq(rq) \
	(((rq)->cmd_flags & REQ_STARTED) && \
600
	 ((rq)->cmd_type == REQ_TYPE_FS))
601

L
Linus Torvalds 已提交
602
#define blk_pm_request(rq)	\
603 604
	((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
	 (rq)->cmd_type == REQ_TYPE_PM_RESUME)
L
Linus Torvalds 已提交
605

606
#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
607
#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
608 609
/* rq->queuelist of dequeued request must be list_empty() */
#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
L
Linus Torvalds 已提交
610 611 612

#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)

J
Jens Axboe 已提交
613
#define rq_data_dir(rq)		(((rq)->cmd_flags & 1) != 0)
L
Linus Torvalds 已提交
614

615 616 617 618 619
static inline unsigned int blk_queue_cluster(struct request_queue *q)
{
	return q->limits.cluster;
}

620
/*
621
 * We regard a request as sync, if either a read or a sync write
622
 */
623 624
static inline bool rw_is_sync(unsigned int rw_flags)
{
625
	return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
626 627 628 629 630 631 632
}

static inline bool rq_is_sync(struct request *rq)
{
	return rw_is_sync(rq->cmd_flags);
}

633
static inline bool blk_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
634
{
635 636 637
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	return rl->flags & flag;
L
Linus Torvalds 已提交
638 639
}

640
static inline void blk_set_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
641
{
642 643 644
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags |= flag;
L
Linus Torvalds 已提交
645 646
}

647
static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
648
{
649 650 651
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags &= ~flag;
L
Linus Torvalds 已提交
652 653
}

654 655 656 657
static inline bool rq_mergeable(struct request *rq)
{
	if (rq->cmd_type != REQ_TYPE_FS)
		return false;
L
Linus Torvalds 已提交
658

659 660 661 662 663
	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
		return false;

	return true;
}
L
Linus Torvalds 已提交
664

665 666 667 668 669 670 671 672 673
static inline bool blk_check_merge_flags(unsigned int flags1,
					 unsigned int flags2)
{
	if ((flags1 & REQ_DISCARD) != (flags2 & REQ_DISCARD))
		return false;

	if ((flags1 & REQ_SECURE) != (flags2 & REQ_SECURE))
		return false;

674 675 676
	if ((flags1 & REQ_WRITE_SAME) != (flags2 & REQ_WRITE_SAME))
		return false;

677 678 679
	return true;
}

680 681 682 683 684 685 686 687
static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
{
	if (bio_data(a) == bio_data(b))
		return true;

	return false;
}

L
Linus Torvalds 已提交
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
/*
 * q->prep_rq_fn return values
 */
#define BLKPREP_OK		0	/* serve it */
#define BLKPREP_KILL		1	/* fatal error, kill */
#define BLKPREP_DEFER		2	/* leave on queue */

extern unsigned long blk_max_low_pfn, blk_max_pfn;

/*
 * standard bounce addresses:
 *
 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 * BLK_BOUNCE_ANY	: don't bounce anything
 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 */
704 705

#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
706
#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
707 708 709 710
#else
#define BLK_BOUNCE_HIGH		-1ULL
#endif
#define BLK_BOUNCE_ANY		(-1ULL)
711
#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
L
Linus Torvalds 已提交
712

713 714 715 716
/*
 * default timeout for SG_IO if none specified
 */
#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
L
Linus Torvalds 已提交
717
#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
718

719
#ifdef CONFIG_BOUNCE
L
Linus Torvalds 已提交
720
extern int init_emergency_isa_pool(void);
721
extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
L
Linus Torvalds 已提交
722 723 724 725 726
#else
static inline int init_emergency_isa_pool(void)
{
	return 0;
}
727
static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
L
Linus Torvalds 已提交
728 729 730 731
{
}
#endif /* CONFIG_MMU */

732 733 734 735
struct rq_map_data {
	struct page **pages;
	int page_order;
	int nr_entries;
736
	unsigned long offset;
737
	int null_mapped;
738
	int from_user;
739 740
};

741
struct req_iterator {
742
	struct bvec_iter iter;
743 744 745 746
	struct bio *bio;
};

/* This should not be used directly - use rq_for_each_segment */
747 748
#define for_each_bio(_bio)		\
	for (; _bio; _bio = _bio->bi_next)
749
#define __rq_for_each_bio(_bio, rq)	\
L
Linus Torvalds 已提交
750 751 752
	if ((rq->bio))			\
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

753 754
#define rq_for_each_segment(bvl, _rq, _iter)			\
	__rq_for_each_bio(_iter.bio, _rq)			\
755
		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
756

K
Kent Overstreet 已提交
757
#define rq_iter_last(bvec, _iter)				\
758
		(_iter.bio->bi_next == NULL &&			\
K
Kent Overstreet 已提交
759
		 bio_iter_last(bvec, _iter.iter))
760

761 762 763 764 765 766 767 768 769 770 771
#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
#endif
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
extern void rq_flush_dcache_pages(struct request *rq);
#else
static inline void rq_flush_dcache_pages(struct request *rq)
{
}
#endif

L
Linus Torvalds 已提交
772 773 774
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
extern void generic_make_request(struct bio *bio);
775
extern void blk_rq_init(struct request_queue *q, struct request *rq);
L
Linus Torvalds 已提交
776
extern void blk_put_request(struct request *);
777 778
extern void __blk_put_request(struct request_queue *, struct request *);
extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
779 780
extern struct request *blk_make_request(struct request_queue *, struct bio *,
					gfp_t);
781
extern void blk_requeue_request(struct request_queue *, struct request *);
782 783
extern void blk_add_request_payload(struct request *rq, struct page *page,
		unsigned int len);
784
extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
785
extern int blk_lld_busy(struct request_queue *q);
786 787 788 789 790
extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
			     struct bio_set *bs, gfp_t gfp_mask,
			     int (*bio_ctr)(struct bio *, struct bio *, void *),
			     void *data);
extern void blk_rq_unprep_clone(struct request *rq);
791 792
extern int blk_insert_cloned_request(struct request_queue *q,
				     struct request *rq);
793
extern void blk_delay_queue(struct request_queue *, unsigned long);
794
extern void blk_recount_segments(struct request_queue *, struct bio *);
795
extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
796 797
extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
			      unsigned int, void __user *);
798 799
extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			  unsigned int, void __user *);
800 801
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			 struct scsi_ioctl_command __user *);
802

803
extern void blk_queue_bio(struct request_queue *q, struct bio *bio);
C
Christoph Hellwig 已提交
804

805 806 807 808 809
/*
 * A queue has just exitted congestion.  Note this in the global counter of
 * congested queues, and wake up anyone who was waiting for requests to be
 * put back.
 */
810
static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
811
{
812
	clear_bdi_congested(&q->backing_dev_info, sync);
813 814 815 816 817 818
}

/*
 * A queue has just entered congestion.  Flag that in the queue's VM-visible
 * state flags and increment the global gounter of congested queues.
 */
819
static inline void blk_set_queue_congested(struct request_queue *q, int sync)
820
{
821
	set_bdi_congested(&q->backing_dev_info, sync);
822 823
}

824 825
extern void blk_start_queue(struct request_queue *q);
extern void blk_stop_queue(struct request_queue *q);
L
Linus Torvalds 已提交
826
extern void blk_sync_queue(struct request_queue *q);
827
extern void __blk_stop_queue(struct request_queue *q);
828
extern void __blk_run_queue(struct request_queue *q);
829
extern void blk_run_queue(struct request_queue *);
830
extern void blk_run_queue_async(struct request_queue *q);
831
extern int blk_rq_map_user(struct request_queue *, struct request *,
832 833
			   struct rq_map_data *, void __user *, unsigned long,
			   gfp_t);
J
Jens Axboe 已提交
834
extern int blk_rq_unmap_user(struct bio *);
835 836
extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
837 838
			       struct rq_map_data *, const struct sg_iovec *,
			       int, unsigned int, gfp_t);
839
extern int blk_execute_rq(struct request_queue *, struct gendisk *,
840
			  struct request *, int);
841
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
842
				  struct request *, int, rq_end_io_fn *);
843

844
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
L
Linus Torvalds 已提交
845 846 847 848
{
	return bdev->bd_disk->queue;
}

849
/*
850 851 852 853 854 855
 * blk_rq_pos()			: the current sector
 * blk_rq_bytes()		: bytes left in the entire request
 * blk_rq_cur_bytes()		: bytes left in the current segment
 * blk_rq_err_bytes()		: bytes left till the next error boundary
 * blk_rq_sectors()		: sectors left in the entire request
 * blk_rq_cur_sectors()		: sectors left in the current segment
856
 */
857 858
static inline sector_t blk_rq_pos(const struct request *rq)
{
859
	return rq->__sector;
860 861 862 863
}

static inline unsigned int blk_rq_bytes(const struct request *rq)
{
864
	return rq->__data_len;
865 866
}

867 868 869 870
static inline int blk_rq_cur_bytes(const struct request *rq)
{
	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
}
871

872 873
extern unsigned int blk_rq_err_bytes(const struct request *rq);

874 875
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
876
	return blk_rq_bytes(rq) >> 9;
877 878 879 880
}

static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
881
	return blk_rq_cur_bytes(rq) >> 9;
882 883
}

884 885 886 887
static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
						     unsigned int cmd_flags)
{
	if (unlikely(cmd_flags & REQ_DISCARD))
888
		return min(q->limits.max_discard_sectors, UINT_MAX >> 9);
889

890 891 892
	if (unlikely(cmd_flags & REQ_WRITE_SAME))
		return q->limits.max_write_same_sectors;

893 894 895 896 897 898 899 900 901 902 903 904 905
	return q->limits.max_sectors;
}

static inline unsigned int blk_rq_get_max_sectors(struct request *rq)
{
	struct request_queue *q = rq->q;

	if (unlikely(rq->cmd_type == REQ_TYPE_BLOCK_PC))
		return q->limits.max_hw_sectors;

	return blk_queue_get_max_sectors(q, rq->cmd_flags);
}

906 907 908 909 910 911 912 913 914 915 916
static inline unsigned int blk_rq_count_bios(struct request *rq)
{
	unsigned int nr_bios = 0;
	struct bio *bio;

	__rq_for_each_bio(bio, rq)
		nr_bios++;

	return nr_bios;
}

917 918 919 920 921 922 923
/*
 * Request issue related functions.
 */
extern struct request *blk_peek_request(struct request_queue *q);
extern void blk_start_request(struct request *rq);
extern struct request *blk_fetch_request(struct request_queue *q);

L
Linus Torvalds 已提交
924
/*
925 926 927 928 929
 * Request completion related functions.
 *
 * blk_update_request() completes given number of bytes and updates
 * the request without completing it.
 *
930 931
 * blk_end_request() and friends.  __blk_end_request() must be called
 * with the request queue spinlock acquired.
L
Linus Torvalds 已提交
932 933
 *
 * Several drivers define their own end_request and call
934 935
 * blk_end_request() for parts of the original function.
 * This prevents code duplication in drivers.
L
Linus Torvalds 已提交
936
 */
937 938
extern bool blk_update_request(struct request *rq, int error,
			       unsigned int nr_bytes);
939
extern void blk_finish_request(struct request *rq, int error);
940 941 942 943
extern bool blk_end_request(struct request *rq, int error,
			    unsigned int nr_bytes);
extern void blk_end_request_all(struct request *rq, int error);
extern bool blk_end_request_cur(struct request *rq, int error);
944
extern bool blk_end_request_err(struct request *rq, int error);
945 946 947 948
extern bool __blk_end_request(struct request *rq, int error,
			      unsigned int nr_bytes);
extern void __blk_end_request_all(struct request *rq, int error);
extern bool __blk_end_request_cur(struct request *rq, int error);
949
extern bool __blk_end_request_err(struct request *rq, int error);
950

951
extern void blk_complete_request(struct request *);
J
Jens Axboe 已提交
952 953
extern void __blk_complete_request(struct request *);
extern void blk_abort_request(struct request *);
954
extern void blk_unprep_request(struct request *);
955

L
Linus Torvalds 已提交
956 957 958
/*
 * Access functions for manipulating queue properties
 */
959
extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
960
					spinlock_t *lock, int node_id);
961
extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
962 963
extern struct request_queue *blk_init_allocated_queue(struct request_queue *,
						      request_fn_proc *, spinlock_t *);
964 965 966
extern void blk_cleanup_queue(struct request_queue *);
extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
extern void blk_queue_bounce_limit(struct request_queue *, u64);
967
extern void blk_limits_max_hw_sectors(struct queue_limits *, unsigned int);
968
extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
969
extern void blk_queue_max_segments(struct request_queue *, unsigned short);
970
extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
971 972
extern void blk_queue_max_discard_sectors(struct request_queue *q,
		unsigned int max_discard_sectors);
973 974
extern void blk_queue_max_write_same_sectors(struct request_queue *q,
		unsigned int max_write_same_sectors);
975
extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
976
extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
977 978
extern void blk_queue_alignment_offset(struct request_queue *q,
				       unsigned int alignment);
979
extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
980
extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
981
extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
982
extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
983
extern void blk_set_default_limits(struct queue_limits *lim);
984
extern void blk_set_stacking_limits(struct queue_limits *lim);
985 986
extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
			    sector_t offset);
987 988
extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
			    sector_t offset);
989 990
extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
			      sector_t offset);
991
extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
992
extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
993
extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
994 995 996
extern int blk_queue_dma_drain(struct request_queue *q,
			       dma_drain_needed_fn *dma_drain_needed,
			       void *buf, unsigned int size);
997
extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
998 999
extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1000
extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1001 1002
extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *);
extern void blk_queue_dma_alignment(struct request_queue *, int);
1003
extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1004
extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
J
Jens Axboe 已提交
1005 1006
extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1007
extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
1008
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
L
Linus Torvalds 已提交
1009 1010
extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);

1011
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
A
Asias He 已提交
1012 1013
extern int blk_bio_map_sg(struct request_queue *q, struct bio *bio,
			  struct scatterlist *sglist);
L
Linus Torvalds 已提交
1014 1015 1016
extern void blk_dump_rq_flags(struct request *, char *);
extern long nr_blockdev_pages(void);

T
Tejun Heo 已提交
1017
bool __must_check blk_get_queue(struct request_queue *);
1018 1019 1020
struct request_queue *blk_alloc_queue(gfp_t);
struct request_queue *blk_alloc_queue_node(gfp_t, int);
extern void blk_put_queue(struct request_queue *);
L
Linus Torvalds 已提交
1021

L
Lin Ming 已提交
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
/*
 * block layer runtime pm functions
 */
#ifdef CONFIG_PM_RUNTIME
extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
extern int blk_pre_runtime_suspend(struct request_queue *q);
extern void blk_post_runtime_suspend(struct request_queue *q, int err);
extern void blk_pre_runtime_resume(struct request_queue *q);
extern void blk_post_runtime_resume(struct request_queue *q, int err);
#else
static inline void blk_pm_runtime_init(struct request_queue *q,
	struct device *dev) {}
static inline int blk_pre_runtime_suspend(struct request_queue *q)
{
	return -ENOSYS;
}
static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
static inline void blk_pre_runtime_resume(struct request_queue *q) {}
static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
#endif

1043
/*
S
Suresh Jayaraman 已提交
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
 * blk_plug permits building a queue of related requests by holding the I/O
 * fragments for a short period. This allows merging of sequential requests
 * into single larger request. As the requests are moved from a per-task list to
 * the device's request_queue in a batch, this results in improved scalability
 * as the lock contention for request_queue lock is reduced.
 *
 * It is ok not to disable preemption when adding the request to the plug list
 * or when attempting a merge, because blk_schedule_flush_list() will only flush
 * the plug list when the task sleeps by itself. For details, please see
 * schedule() where blk_schedule_flush_plug() is called.
1054
 */
1055
struct blk_plug {
S
Suresh Jayaraman 已提交
1056 1057
	unsigned long magic; /* detect uninitialized use-cases */
	struct list_head list; /* requests */
1058
	struct list_head mq_list; /* blk-mq requests */
S
Suresh Jayaraman 已提交
1059
	struct list_head cb_list; /* md requires an unplug callback */
1060
};
1061 1062
#define BLK_MAX_REQUEST_COUNT 16

1063
struct blk_plug_cb;
1064
typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1065 1066
struct blk_plug_cb {
	struct list_head list;
1067 1068
	blk_plug_cb_fn callback;
	void *data;
1069
};
1070 1071
extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
					     void *data, int size);
1072 1073
extern void blk_start_plug(struct blk_plug *);
extern void blk_finish_plug(struct blk_plug *);
1074
extern void blk_flush_plug_list(struct blk_plug *, bool);
1075 1076 1077 1078 1079

static inline void blk_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1080 1081 1082 1083 1084 1085 1086 1087
	if (plug)
		blk_flush_plug_list(plug, false);
}

static inline void blk_schedule_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1088
	if (plug)
1089
		blk_flush_plug_list(plug, true);
1090 1091 1092 1093 1094 1095
}

static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1096 1097 1098 1099
	return plug &&
		(!list_empty(&plug->list) ||
		 !list_empty(&plug->mq_list) ||
		 !list_empty(&plug->cb_list));
1100 1101
}

L
Linus Torvalds 已提交
1102 1103 1104
/*
 * tag stuff
 */
1105 1106
#define blk_rq_tagged(rq) \
	((rq)->mq_ctx || ((rq)->cmd_flags & REQ_QUEUED))
1107 1108 1109 1110 1111 1112 1113
extern int blk_queue_start_tag(struct request_queue *, struct request *);
extern struct request *blk_queue_find_tag(struct request_queue *, int);
extern void blk_queue_end_tag(struct request_queue *, struct request *);
extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *);
extern void blk_queue_free_tags(struct request_queue *);
extern int blk_queue_resize_tags(struct request_queue *, int);
extern void blk_queue_invalidate_tags(struct request_queue *);
1114 1115
extern struct blk_queue_tag *blk_init_tags(int);
extern void blk_free_tags(struct blk_queue_tag *);
L
Linus Torvalds 已提交
1116

1117 1118 1119 1120 1121 1122 1123
static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
						int tag)
{
	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
		return NULL;
	return bqt->tag_index[tag];
}
1124 1125 1126 1127

#define BLKDEV_DISCARD_SECURE  0x01    /* secure discard */

extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
1128 1129
extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1130 1131
extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1132
extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1133
			sector_t nr_sects, gfp_t gfp_mask);
1134 1135
static inline int sb_issue_discard(struct super_block *sb, sector_t block,
		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
D
David Woodhouse 已提交
1136
{
1137 1138 1139
	return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9),
				    nr_blocks << (sb->s_blocksize_bits - 9),
				    gfp_mask, flags);
D
David Woodhouse 已提交
1140
}
1141
static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1142
		sector_t nr_blocks, gfp_t gfp_mask)
1143 1144 1145 1146
{
	return blkdev_issue_zeroout(sb->s_bdev,
				    block << (sb->s_blocksize_bits - 9),
				    nr_blocks << (sb->s_blocksize_bits - 9),
1147
				    gfp_mask);
1148
}
L
Linus Torvalds 已提交
1149

1150
extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
1151

1152 1153 1154 1155 1156 1157 1158
enum blk_default_limits {
	BLK_MAX_SEGMENTS	= 128,
	BLK_SAFE_MAX_SECTORS	= 255,
	BLK_DEF_MAX_SECTORS	= 1024,
	BLK_MAX_SEGMENT_SIZE	= 65536,
	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
};
1159

L
Linus Torvalds 已提交
1160 1161
#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)

1162 1163
static inline unsigned long queue_bounce_pfn(struct request_queue *q)
{
1164
	return q->limits.bounce_pfn;
1165 1166 1167 1168
}

static inline unsigned long queue_segment_boundary(struct request_queue *q)
{
1169
	return q->limits.seg_boundary_mask;
1170 1171 1172 1173
}

static inline unsigned int queue_max_sectors(struct request_queue *q)
{
1174
	return q->limits.max_sectors;
1175 1176 1177 1178
}

static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
{
1179
	return q->limits.max_hw_sectors;
1180 1181
}

1182
static inline unsigned short queue_max_segments(struct request_queue *q)
1183
{
1184
	return q->limits.max_segments;
1185 1186 1187 1188
}

static inline unsigned int queue_max_segment_size(struct request_queue *q)
{
1189
	return q->limits.max_segment_size;
1190 1191
}

1192
static inline unsigned short queue_logical_block_size(struct request_queue *q)
L
Linus Torvalds 已提交
1193 1194 1195
{
	int retval = 512;

1196 1197
	if (q && q->limits.logical_block_size)
		retval = q->limits.logical_block_size;
L
Linus Torvalds 已提交
1198 1199 1200 1201

	return retval;
}

1202
static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1203
{
1204
	return queue_logical_block_size(bdev_get_queue(bdev));
L
Linus Torvalds 已提交
1205 1206
}

1207 1208 1209 1210 1211
static inline unsigned int queue_physical_block_size(struct request_queue *q)
{
	return q->limits.physical_block_size;
}

1212
static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
M
Martin K. Petersen 已提交
1213 1214 1215 1216
{
	return queue_physical_block_size(bdev_get_queue(bdev));
}

1217 1218 1219 1220 1221
static inline unsigned int queue_io_min(struct request_queue *q)
{
	return q->limits.io_min;
}

M
Martin K. Petersen 已提交
1222 1223 1224 1225 1226
static inline int bdev_io_min(struct block_device *bdev)
{
	return queue_io_min(bdev_get_queue(bdev));
}

1227 1228 1229 1230 1231
static inline unsigned int queue_io_opt(struct request_queue *q)
{
	return q->limits.io_opt;
}

M
Martin K. Petersen 已提交
1232 1233 1234 1235 1236
static inline int bdev_io_opt(struct block_device *bdev)
{
	return queue_io_opt(bdev_get_queue(bdev));
}

1237 1238
static inline int queue_alignment_offset(struct request_queue *q)
{
M
Martin K. Petersen 已提交
1239
	if (q->limits.misaligned)
1240 1241
		return -1;

M
Martin K. Petersen 已提交
1242
	return q->limits.alignment_offset;
1243 1244
}

1245
static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1246 1247
{
	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1248
	unsigned int alignment = (sector << 9) & (granularity - 1);
1249

1250 1251
	return (granularity + lim->alignment_offset - alignment)
		& (granularity - 1);
1252 1253
}

M
Martin K. Petersen 已提交
1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
static inline int bdev_alignment_offset(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q->limits.misaligned)
		return -1;

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->alignment_offset;

	return q->limits.alignment_offset;
}

1267 1268 1269 1270 1271 1272 1273 1274
static inline int queue_discard_alignment(struct request_queue *q)
{
	if (q->limits.discard_misaligned)
		return -1;

	return q->limits.discard_alignment;
}

1275
static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1276
{
1277
	unsigned int alignment, granularity, offset;
1278

1279 1280 1281
	if (!lim->max_discard_sectors)
		return 0;

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
	/* Why are these in bytes, not sectors? */
	alignment = lim->discard_alignment >> 9;
	granularity = lim->discard_granularity >> 9;
	if (!granularity)
		return 0;

	/* Offset of the partition start in 'granularity' sectors */
	offset = sector_div(sector, granularity);

	/* And why do we do this modulus *again* in blkdev_issue_discard()? */
	offset = (granularity + alignment - offset) % granularity;

	/* Turn it back into bytes, gaah */
	return offset << 9;
1296 1297
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
static inline int bdev_discard_alignment(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->discard_alignment;

	return q->limits.discard_alignment;
}

1308 1309
static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
{
1310
	if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
		return 1;

	return 0;
}

static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev)
{
	return queue_discard_zeroes_data(bdev_get_queue(bdev));
}

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
static inline unsigned int bdev_write_same(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return q->limits.max_write_same_sectors;

	return 0;
}

1331
static inline int queue_dma_alignment(struct request_queue *q)
L
Linus Torvalds 已提交
1332
{
1333
	return q ? q->dma_alignment : 511;
L
Linus Torvalds 已提交
1334 1335
}

1336
static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1337 1338 1339
				 unsigned int len)
{
	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1340
	return !(addr & alignment) && !(len & alignment);
1341 1342
}

L
Linus Torvalds 已提交
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
/* assumes size > 256 */
static inline unsigned int blksize_bits(unsigned int size)
{
	unsigned int bits = 8;
	do {
		bits++;
		size >>= 1;
	} while (size > 256);
	return bits;
}

1354
static inline unsigned int block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1355 1356 1357 1358
{
	return bdev->bd_block_size;
}

1359 1360 1361 1362 1363
static inline bool queue_flush_queueable(struct request_queue *q)
{
	return !q->flush_not_queueable;
}

L
Linus Torvalds 已提交
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
typedef struct {struct page *v;} Sector;

unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);

static inline void put_dev_sector(Sector p)
{
	page_cache_release(p.v);
}

struct work_struct;
1374 1375
int kblockd_schedule_work(struct work_struct *work);
int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
1376
int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
L
Linus Torvalds 已提交
1377

1378
#ifdef CONFIG_BLK_CGROUP
1379 1380 1381 1382 1383
/*
 * This should not be using sched_clock(). A real patch is in progress
 * to fix this up, until that is in place we need to disable preemption
 * around sched_clock() in this function and set_io_start_time_ns().
 */
1384 1385
static inline void set_start_time_ns(struct request *req)
{
1386
	preempt_disable();
1387
	req->start_time_ns = sched_clock();
1388
	preempt_enable();
1389 1390 1391 1392
}

static inline void set_io_start_time_ns(struct request *req)
{
1393
	preempt_disable();
1394
	req->io_start_time_ns = sched_clock();
1395
	preempt_enable();
1396
}
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406

static inline uint64_t rq_start_time_ns(struct request *req)
{
        return req->start_time_ns;
}

static inline uint64_t rq_io_start_time_ns(struct request *req)
{
        return req->io_start_time_ns;
}
1407 1408 1409
#else
static inline void set_start_time_ns(struct request *req) {}
static inline void set_io_start_time_ns(struct request *req) {}
1410 1411 1412 1413 1414 1415 1416 1417
static inline uint64_t rq_start_time_ns(struct request *req)
{
	return 0;
}
static inline uint64_t rq_io_start_time_ns(struct request *req)
{
	return 0;
}
1418 1419
#endif

L
Linus Torvalds 已提交
1420 1421 1422 1423 1424
#define MODULE_ALIAS_BLOCKDEV(major,minor) \
	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
	MODULE_ALIAS("block-major-" __stringify(major) "-*")

1425 1426
#if defined(CONFIG_BLK_DEV_INTEGRITY)

1427 1428
#define INTEGRITY_FLAG_READ	2	/* verify data integrity on read */
#define INTEGRITY_FLAG_WRITE	4	/* generate data integrity on write */
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459

struct blk_integrity_exchg {
	void			*prot_buf;
	void			*data_buf;
	sector_t		sector;
	unsigned int		data_size;
	unsigned short		sector_size;
	const char		*disk_name;
};

typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);

struct blk_integrity {
	integrity_gen_fn	*generate_fn;
	integrity_vrfy_fn	*verify_fn;
	integrity_set_tag_fn	*set_tag_fn;
	integrity_get_tag_fn	*get_tag_fn;

	unsigned short		flags;
	unsigned short		tuple_size;
	unsigned short		sector_size;
	unsigned short		tag_size;

	const char		*name;

	struct kobject		kobj;
};

1460
extern bool blk_integrity_is_initialized(struct gendisk *);
1461 1462
extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
extern void blk_integrity_unregister(struct gendisk *);
1463
extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1464 1465 1466 1467 1468 1469 1470
extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
				   struct scatterlist *);
extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
extern int blk_integrity_merge_rq(struct request_queue *, struct request *,
				  struct request *);
extern int blk_integrity_merge_bio(struct request_queue *, struct request *,
				   struct bio *);
1471

1472 1473 1474 1475 1476 1477
static inline
struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
{
	return bdev->bd_disk->integrity;
}

1478 1479 1480 1481 1482
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
	return disk->integrity;
}

1483 1484
static inline int blk_integrity_rq(struct request *rq)
{
1485 1486 1487
	if (rq->bio == NULL)
		return 0;

1488 1489 1490
	return bio_integrity(rq->bio);
}

1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
	q->limits.max_integrity_segments = segs;
}

static inline unsigned short
queue_max_integrity_segments(struct request_queue *q)
{
	return q->limits.max_integrity_segments;
}

1503 1504
#else /* CONFIG_BLK_DEV_INTEGRITY */

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
struct bio;
struct block_device;
struct gendisk;
struct blk_integrity;

static inline int blk_integrity_rq(struct request *rq)
{
	return 0;
}
static inline int blk_rq_count_integrity_sg(struct request_queue *q,
					    struct bio *b)
{
	return 0;
}
static inline int blk_rq_map_integrity_sg(struct request_queue *q,
					  struct bio *b,
					  struct scatterlist *s)
{
	return 0;
}
static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
{
	return 0;
}
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
	return NULL;
}
static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
{
	return 0;
}
static inline int blk_integrity_register(struct gendisk *d,
					 struct blk_integrity *b)
{
	return 0;
}
static inline void blk_integrity_unregister(struct gendisk *d)
{
}
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
}
static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
{
	return 0;
}
static inline int blk_integrity_merge_rq(struct request_queue *rq,
					 struct request *r1,
					 struct request *r2)
{
	return 0;
}
static inline int blk_integrity_merge_bio(struct request_queue *rq,
					  struct request *r,
					  struct bio *b)
{
	return 0;
}
static inline bool blk_integrity_is_initialized(struct gendisk *g)
{
	return 0;
}
1569 1570 1571

#endif /* CONFIG_BLK_DEV_INTEGRITY */

1572
struct block_device_operations {
A
Al Viro 已提交
1573
	int (*open) (struct block_device *, fmode_t);
1574
	void (*release) (struct gendisk *, fmode_t);
A
Al Viro 已提交
1575 1576
	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1577 1578
	int (*direct_access) (struct block_device *, sector_t,
						void **, unsigned long *);
1579 1580 1581
	unsigned int (*check_events) (struct gendisk *disk,
				      unsigned int clearing);
	/* ->media_changed() is DEPRECATED, use ->check_events() instead */
1582
	int (*media_changed) (struct gendisk *);
1583
	void (*unlock_native_capacity) (struct gendisk *);
1584 1585
	int (*revalidate_disk) (struct gendisk *);
	int (*getgeo)(struct block_device *, struct hd_geometry *);
1586 1587
	/* this callback is with swap_lock and sometimes page table lock held */
	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1588 1589 1590
	struct module *owner;
};

1591 1592
extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
				 unsigned long);
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
#else /* CONFIG_BLOCK */
/*
 * stubs for when the block layer is configured out
 */
#define buffer_heads_over_limit 0

static inline long nr_blockdev_pages(void)
{
	return 0;
}

1604 1605 1606 1607
struct blk_plug {
};

static inline void blk_start_plug(struct blk_plug *plug)
1608 1609 1610
{
}

1611
static inline void blk_finish_plug(struct blk_plug *plug)
1612 1613 1614
{
}

1615
static inline void blk_flush_plug(struct task_struct *task)
1616 1617 1618
{
}

1619 1620 1621 1622 1623
static inline void blk_schedule_flush_plug(struct task_struct *task)
{
}


1624 1625 1626 1627 1628
static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	return false;
}

1629 1630
#endif /* CONFIG_BLOCK */

L
Linus Torvalds 已提交
1631
#endif