blkdev.h 37.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
#ifndef _LINUX_BLKDEV_H
#define _LINUX_BLKDEV_H

J
Jens Axboe 已提交
4 5
#ifdef CONFIG_BLOCK

6
#include <linux/sched.h>
L
Linus Torvalds 已提交
7 8 9 10 11 12 13 14 15 16 17 18
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/list.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/pagemap.h>
#include <linux/backing-dev.h>
#include <linux/wait.h>
#include <linux/mempool.h>
#include <linux/bio.h>
#include <linux/module.h>
#include <linux/stringify.h>
19
#include <linux/gfp.h>
20
#include <linux/bsg.h>
21
#include <linux/smp.h>
L
Linus Torvalds 已提交
22 23 24

#include <asm/scatterlist.h>

25 26
struct scsi_ioctl_command;

L
Linus Torvalds 已提交
27 28 29
struct request_queue;
struct elevator_queue;
struct request_pm_state;
30
struct blk_trace;
31 32
struct request;
struct sg_io_hdr;
L
Linus Torvalds 已提交
33 34 35 36 37

#define BLKDEV_MIN_RQ	4
#define BLKDEV_MAX_RQ	128	/* Default maximum */

struct request;
38
typedef void (rq_end_io_fn)(struct request *, int);
L
Linus Torvalds 已提交
39 40

struct request_list {
41 42 43 44
	/*
	 * count[], starved[], and wait[] are indexed by
	 * BLK_RW_SYNC/BLK_RW_ASYNC
	 */
L
Linus Torvalds 已提交
45 46
	int count[2];
	int starved[2];
T
Tejun Heo 已提交
47
	int elvpriv;
L
Linus Torvalds 已提交
48 49 50 51
	mempool_t *rq_pool;
	wait_queue_head_t wait[2];
};

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
/*
 * request command types
 */
enum rq_cmd_type_bits {
	REQ_TYPE_FS		= 1,	/* fs request */
	REQ_TYPE_BLOCK_PC,		/* scsi command */
	REQ_TYPE_SENSE,			/* sense request */
	REQ_TYPE_PM_SUSPEND,		/* suspend request */
	REQ_TYPE_PM_RESUME,		/* resume request */
	REQ_TYPE_PM_SHUTDOWN,		/* shutdown request */
	REQ_TYPE_SPECIAL,		/* driver defined type */
	REQ_TYPE_LINUX_BLOCK,		/* generic block layer message */
	/*
	 * for ATA/ATAPI devices. this really doesn't belong here, ide should
	 * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
	 * private REQ_LB opcodes to differentiate what type of request this is
	 */
	REQ_TYPE_ATA_TASKFILE,
70
	REQ_TYPE_ATA_PC,
71 72 73 74 75 76 77 78 79 80 81 82 83
};

/*
 * For request of type REQ_TYPE_LINUX_BLOCK, rq->cmd[0] is the opcode being
 * sent down (similar to how REQ_TYPE_BLOCK_PC means that ->cmd[] holds a
 * SCSI cdb.
 *
 * 0x00 -> 0x3f are driver private, to be used for whatever purpose they need,
 * typically to differentiate REQ_TYPE_SPECIAL requests.
 *
 */
enum {
	REQ_LB_OP_EJECT	= 0x40,		/* eject request */
D
David Woodhouse 已提交
84
	REQ_LB_OP_FLUSH = 0x41,		/* flush request */
85 86
};

L
Linus Torvalds 已提交
87 88 89
#define BLK_MAX_CDB	16

/*
J
Jens Axboe 已提交
90 91 92
 * try to put the fields that are referenced together in the same cacheline.
 * if you modify this structure, be sure to check block/blk-core.c:rq_init()
 * as well!
L
Linus Torvalds 已提交
93 94
 */
struct request {
95
	struct list_head queuelist;
96
	struct call_single_data csd;
97

98
	struct request_queue *q;
99

100 101
	unsigned int cmd_flags;
	enum rq_cmd_type_bits cmd_type;
J
Jens Axboe 已提交
102
	unsigned long atomic_flags;
L
Linus Torvalds 已提交
103

104 105
	int cpu;

106 107
	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
108
	sector_t __sector;		/* sector cursor */
L
Linus Torvalds 已提交
109 110 111 112

	struct bio *bio;
	struct bio *biotail;

113
	struct hlist_node hash;	/* merge hash */
114 115 116 117 118 119 120 121 122
	/*
	 * The rb_node is only used inside the io scheduler, requests
	 * are pruned when moved to the dispatch queue. So let the
	 * completion_data share space with the rb_node.
	 */
	union {
		struct rb_node rb_node;	/* sort/lookup */
		void *completion_data;
	};
123

124
	/*
125
	 * Three pointers are available for the IO schedulers, if they need
126 127
	 * more they have to dynamically allocate it.
	 */
L
Linus Torvalds 已提交
128
	void *elevator_private;
129
	void *elevator_private2;
130
	void *elevator_private3;
131

132
	struct gendisk *rq_disk;
L
Linus Torvalds 已提交
133
	unsigned long start_time;
134 135 136 137
#ifdef CONFIG_BLK_CGROUP
	unsigned long long start_time_ns;
	unsigned long long io_start_time_ns;    /* when passed to hardware */
#endif
L
Linus Torvalds 已提交
138 139 140 141 142
	/* Number of scatter-gather DMA addr+len pairs after
	 * physical address coalescing is performed.
	 */
	unsigned short nr_phys_segments;

143 144
	unsigned short ioprio;

145 146
	int ref_count;

T
Tejun Heo 已提交
147 148
	void *special;		/* opaque pointer available for LLD use */
	char *buffer;		/* kaddr of the current segment if available */
L
Linus Torvalds 已提交
149

150 151 152
	int tag;
	int errors;

L
Linus Torvalds 已提交
153 154 155
	/*
	 * when request is used as a packet command carrier
	 */
156 157
	unsigned char __cmd[BLK_MAX_CDB];
	unsigned char *cmd;
158
	unsigned short cmd_len;
L
Linus Torvalds 已提交
159

160
	unsigned int extra_len;	/* length of alignment and padding */
L
Linus Torvalds 已提交
161
	unsigned int sense_len;
T
Tejun Heo 已提交
162
	unsigned int resid_len;	/* residual count */
L
Linus Torvalds 已提交
163 164
	void *sense;

J
Jens Axboe 已提交
165 166
	unsigned long deadline;
	struct list_head timeout_list;
L
Linus Torvalds 已提交
167
	unsigned int timeout;
168
	int retries;
L
Linus Torvalds 已提交
169 170

	/*
171
	 * completion callback.
L
Linus Torvalds 已提交
172 173 174
	 */
	rq_end_io_fn *end_io;
	void *end_io_data;
175 176 177

	/* for bidi */
	struct request *next_rq;
L
Linus Torvalds 已提交
178 179
};

180 181 182 183 184
static inline unsigned short req_get_ioprio(struct request *req)
{
	return req->ioprio;
}

L
Linus Torvalds 已提交
185
/*
186
 * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
L
Linus Torvalds 已提交
187 188 189 190 191 192 193 194 195 196 197 198 199
 * requests. Some step values could eventually be made generic.
 */
struct request_pm_state
{
	/* PM state machine step value, currently driver specific */
	int	pm_step;
	/* requested PM state value (S1, S2, S3, S4, ...) */
	u32	pm_state;
	void*	data;		/* for driver use */
};

#include <linux/elevator.h>

200 201 202 203
typedef void (request_fn_proc) (struct request_queue *q);
typedef int (make_request_fn) (struct request_queue *q, struct bio *bio);
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
typedef void (unplug_fn) (struct request_queue *);
L
Linus Torvalds 已提交
204 205

struct bio_vec;
206 207 208 209 210 211 212 213
struct bvec_merge_data {
	struct block_device *bi_bdev;
	sector_t bi_sector;
	unsigned bi_size;
	unsigned long bi_rw;
};
typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
			     struct bio_vec *);
214
typedef void (prepare_flush_fn) (struct request_queue *, struct request *);
215
typedef void (softirq_done_fn)(struct request *);
216
typedef int (dma_drain_needed_fn)(struct request *);
217
typedef int (lld_busy_fn) (struct request_queue *q);
L
Linus Torvalds 已提交
218

J
Jens Axboe 已提交
219 220 221 222 223 224 225 226
enum blk_eh_timer_return {
	BLK_EH_NOT_HANDLED,
	BLK_EH_HANDLED,
	BLK_EH_RESET_TIMER,
};

typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);

L
Linus Torvalds 已提交
227 228 229 230 231 232 233 234 235 236
enum blk_queue_state {
	Queue_down,
	Queue_up,
};

struct blk_queue_tag {
	struct request **tag_index;	/* map of busy tags */
	unsigned long *tag_map;		/* bit map of free/busy tags */
	int busy;			/* current depth */
	int max_depth;			/* what we will send to device */
237
	int real_max_depth;		/* what the array can hold */
L
Linus Torvalds 已提交
238 239 240
	atomic_t refcnt;		/* map can be shared */
};

241 242 243
#define BLK_SCSI_MAX_CMDS	(256)
#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))

244 245 246 247 248 249 250
struct queue_limits {
	unsigned long		bounce_pfn;
	unsigned long		seg_boundary_mask;

	unsigned int		max_hw_sectors;
	unsigned int		max_sectors;
	unsigned int		max_segment_size;
251 252 253 254
	unsigned int		physical_block_size;
	unsigned int		alignment_offset;
	unsigned int		io_min;
	unsigned int		io_opt;
255
	unsigned int		max_discard_sectors;
256 257
	unsigned int		discard_granularity;
	unsigned int		discard_alignment;
258 259

	unsigned short		logical_block_size;
260
	unsigned short		max_segments;
261

262
	unsigned char		misaligned;
263
	unsigned char		discard_misaligned;
264
	unsigned char		no_cluster;
265
	signed char		discard_zeroes_data;
266 267
};

L
Linus Torvalds 已提交
268 269 270 271 272 273 274
struct request_queue
{
	/*
	 * Together with queue_head for cacheline sharing
	 */
	struct list_head	queue_head;
	struct request		*last_merge;
J
Jens Axboe 已提交
275
	struct elevator_queue	*elevator;
L
Linus Torvalds 已提交
276 277 278 279 280 281 282 283 284 285 286 287

	/*
	 * the queue request freelist, one for reads and one for writes
	 */
	struct request_list	rq;

	request_fn_proc		*request_fn;
	make_request_fn		*make_request_fn;
	prep_rq_fn		*prep_rq_fn;
	unplug_fn		*unplug_fn;
	merge_bvec_fn		*merge_bvec_fn;
	prepare_flush_fn	*prepare_flush_fn;
288
	softirq_done_fn		*softirq_done_fn;
J
Jens Axboe 已提交
289
	rq_timed_out_fn		*rq_timed_out_fn;
290
	dma_drain_needed_fn	*dma_drain_needed;
291
	lld_busy_fn		*lld_busy_fn;
L
Linus Torvalds 已提交
292

293 294 295
	/*
	 * Dispatch queue sorting
	 */
J
Jens Axboe 已提交
296
	sector_t		end_sector;
297 298
	struct request		*boundary_rq;

L
Linus Torvalds 已提交
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	/*
	 * Auto-unplugging state
	 */
	struct timer_list	unplug_timer;
	int			unplug_thresh;	/* After this many requests */
	unsigned long		unplug_delay;	/* After this many jiffies */
	struct work_struct	unplug_work;

	struct backing_dev_info	backing_dev_info;

	/*
	 * The queue owner gets to use this for whatever they like.
	 * ll_rw_blk doesn't touch it.
	 */
	void			*queuedata;

	/*
	 * queue needs bounce pages for pages above this limit
	 */
A
Al Viro 已提交
318
	gfp_t			bounce_gfp;
L
Linus Torvalds 已提交
319 320 321 322 323 324 325

	/*
	 * various queue flags, see QUEUE_* below
	 */
	unsigned long		queue_flags;

	/*
已提交
326 327 328
	 * protects queue structures from reentrancy. ->__queue_lock should
	 * _never_ be used directly, it is queue private. always use
	 * ->queue_lock.
L
Linus Torvalds 已提交
329
	 */
已提交
330
	spinlock_t		__queue_lock;
L
Linus Torvalds 已提交
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
	spinlock_t		*queue_lock;

	/*
	 * queue kobject
	 */
	struct kobject kobj;

	/*
	 * queue settings
	 */
	unsigned long		nr_requests;	/* Max # of requests */
	unsigned int		nr_congestion_on;
	unsigned int		nr_congestion_off;
	unsigned int		nr_batching;

J
James Bottomley 已提交
346 347
	void			*dma_drain_buffer;
	unsigned int		dma_drain_size;
348
	unsigned int		dma_pad_mask;
L
Linus Torvalds 已提交
349 350 351
	unsigned int		dma_alignment;

	struct blk_queue_tag	*queue_tags;
352
	struct list_head	tag_busy_list;
L
Linus Torvalds 已提交
353

354
	unsigned int		nr_sorted;
355
	unsigned int		in_flight[2];
L
Linus Torvalds 已提交
356

J
Jens Axboe 已提交
357 358 359 360
	unsigned int		rq_timeout;
	struct timer_list	timeout;
	struct list_head	timeout_list;

361 362
	struct queue_limits	limits;

L
Linus Torvalds 已提交
363 364 365 366 367
	/*
	 * sg stuff
	 */
	unsigned int		sg_timeout;
	unsigned int		sg_reserved_size;
368
	int			node;
369
#ifdef CONFIG_BLK_DEV_IO_TRACE
370
	struct blk_trace	*blk_trace;
371
#endif
L
Linus Torvalds 已提交
372 373 374
	/*
	 * reserved for flush operations
	 */
375 376 377 378
	unsigned int		ordered, next_ordered, ordseq;
	int			orderr, ordcolor;
	struct request		pre_flush_rq, bar_rq, post_flush_rq;
	struct request		*orig_bar_rq;
379 380

	struct mutex		sysfs_lock;
381 382 383 384

#if defined(CONFIG_BLK_DEV_BSG)
	struct bsg_class_device bsg_dev;
#endif
L
Linus Torvalds 已提交
385 386 387 388 389
};

#define QUEUE_FLAG_CLUSTER	0	/* cluster several segments into 1 */
#define QUEUE_FLAG_QUEUED	1	/* uses generic tag queueing */
#define QUEUE_FLAG_STOPPED	2	/* queue is stopped */
390 391
#define	QUEUE_FLAG_SYNCFULL	3	/* read queue has been filled */
#define QUEUE_FLAG_ASYNCFULL	4	/* write queue has been filled */
L
Linus Torvalds 已提交
392 393 394
#define QUEUE_FLAG_DEAD		5	/* queue being torn down */
#define QUEUE_FLAG_REENTER	6	/* Re-entrancy avoidance */
#define QUEUE_FLAG_PLUGGED	7	/* queue is plugged */
J
Jens Axboe 已提交
395
#define QUEUE_FLAG_ELVSWITCH	8	/* don't use elevator, just do FIFO */
396
#define QUEUE_FLAG_BIDI		9	/* queue supports bidi requests */
397
#define QUEUE_FLAG_NOMERGES    10	/* disable merge attempts */
398
#define QUEUE_FLAG_SAME_COMP   11	/* force complete on same CPU */
399
#define QUEUE_FLAG_FAIL_IO     12	/* fake timeout */
400
#define QUEUE_FLAG_STACKABLE   13	/* supports request stacking */
401
#define QUEUE_FLAG_NONROT      14	/* non-rotational device (SSD) */
402
#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
403
#define QUEUE_FLAG_IO_STAT     15	/* do IO stats */
404
#define QUEUE_FLAG_DISCARD     16	/* supports DISCARD */
405
#define QUEUE_FLAG_NOXMERGES   17	/* No extended merges */
406
#define QUEUE_FLAG_ADD_RANDOM  18	/* Contributes to random pool */
407 408 409

#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
				 (1 << QUEUE_FLAG_CLUSTER) |		\
410
				 (1 << QUEUE_FLAG_STACKABLE)	|	\
411 412
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_ADD_RANDOM))
413

414 415
static inline int queue_is_locked(struct request_queue *q)
{
J
Jens Axboe 已提交
416
#ifdef CONFIG_SMP
417 418
	spinlock_t *lock = q->queue_lock;
	return lock && spin_is_locked(lock);
J
Jens Axboe 已提交
419 420 421
#else
	return 1;
#endif
422 423
}

N
Nick Piggin 已提交
424 425 426 427 428 429
static inline void queue_flag_set_unlocked(unsigned int flag,
					   struct request_queue *q)
{
	__set_bit(flag, &q->queue_flags);
}

J
Jens Axboe 已提交
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
static inline int queue_flag_test_and_clear(unsigned int flag,
					    struct request_queue *q)
{
	WARN_ON_ONCE(!queue_is_locked(q));

	if (test_bit(flag, &q->queue_flags)) {
		__clear_bit(flag, &q->queue_flags);
		return 1;
	}

	return 0;
}

static inline int queue_flag_test_and_set(unsigned int flag,
					  struct request_queue *q)
{
	WARN_ON_ONCE(!queue_is_locked(q));

	if (!test_bit(flag, &q->queue_flags)) {
		__set_bit(flag, &q->queue_flags);
		return 0;
	}

	return 1;
}

N
Nick Piggin 已提交
456 457
static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
{
458
	WARN_ON_ONCE(!queue_is_locked(q));
N
Nick Piggin 已提交
459 460 461 462 463 464 465 466 467
	__set_bit(flag, &q->queue_flags);
}

static inline void queue_flag_clear_unlocked(unsigned int flag,
					     struct request_queue *q)
{
	__clear_bit(flag, &q->queue_flags);
}

468 469 470 471 472
static inline int queue_in_flight(struct request_queue *q)
{
	return q->in_flight[0] + q->in_flight[1];
}

N
Nick Piggin 已提交
473 474
static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
{
475
	WARN_ON_ONCE(!queue_is_locked(q));
N
Nick Piggin 已提交
476 477 478
	__clear_bit(flag, &q->queue_flags);
}

479 480 481 482 483 484 485 486 487 488 489 490
enum {
	/*
	 * Hardbarrier is supported with one of the following methods.
	 *
	 * NONE		: hardbarrier unsupported
	 * DRAIN	: ordering by draining is enough
	 * DRAIN_FLUSH	: ordering by draining w/ pre and post flushes
	 * DRAIN_FUA	: ordering by draining w/ pre flush and FUA write
	 * TAG		: ordering by tag is enough
	 * TAG_FLUSH	: ordering by tag w/ pre and post flushes
	 * TAG_FUA	: ordering by tag w/ pre flush and FUA write
	 */
491 492 493
	QUEUE_ORDERED_BY_DRAIN		= 0x01,
	QUEUE_ORDERED_BY_TAG		= 0x02,
	QUEUE_ORDERED_DO_PREFLUSH	= 0x10,
494
	QUEUE_ORDERED_DO_BAR		= 0x20,
495 496 497 498 499
	QUEUE_ORDERED_DO_POSTFLUSH	= 0x40,
	QUEUE_ORDERED_DO_FUA		= 0x80,

	QUEUE_ORDERED_NONE		= 0x00,

500 501
	QUEUE_ORDERED_DRAIN		= QUEUE_ORDERED_BY_DRAIN |
					  QUEUE_ORDERED_DO_BAR,
502 503 504 505 506 507 508
	QUEUE_ORDERED_DRAIN_FLUSH	= QUEUE_ORDERED_DRAIN |
					  QUEUE_ORDERED_DO_PREFLUSH |
					  QUEUE_ORDERED_DO_POSTFLUSH,
	QUEUE_ORDERED_DRAIN_FUA		= QUEUE_ORDERED_DRAIN |
					  QUEUE_ORDERED_DO_PREFLUSH |
					  QUEUE_ORDERED_DO_FUA,

509 510
	QUEUE_ORDERED_TAG		= QUEUE_ORDERED_BY_TAG |
					  QUEUE_ORDERED_DO_BAR,
511 512 513 514 515 516
	QUEUE_ORDERED_TAG_FLUSH		= QUEUE_ORDERED_TAG |
					  QUEUE_ORDERED_DO_PREFLUSH |
					  QUEUE_ORDERED_DO_POSTFLUSH,
	QUEUE_ORDERED_TAG_FUA		= QUEUE_ORDERED_TAG |
					  QUEUE_ORDERED_DO_PREFLUSH |
					  QUEUE_ORDERED_DO_FUA,
517 518 519 520 521 522 523 524 525 526 527

	/*
	 * Ordered operation sequence
	 */
	QUEUE_ORDSEQ_STARTED	= 0x01,	/* flushing in progress */
	QUEUE_ORDSEQ_DRAIN	= 0x02,	/* waiting for the queue to be drained */
	QUEUE_ORDSEQ_PREFLUSH	= 0x04,	/* pre-flushing in progress */
	QUEUE_ORDSEQ_BAR	= 0x08,	/* original barrier req in progress */
	QUEUE_ORDSEQ_POSTFLUSH	= 0x10,	/* post-flushing in progress */
	QUEUE_ORDSEQ_DONE	= 0x20,
};
L
Linus Torvalds 已提交
528 529 530 531

#define blk_queue_plugged(q)	test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags)
#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
532
#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
533 534
#define blk_queue_noxmerges(q)	\
	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
535
#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
536
#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
537
#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
538
#define blk_queue_flushing(q)	((q)->ordseq)
539 540
#define blk_queue_stackable(q)	\
	test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
541
#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
L
Linus Torvalds 已提交
542

543 544 545 546 547 548 549 550 551
#define blk_noretry_request(rq) \
	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
			     REQ_FAILFAST_DRIVER))

#define blk_account_rq(rq) \
	(((rq)->cmd_flags & REQ_STARTED) && \
	 ((rq)->cmd_type == REQ_TYPE_FS || \
	  ((rq)->cmd_flags & REQ_DISCARD)))

L
Linus Torvalds 已提交
552
#define blk_pm_request(rq)	\
553 554
	((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
	 (rq)->cmd_type == REQ_TYPE_PM_RESUME)
L
Linus Torvalds 已提交
555

556
#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
557
#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
558 559
/* rq->queuelist of dequeued request must be list_empty() */
#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
L
Linus Torvalds 已提交
560 561 562

#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)

563
#define rq_data_dir(rq)		((rq)->cmd_flags & 1)
L
Linus Torvalds 已提交
564

565
/*
566
 * We regard a request as sync, if either a read or a sync write
567
 */
568 569
static inline bool rw_is_sync(unsigned int rw_flags)
{
570
	return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
571 572 573 574 575 576 577 578
}

static inline bool rq_is_sync(struct request *rq)
{
	return rw_is_sync(rq->cmd_flags);
}

static inline int blk_queue_full(struct request_queue *q, int sync)
L
Linus Torvalds 已提交
579
{
580 581 582
	if (sync)
		return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags);
	return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags);
L
Linus Torvalds 已提交
583 584
}

585
static inline void blk_set_queue_full(struct request_queue *q, int sync)
L
Linus Torvalds 已提交
586
{
587 588
	if (sync)
		queue_flag_set(QUEUE_FLAG_SYNCFULL, q);
L
Linus Torvalds 已提交
589
	else
590
		queue_flag_set(QUEUE_FLAG_ASYNCFULL, q);
L
Linus Torvalds 已提交
591 592
}

593
static inline void blk_clear_queue_full(struct request_queue *q, int sync)
L
Linus Torvalds 已提交
594
{
595 596
	if (sync)
		queue_flag_clear(QUEUE_FLAG_SYNCFULL, q);
L
Linus Torvalds 已提交
597
	else
598
		queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q);
L
Linus Torvalds 已提交
599 600 601 602 603 604 605 606 607 608
}


/*
 * mergeable request must not have _NOMERGE or _BARRIER bit set, nor may
 * it already be started by driver.
 */
#define RQ_NOMERGE_FLAGS	\
	(REQ_NOMERGE | REQ_STARTED | REQ_HARDBARRIER | REQ_SOFTBARRIER)
#define rq_mergeable(rq)	\
609
	(!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \
610 611
	 (((rq)->cmd_flags & REQ_DISCARD) || \
	  (rq)->cmd_type == REQ_TYPE_FS))
L
Linus Torvalds 已提交
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628

/*
 * q->prep_rq_fn return values
 */
#define BLKPREP_OK		0	/* serve it */
#define BLKPREP_KILL		1	/* fatal error, kill */
#define BLKPREP_DEFER		2	/* leave on queue */

extern unsigned long blk_max_low_pfn, blk_max_pfn;

/*
 * standard bounce addresses:
 *
 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 * BLK_BOUNCE_ANY	: don't bounce anything
 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 */
629 630

#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
631
#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
632 633 634 635
#else
#define BLK_BOUNCE_HIGH		-1ULL
#endif
#define BLK_BOUNCE_ANY		(-1ULL)
636
#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
L
Linus Torvalds 已提交
637

638 639 640 641
/*
 * default timeout for SG_IO if none specified
 */
#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
L
Linus Torvalds 已提交
642
#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
643

644
#ifdef CONFIG_BOUNCE
L
Linus Torvalds 已提交
645
extern int init_emergency_isa_pool(void);
646
extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
L
Linus Torvalds 已提交
647 648 649 650 651
#else
static inline int init_emergency_isa_pool(void)
{
	return 0;
}
652
static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
L
Linus Torvalds 已提交
653 654 655 656
{
}
#endif /* CONFIG_MMU */

657 658 659 660
struct rq_map_data {
	struct page **pages;
	int page_order;
	int nr_entries;
661
	unsigned long offset;
662
	int null_mapped;
663
	int from_user;
664 665
};

666 667 668 669 670 671
struct req_iterator {
	int i;
	struct bio *bio;
};

/* This should not be used directly - use rq_for_each_segment */
672 673
#define for_each_bio(_bio)		\
	for (; _bio; _bio = _bio->bi_next)
674
#define __rq_for_each_bio(_bio, rq)	\
L
Linus Torvalds 已提交
675 676 677
	if ((rq->bio))			\
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

678 679 680 681 682 683 684
#define rq_for_each_segment(bvl, _rq, _iter)			\
	__rq_for_each_bio(_iter.bio, _rq)			\
		bio_for_each_segment(bvl, _iter.bio, _iter.i)

#define rq_iter_last(rq, _iter)					\
		(_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1)

685 686 687 688 689 690 691 692 693 694 695
#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
#endif
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
extern void rq_flush_dcache_pages(struct request *rq);
#else
static inline void rq_flush_dcache_pages(struct request *rq)
{
}
#endif

L
Linus Torvalds 已提交
696 697 698 699
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
extern void register_disk(struct gendisk *dev);
extern void generic_make_request(struct bio *bio);
700
extern void blk_rq_init(struct request_queue *q, struct request *rq);
L
Linus Torvalds 已提交
701
extern void blk_put_request(struct request *);
702 703
extern void __blk_put_request(struct request_queue *, struct request *);
extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
704 705
extern struct request *blk_make_request(struct request_queue *, struct bio *,
					gfp_t);
706 707
extern void blk_insert_request(struct request_queue *, struct request *, int, void *);
extern void blk_requeue_request(struct request_queue *, struct request *);
708 709
extern void blk_add_request_payload(struct request *rq, struct page *page,
		unsigned int len);
710
extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
711
extern int blk_lld_busy(struct request_queue *q);
712 713 714 715 716
extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
			     struct bio_set *bs, gfp_t gfp_mask,
			     int (*bio_ctr)(struct bio *, struct bio *, void *),
			     void *data);
extern void blk_rq_unprep_clone(struct request *rq);
717 718
extern int blk_insert_cloned_request(struct request_queue *q,
				     struct request *rq);
719
extern void blk_plug_device(struct request_queue *);
720
extern void blk_plug_device_unlocked(struct request_queue *);
721 722
extern int blk_remove_plug(struct request_queue *);
extern void blk_recount_segments(struct request_queue *, struct bio *);
723 724
extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			  unsigned int, void __user *);
725 726
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			 struct scsi_ioctl_command __user *);
727 728 729 730 731 732

/*
 * A queue has just exitted congestion.  Note this in the global counter of
 * congested queues, and wake up anyone who was waiting for requests to be
 * put back.
 */
733
static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
734
{
735
	clear_bdi_congested(&q->backing_dev_info, sync);
736 737 738 739 740 741
}

/*
 * A queue has just entered congestion.  Flag that in the queue's VM-visible
 * state flags and increment the global gounter of congested queues.
 */
742
static inline void blk_set_queue_congested(struct request_queue *q, int sync)
743
{
744
	set_bdi_congested(&q->backing_dev_info, sync);
745 746
}

747 748
extern void blk_start_queue(struct request_queue *q);
extern void blk_stop_queue(struct request_queue *q);
L
Linus Torvalds 已提交
749
extern void blk_sync_queue(struct request_queue *q);
750
extern void __blk_stop_queue(struct request_queue *q);
N
Nick Piggin 已提交
751
extern void __blk_run_queue(struct request_queue *);
752
extern void blk_run_queue(struct request_queue *);
753
extern int blk_rq_map_user(struct request_queue *, struct request *,
754 755
			   struct rq_map_data *, void __user *, unsigned long,
			   gfp_t);
J
Jens Axboe 已提交
756
extern int blk_rq_unmap_user(struct bio *);
757 758
extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
759 760
			       struct rq_map_data *, struct sg_iovec *, int,
			       unsigned int, gfp_t);
761
extern int blk_execute_rq(struct request_queue *, struct gendisk *,
762
			  struct request *, int);
763
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
764
				  struct request *, int, rq_end_io_fn *);
765
extern void blk_unplug(struct request_queue *q);
766

767
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
L
Linus Torvalds 已提交
768 769 770 771
{
	return bdev->bd_disk->queue;
}

772
/*
773 774 775 776 777 778
 * blk_rq_pos()			: the current sector
 * blk_rq_bytes()		: bytes left in the entire request
 * blk_rq_cur_bytes()		: bytes left in the current segment
 * blk_rq_err_bytes()		: bytes left till the next error boundary
 * blk_rq_sectors()		: sectors left in the entire request
 * blk_rq_cur_sectors()		: sectors left in the current segment
779
 */
780 781
static inline sector_t blk_rq_pos(const struct request *rq)
{
782
	return rq->__sector;
783 784 785 786
}

static inline unsigned int blk_rq_bytes(const struct request *rq)
{
787
	return rq->__data_len;
788 789
}

790 791 792 793
static inline int blk_rq_cur_bytes(const struct request *rq)
{
	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
}
794

795 796
extern unsigned int blk_rq_err_bytes(const struct request *rq);

797 798
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
799
	return blk_rq_bytes(rq) >> 9;
800 801 802 803
}

static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
804
	return blk_rq_cur_bytes(rq) >> 9;
805 806
}

807 808 809 810 811 812 813
/*
 * Request issue related functions.
 */
extern struct request *blk_peek_request(struct request_queue *q);
extern void blk_start_request(struct request *rq);
extern struct request *blk_fetch_request(struct request_queue *q);

L
Linus Torvalds 已提交
814
/*
815 816 817 818 819
 * Request completion related functions.
 *
 * blk_update_request() completes given number of bytes and updates
 * the request without completing it.
 *
820 821
 * blk_end_request() and friends.  __blk_end_request() must be called
 * with the request queue spinlock acquired.
L
Linus Torvalds 已提交
822 823
 *
 * Several drivers define their own end_request and call
824 825
 * blk_end_request() for parts of the original function.
 * This prevents code duplication in drivers.
L
Linus Torvalds 已提交
826
 */
827 828
extern bool blk_update_request(struct request *rq, int error,
			       unsigned int nr_bytes);
829 830 831 832
extern bool blk_end_request(struct request *rq, int error,
			    unsigned int nr_bytes);
extern void blk_end_request_all(struct request *rq, int error);
extern bool blk_end_request_cur(struct request *rq, int error);
833
extern bool blk_end_request_err(struct request *rq, int error);
834 835 836 837
extern bool __blk_end_request(struct request *rq, int error,
			      unsigned int nr_bytes);
extern void __blk_end_request_all(struct request *rq, int error);
extern bool __blk_end_request_cur(struct request *rq, int error);
838
extern bool __blk_end_request_err(struct request *rq, int error);
839

840
extern void blk_complete_request(struct request *);
J
Jens Axboe 已提交
841 842
extern void __blk_complete_request(struct request *);
extern void blk_abort_request(struct request *);
843
extern void blk_abort_queue(struct request_queue *);
844

L
Linus Torvalds 已提交
845 846 847
/*
 * Access functions for manipulating queue properties
 */
848
extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
849
					spinlock_t *lock, int node_id);
850 851 852
extern struct request_queue *blk_init_allocated_queue_node(struct request_queue *,
							   request_fn_proc *,
							   spinlock_t *, int node_id);
853
extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
854 855
extern struct request_queue *blk_init_allocated_queue(struct request_queue *,
						      request_fn_proc *, spinlock_t *);
856 857 858
extern void blk_cleanup_queue(struct request_queue *);
extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
extern void blk_queue_bounce_limit(struct request_queue *, u64);
859
extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
860
extern void blk_queue_max_segments(struct request_queue *, unsigned short);
861
extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
862 863
extern void blk_queue_max_discard_sectors(struct request_queue *q,
		unsigned int max_discard_sectors);
864
extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
865 866 867
extern void blk_queue_physical_block_size(struct request_queue *, unsigned short);
extern void blk_queue_alignment_offset(struct request_queue *q,
				       unsigned int alignment);
868
extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
869
extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
870
extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
871
extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
872
extern void blk_set_default_limits(struct queue_limits *lim);
873 874
extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
			    sector_t offset);
875 876
extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
			    sector_t offset);
877 878
extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
			      sector_t offset);
879
extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
880
extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
881
extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
882 883 884
extern int blk_queue_dma_drain(struct request_queue *q,
			       dma_drain_needed_fn *dma_drain_needed,
			       void *buf, unsigned int size);
885
extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
886 887 888 889
extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *);
extern void blk_queue_dma_alignment(struct request_queue *, int);
890
extern void blk_queue_update_dma_alignment(struct request_queue *, int);
891
extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
J
Jens Axboe 已提交
892 893
extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
L
Linus Torvalds 已提交
894
extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
895
extern int blk_queue_ordered(struct request_queue *, unsigned, prepare_flush_fn *);
896
extern bool blk_do_ordered(struct request_queue *, struct request **);
897
extern unsigned blk_ordered_cur_seq(struct request_queue *);
898
extern unsigned blk_ordered_req_seq(struct request *);
899
extern bool blk_ordered_complete_seq(struct request_queue *, unsigned, int);
L
Linus Torvalds 已提交
900

901
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
L
Linus Torvalds 已提交
902
extern void blk_dump_rq_flags(struct request *, char *);
903
extern void generic_unplug_device(struct request_queue *);
L
Linus Torvalds 已提交
904 905
extern long nr_blockdev_pages(void);

906 907 908 909
int blk_get_queue(struct request_queue *);
struct request_queue *blk_alloc_queue(gfp_t);
struct request_queue *blk_alloc_queue_node(gfp_t, int);
extern void blk_put_queue(struct request_queue *);
L
Linus Torvalds 已提交
910 911 912 913

/*
 * tag stuff
 */
914
#define blk_rq_tagged(rq)		((rq)->cmd_flags & REQ_QUEUED)
915 916 917 918 919 920 921
extern int blk_queue_start_tag(struct request_queue *, struct request *);
extern struct request *blk_queue_find_tag(struct request_queue *, int);
extern void blk_queue_end_tag(struct request_queue *, struct request *);
extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *);
extern void blk_queue_free_tags(struct request_queue *);
extern int blk_queue_resize_tags(struct request_queue *, int);
extern void blk_queue_invalidate_tags(struct request_queue *);
922 923
extern struct blk_queue_tag *blk_init_tags(int);
extern void blk_free_tags(struct blk_queue_tag *);
L
Linus Torvalds 已提交
924

925 926 927 928 929 930 931
static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
						int tag)
{
	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
		return NULL;
	return bqt->tag_index[tag];
}
932 933 934 935 936 937 938 939 940 941
enum{
	BLKDEV_WAIT,	/* wait for completion */
	BLKDEV_BARRIER,	/*issue request with barrier */
};
#define BLKDEV_IFL_WAIT		(1 << BLKDEV_WAIT)
#define BLKDEV_IFL_BARRIER	(1 << BLKDEV_BARRIER)
extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *,
			unsigned long);
extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
942 943
extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
			sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
D
David Woodhouse 已提交
944
static inline int sb_issue_discard(struct super_block *sb,
945
				   sector_t block, sector_t nr_blocks)
D
David Woodhouse 已提交
946 947 948
{
	block <<= (sb->s_blocksize_bits - 9);
	nr_blocks <<= (sb->s_blocksize_bits - 9);
949
	return blkdev_issue_discard(sb->s_bdev, block, nr_blocks, GFP_KERNEL,
950
				   BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER);
D
David Woodhouse 已提交
951
}
L
Linus Torvalds 已提交
952

953
extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
954

955 956 957 958 959 960 961
enum blk_default_limits {
	BLK_MAX_SEGMENTS	= 128,
	BLK_SAFE_MAX_SECTORS	= 255,
	BLK_DEF_MAX_SECTORS	= 1024,
	BLK_MAX_SEGMENT_SIZE	= 65536,
	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
};
962

L
Linus Torvalds 已提交
963 964
#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)

965 966
static inline unsigned long queue_bounce_pfn(struct request_queue *q)
{
967
	return q->limits.bounce_pfn;
968 969 970 971
}

static inline unsigned long queue_segment_boundary(struct request_queue *q)
{
972
	return q->limits.seg_boundary_mask;
973 974 975 976
}

static inline unsigned int queue_max_sectors(struct request_queue *q)
{
977
	return q->limits.max_sectors;
978 979 980 981
}

static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
{
982
	return q->limits.max_hw_sectors;
983 984
}

985
static inline unsigned short queue_max_segments(struct request_queue *q)
986
{
987
	return q->limits.max_segments;
988 989 990 991
}

static inline unsigned int queue_max_segment_size(struct request_queue *q)
{
992
	return q->limits.max_segment_size;
993 994
}

995
static inline unsigned short queue_logical_block_size(struct request_queue *q)
L
Linus Torvalds 已提交
996 997 998
{
	int retval = 512;

999 1000
	if (q && q->limits.logical_block_size)
		retval = q->limits.logical_block_size;
L
Linus Torvalds 已提交
1001 1002 1003 1004

	return retval;
}

1005
static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1006
{
1007
	return queue_logical_block_size(bdev_get_queue(bdev));
L
Linus Torvalds 已提交
1008 1009
}

1010 1011 1012 1013 1014
static inline unsigned int queue_physical_block_size(struct request_queue *q)
{
	return q->limits.physical_block_size;
}

M
Martin K. Petersen 已提交
1015 1016 1017 1018 1019
static inline int bdev_physical_block_size(struct block_device *bdev)
{
	return queue_physical_block_size(bdev_get_queue(bdev));
}

1020 1021 1022 1023 1024
static inline unsigned int queue_io_min(struct request_queue *q)
{
	return q->limits.io_min;
}

M
Martin K. Petersen 已提交
1025 1026 1027 1028 1029
static inline int bdev_io_min(struct block_device *bdev)
{
	return queue_io_min(bdev_get_queue(bdev));
}

1030 1031 1032 1033 1034
static inline unsigned int queue_io_opt(struct request_queue *q)
{
	return q->limits.io_opt;
}

M
Martin K. Petersen 已提交
1035 1036 1037 1038 1039
static inline int bdev_io_opt(struct block_device *bdev)
{
	return queue_io_opt(bdev_get_queue(bdev));
}

1040 1041
static inline int queue_alignment_offset(struct request_queue *q)
{
M
Martin K. Petersen 已提交
1042
	if (q->limits.misaligned)
1043 1044
		return -1;

M
Martin K. Petersen 已提交
1045
	return q->limits.alignment_offset;
1046 1047
}

1048
static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1049 1050
{
	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1051
	unsigned int alignment = (sector << 9) & (granularity - 1);
1052

1053 1054
	return (granularity + lim->alignment_offset - alignment)
		& (granularity - 1);
1055 1056
}

M
Martin K. Petersen 已提交
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069
static inline int bdev_alignment_offset(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q->limits.misaligned)
		return -1;

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->alignment_offset;

	return q->limits.alignment_offset;
}

1070 1071 1072 1073 1074 1075 1076 1077
static inline int queue_discard_alignment(struct request_queue *q)
{
	if (q->limits.discard_misaligned)
		return -1;

	return q->limits.discard_alignment;
}

1078
static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1079
{
1080 1081 1082 1083
	unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1);

	return (lim->discard_granularity + lim->discard_alignment - alignment)
		& (lim->discard_granularity - 1);
1084 1085
}

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
{
	if (q->limits.discard_zeroes_data == 1)
		return 1;

	return 0;
}

static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev)
{
	return queue_discard_zeroes_data(bdev_get_queue(bdev));
}

1099
static inline int queue_dma_alignment(struct request_queue *q)
L
Linus Torvalds 已提交
1100
{
1101
	return q ? q->dma_alignment : 511;
L
Linus Torvalds 已提交
1102 1103
}

1104 1105 1106 1107 1108 1109 1110
static inline int blk_rq_aligned(struct request_queue *q, void *addr,
				 unsigned int len)
{
	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
	return !((unsigned long)addr & alignment) && !(len & alignment);
}

L
Linus Torvalds 已提交
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
/* assumes size > 256 */
static inline unsigned int blksize_bits(unsigned int size)
{
	unsigned int bits = 8;
	do {
		bits++;
		size >>= 1;
	} while (size > 256);
	return bits;
}

1122
static inline unsigned int block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
{
	return bdev->bd_block_size;
}

typedef struct {struct page *v;} Sector;

unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);

static inline void put_dev_sector(Sector p)
{
	page_cache_release(p.v);
}

struct work_struct;
1137
int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
L
Linus Torvalds 已提交
1138

1139
#ifdef CONFIG_BLK_CGROUP
1140 1141 1142 1143 1144
/*
 * This should not be using sched_clock(). A real patch is in progress
 * to fix this up, until that is in place we need to disable preemption
 * around sched_clock() in this function and set_io_start_time_ns().
 */
1145 1146
static inline void set_start_time_ns(struct request *req)
{
1147
	preempt_disable();
1148
	req->start_time_ns = sched_clock();
1149
	preempt_enable();
1150 1151 1152 1153
}

static inline void set_io_start_time_ns(struct request *req)
{
1154
	preempt_disable();
1155
	req->io_start_time_ns = sched_clock();
1156
	preempt_enable();
1157
}
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167

static inline uint64_t rq_start_time_ns(struct request *req)
{
        return req->start_time_ns;
}

static inline uint64_t rq_io_start_time_ns(struct request *req)
{
        return req->io_start_time_ns;
}
1168 1169 1170
#else
static inline void set_start_time_ns(struct request *req) {}
static inline void set_io_start_time_ns(struct request *req) {}
1171 1172 1173 1174 1175 1176 1177 1178
static inline uint64_t rq_start_time_ns(struct request *req)
{
	return 0;
}
static inline uint64_t rq_io_start_time_ns(struct request *req)
{
	return 0;
}
1179 1180
#endif

L
Linus Torvalds 已提交
1181 1182 1183 1184 1185
#define MODULE_ALIAS_BLOCKDEV(major,minor) \
	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
	MODULE_ALIAS("block-major-" __stringify(major) "-*")

1186 1187
#if defined(CONFIG_BLK_DEV_INTEGRITY)

1188 1189
#define INTEGRITY_FLAG_READ	2	/* verify data integrity on read */
#define INTEGRITY_FLAG_WRITE	4	/* generate data integrity on write */
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222

struct blk_integrity_exchg {
	void			*prot_buf;
	void			*data_buf;
	sector_t		sector;
	unsigned int		data_size;
	unsigned short		sector_size;
	const char		*disk_name;
};

typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);

struct blk_integrity {
	integrity_gen_fn	*generate_fn;
	integrity_vrfy_fn	*verify_fn;
	integrity_set_tag_fn	*set_tag_fn;
	integrity_get_tag_fn	*get_tag_fn;

	unsigned short		flags;
	unsigned short		tuple_size;
	unsigned short		sector_size;
	unsigned short		tag_size;

	const char		*name;

	struct kobject		kobj;
};

extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
extern void blk_integrity_unregister(struct gendisk *);
1223
extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1224 1225 1226
extern int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
extern int blk_rq_count_integrity_sg(struct request *);

1227 1228 1229 1230 1231 1232
static inline
struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
{
	return bdev->bd_disk->integrity;
}

1233 1234 1235 1236 1237
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
	return disk->integrity;
}

1238 1239
static inline int blk_integrity_rq(struct request *rq)
{
1240 1241 1242
	if (rq->bio == NULL)
		return 0;

1243 1244 1245 1246 1247 1248 1249 1250
	return bio_integrity(rq->bio);
}

#else /* CONFIG_BLK_DEV_INTEGRITY */

#define blk_integrity_rq(rq)			(0)
#define blk_rq_count_integrity_sg(a)		(0)
#define blk_rq_map_integrity_sg(a, b)		(0)
1251
#define bdev_get_integrity(a)			(0)
1252
#define blk_get_integrity(a)			(0)
1253 1254 1255 1256 1257 1258
#define blk_integrity_compare(a, b)		(0)
#define blk_integrity_register(a, b)		(0)
#define blk_integrity_unregister(a)		do { } while (0);

#endif /* CONFIG_BLK_DEV_INTEGRITY */

1259
struct block_device_operations {
A
Al Viro 已提交
1260 1261 1262 1263 1264
	int (*open) (struct block_device *, fmode_t);
	int (*release) (struct gendisk *, fmode_t);
	int (*locked_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1265 1266 1267
	int (*direct_access) (struct block_device *, sector_t,
						void **, unsigned long *);
	int (*media_changed) (struct gendisk *);
1268
	void (*unlock_native_capacity) (struct gendisk *);
1269 1270
	int (*revalidate_disk) (struct gendisk *);
	int (*getgeo)(struct block_device *, struct hd_geometry *);
1271 1272
	/* this callback is with swap_lock and sometimes page table lock held */
	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1273 1274 1275
	struct module *owner;
};

1276 1277
extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
				 unsigned long);
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
#else /* CONFIG_BLOCK */
/*
 * stubs for when the block layer is configured out
 */
#define buffer_heads_over_limit 0

static inline long nr_blockdev_pages(void)
{
	return 0;
}

#endif /* CONFIG_BLOCK */

L
Linus Torvalds 已提交
1291
#endif