blkdev.h 60.0 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4
#ifndef _LINUX_BLKDEV_H
#define _LINUX_BLKDEV_H

5
#include <linux/sched.h>
6
#include <linux/sched/clock.h>
7

J
Jens Axboe 已提交
8 9
#ifdef CONFIG_BLOCK

L
Linus Torvalds 已提交
10 11 12
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/list.h>
13
#include <linux/llist.h>
L
Linus Torvalds 已提交
14 15 16
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/pagemap.h>
17
#include <linux/backing-dev-defs.h>
L
Linus Torvalds 已提交
18 19
#include <linux/wait.h>
#include <linux/mempool.h>
D
Dan Williams 已提交
20
#include <linux/pfn.h>
L
Linus Torvalds 已提交
21 22
#include <linux/bio.h>
#include <linux/stringify.h>
23
#include <linux/gfp.h>
24
#include <linux/bsg.h>
25
#include <linux/smp.h>
T
Tejun Heo 已提交
26
#include <linux/rcupdate.h>
27
#include <linux/percpu-refcount.h>
C
Christoph Hellwig 已提交
28
#include <linux/scatterlist.h>
29
#include <linux/blkzoned.h>
30 31
#include <linux/seqlock.h>
#include <linux/u64_stats_sync.h>
L
Linus Torvalds 已提交
32

33
struct module;
34 35
struct scsi_ioctl_command;

L
Linus Torvalds 已提交
36 37
struct request_queue;
struct elevator_queue;
38
struct blk_trace;
39 40
struct request;
struct sg_io_hdr;
M
Mike Christie 已提交
41
struct bsg_job;
T
Tejun Heo 已提交
42
struct blkcg_gq;
43
struct blk_flush_queue;
44
struct pr_ops;
J
Jens Axboe 已提交
45
struct rq_wb;
46 47
struct blk_queue_stats;
struct blk_stat_callback;
L
Linus Torvalds 已提交
48 49 50 51

#define BLKDEV_MIN_RQ	4
#define BLKDEV_MAX_RQ	128	/* Default maximum */

52
/* Must be consistent with blk_mq_poll_stats_bkt() */
53 54
#define BLK_MQ_POLL_STATS_BKTS 16

T
Tejun Heo 已提交
55 56 57 58
/*
 * Maximum number of blkcg policies allowed to be registered concurrently.
 * Defined here to simplify include dependency.
 */
59
#define BLKCG_MAX_POLS		3
T
Tejun Heo 已提交
60

61
typedef void (rq_end_io_fn)(struct request *, blk_status_t);
L
Linus Torvalds 已提交
62

63 64 65
#define BLK_RL_SYNCFULL		(1U << 0)
#define BLK_RL_ASYNCFULL	(1U << 1)

L
Linus Torvalds 已提交
66
struct request_list {
67
	struct request_queue	*q;	/* the queue this rl belongs to */
68 69 70
#ifdef CONFIG_BLK_CGROUP
	struct blkcg_gq		*blkg;	/* blkg this request pool belongs to */
#endif
71 72 73 74
	/*
	 * count[], starved[], and wait[] are indexed by
	 * BLK_RW_SYNC/BLK_RW_ASYNC
	 */
75 76 77 78
	int			count[2];
	int			starved[2];
	mempool_t		*rq_pool;
	wait_queue_head_t	wait[2];
79
	unsigned int		flags;
L
Linus Torvalds 已提交
80 81
};

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
/*
 * request flags */
typedef __u32 __bitwise req_flags_t;

/* elevator knows about this request */
#define RQF_SORTED		((__force req_flags_t)(1 << 0))
/* drive already may have started this one */
#define RQF_STARTED		((__force req_flags_t)(1 << 1))
/* uses tagged queueing */
#define RQF_QUEUED		((__force req_flags_t)(1 << 2))
/* may not be passed by ioscheduler */
#define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
/* request for flush sequence */
#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << 4))
/* merge of different types, fail separately */
#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << 5))
/* track inflight for MQ */
#define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
/* don't call prep for this one */
#define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
/* set for "ide_preempt" requests and also for requests for which the SCSI
   "quiesce" state must be ignored. */
#define RQF_PREEMPT		((__force req_flags_t)(1 << 8))
/* contains copies of user pages */
#define RQF_COPY_USER		((__force req_flags_t)(1 << 9))
/* vaguely specified driver internal error.  Ignored by the block layer */
#define RQF_FAILED		((__force req_flags_t)(1 << 10))
/* don't warn about errors */
#define RQF_QUIET		((__force req_flags_t)(1 << 11))
/* elevator private data attached */
#define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
/* account I/O stat */
#define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
/* request came from our alloc pool */
#define RQF_ALLOCED		((__force req_flags_t)(1 << 14))
/* runtime pm request */
#define RQF_PM			((__force req_flags_t)(1 << 15))
/* on IO scheduler merge hash */
#define RQF_HASHED		((__force req_flags_t)(1 << 16))
121 122
/* IO stats tracking on */
#define RQF_STATS		((__force req_flags_t)(1 << 17))
123 124 125
/* Look at ->special_vec for the actual data payload instead of the
   bio chain. */
#define RQF_SPECIAL_PAYLOAD	((__force req_flags_t)(1 << 18))
126 127
/* The per-zone write lock is held for this request */
#define RQF_ZONE_WRITE_LOCKED	((__force req_flags_t)(1 << 19))
J
Jens Axboe 已提交
128
/* already slept for hybrid poll */
K
Keith Busch 已提交
129
#define RQF_MQ_POLL_SLEPT	((__force req_flags_t)(1 << 20))
130 131 132

/* flags that prevent us from merging requests: */
#define RQF_NOMERGE_FLAGS \
133
	(RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
134

K
Keith Busch 已提交
135 136 137 138 139 140 141 142 143
/*
 * Request state for blk-mq.
 */
enum mq_rq_state {
	MQ_RQ_IDLE		= 0,
	MQ_RQ_IN_FLIGHT		= 1,
	MQ_RQ_COMPLETE		= 2,
};

L
Linus Torvalds 已提交
144
/*
145 146 147 148
 * Try to put the fields that are referenced together in the same cacheline.
 *
 * If you modify this structure, make sure to update blk_rq_init() and
 * especially blk_mq_rq_ctx_init() to take care of the added fields.
L
Linus Torvalds 已提交
149 150
 */
struct request {
151
	struct request_queue *q;
152
	struct blk_mq_ctx *mq_ctx;
153

154
	int cpu;
155
	unsigned int cmd_flags;		/* op and common flags */
156
	req_flags_t rq_flags;
157 158 159

	int internal_tag;

160 161
	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
162
	int tag;
163
	sector_t __sector;		/* sector cursor */
L
Linus Torvalds 已提交
164 165 166 167

	struct bio *bio;
	struct bio *biotail;

168 169
	struct list_head queuelist;

170 171 172 173 174 175 176 177 178 179 180 181
	/*
	 * The hash is used inside the scheduler, and killed once the
	 * request reaches the dispatch list. The ipi_list is only used
	 * to queue the request for softirq completion, which is long
	 * after the request has been unhashed (and even removed from
	 * the dispatch list).
	 */
	union {
		struct hlist_node hash;	/* merge hash */
		struct list_head ipi_list;
	};

182 183 184
	/*
	 * The rb_node is only used inside the io scheduler, requests
	 * are pruned when moved to the dispatch queue. So let the
185
	 * completion_data share space with the rb_node.
186 187 188
	 */
	union {
		struct rb_node rb_node;	/* sort/lookup */
189
		struct bio_vec special_vec;
190
		void *completion_data;
191
		int error_count; /* for legacy drivers, don't use */
192
	};
193

194
	/*
195
	 * Three pointers are available for the IO schedulers, if they need
196 197
	 * more they have to dynamically allocate it.  Flush requests are
	 * never put on the IO scheduler. So let the flush fields share
198
	 * space with the elevator data.
199
	 */
200
	union {
201 202 203 204 205
		struct {
			struct io_cq		*icq;
			void			*priv[2];
		} elv;

206 207 208
		struct {
			unsigned int		seq;
			struct list_head	list;
209
			rq_end_io_fn		*saved_end_io;
210 211
		} flush;
	};
212

213
	struct gendisk *rq_disk;
214
	struct hd_struct *part;
215 216
	/* Time that I/O was submitted to the kernel. */
	u64 start_time_ns;
217 218 219 220 221 222 223 224 225 226 227 228
	/* Time that I/O was submitted to the device. */
	u64 io_start_time_ns;

#ifdef CONFIG_BLK_WBT
	unsigned short wbt_flags;
#endif
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
	unsigned short throtl_size;
#endif

	/*
	 * Number of scatter-gather DMA addr+len pairs after
L
Linus Torvalds 已提交
229 230 231
	 * physical address coalescing is performed.
	 */
	unsigned short nr_phys_segments;
232

233 234 235
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	unsigned short nr_integrity_segments;
#endif
L
Linus Torvalds 已提交
236

237
	unsigned short write_hint;
238 239
	unsigned short ioprio;

T
Tejun Heo 已提交
240
	void *special;		/* opaque pointer available for LLD use */
241

242
	unsigned int extra_len;	/* length of alignment and padding */
L
Linus Torvalds 已提交
243

K
Keith Busch 已提交
244 245
	enum mq_rq_state state;
	refcount_t ref;
246

247 248
	unsigned int timeout;

249 250
	/* access through blk_rq_set_deadline, blk_rq_deadline */
	unsigned long __deadline;
251

J
Jens Axboe 已提交
252
	struct list_head timeout_list;
L
Linus Torvalds 已提交
253

254
	union {
255
		struct __call_single_data csd;
256 257 258
		u64 fifo_time;
	};

L
Linus Torvalds 已提交
259
	/*
260
	 * completion callback.
L
Linus Torvalds 已提交
261 262 263
	 */
	rq_end_io_fn *end_io;
	void *end_io_data;
264 265 266

	/* for bidi */
	struct request *next_rq;
267 268 269 270

#ifdef CONFIG_BLK_CGROUP
	struct request_list *rl;		/* rl this rq is alloced from */
#endif
L
Linus Torvalds 已提交
271 272
};

273 274 275 276 277 278 279 280 281 282
static inline bool blk_op_is_scsi(unsigned int op)
{
	return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
}

static inline bool blk_op_is_private(unsigned int op)
{
	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
}

283 284
static inline bool blk_rq_is_scsi(struct request *rq)
{
285
	return blk_op_is_scsi(req_op(rq));
286 287 288 289
}

static inline bool blk_rq_is_private(struct request *rq)
{
290
	return blk_op_is_private(req_op(rq));
291 292
}

293 294
static inline bool blk_rq_is_passthrough(struct request *rq)
{
295
	return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
296 297
}

298 299 300 301 302 303 304
static inline bool bio_is_passthrough(struct bio *bio)
{
	unsigned op = bio_op(bio);

	return blk_op_is_scsi(op) || blk_op_is_private(op);
}

305 306 307 308 309
static inline unsigned short req_get_ioprio(struct request *req)
{
	return req->ioprio;
}

L
Linus Torvalds 已提交
310 311
#include <linux/elevator.h>

312 313
struct blk_queue_ctx;

314
typedef void (request_fn_proc) (struct request_queue *q);
315
typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
316
typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
317
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
318
typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
L
Linus Torvalds 已提交
319 320

struct bio_vec;
321
typedef void (softirq_done_fn)(struct request *);
322
typedef int (dma_drain_needed_fn)(struct request *);
323
typedef int (lld_busy_fn) (struct request_queue *q);
M
Mike Christie 已提交
324
typedef int (bsg_job_fn) (struct bsg_job *);
325 326
typedef int (init_rq_fn)(struct request_queue *, struct request *, gfp_t);
typedef void (exit_rq_fn)(struct request_queue *, struct request *);
L
Linus Torvalds 已提交
327

J
Jens Axboe 已提交
328
enum blk_eh_timer_return {
329 330
	BLK_EH_DONE,		/* drivers has completed the command */
	BLK_EH_RESET_TIMER,	/* reset timer and try again */
J
Jens Axboe 已提交
331 332 333 334
};

typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);

L
Linus Torvalds 已提交
335 336 337 338 339 340 341 342 343
enum blk_queue_state {
	Queue_down,
	Queue_up,
};

struct blk_queue_tag {
	struct request **tag_index;	/* map of busy tags */
	unsigned long *tag_map;		/* bit map of free/busy tags */
	int max_depth;			/* what we will send to device */
344
	int real_max_depth;		/* what the array can hold */
L
Linus Torvalds 已提交
345
	atomic_t refcnt;		/* map can be shared */
346 347
	int alloc_policy;		/* tag allocation policy */
	int next_tag;			/* next tag */
L
Linus Torvalds 已提交
348
};
349 350
#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
L
Linus Torvalds 已提交
351

352 353 354
#define BLK_SCSI_MAX_CMDS	(256)
#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))

D
Damien Le Moal 已提交
355 356 357 358 359 360 361 362 363
/*
 * Zoned block device models (zoned limit).
 */
enum blk_zoned_model {
	BLK_ZONED_NONE,	/* Regular block device */
	BLK_ZONED_HA,	/* Host-aware zoned block device */
	BLK_ZONED_HM,	/* Host-managed zoned block device */
};

364 365 366
struct queue_limits {
	unsigned long		bounce_pfn;
	unsigned long		seg_boundary_mask;
367
	unsigned long		virt_boundary_mask;
368 369

	unsigned int		max_hw_sectors;
370
	unsigned int		max_dev_sectors;
371
	unsigned int		chunk_sectors;
372 373
	unsigned int		max_sectors;
	unsigned int		max_segment_size;
374 375 376 377
	unsigned int		physical_block_size;
	unsigned int		alignment_offset;
	unsigned int		io_min;
	unsigned int		io_opt;
378
	unsigned int		max_discard_sectors;
379
	unsigned int		max_hw_discard_sectors;
380
	unsigned int		max_write_same_sectors;
381
	unsigned int		max_write_zeroes_sectors;
382 383
	unsigned int		discard_granularity;
	unsigned int		discard_alignment;
384 385

	unsigned short		logical_block_size;
386
	unsigned short		max_segments;
387
	unsigned short		max_integrity_segments;
388
	unsigned short		max_discard_segments;
389

390
	unsigned char		misaligned;
391
	unsigned char		discard_misaligned;
392
	unsigned char		cluster;
393
	unsigned char		raid_partial_stripes_expensive;
D
Damien Le Moal 已提交
394
	enum blk_zoned_model	zoned;
395 396
};

397 398 399 400 401 402 403 404 405 406 407 408 409
#ifdef CONFIG_BLK_DEV_ZONED

struct blk_zone_report_hdr {
	unsigned int	nr_zones;
	u8		padding[60];
};

extern int blkdev_report_zones(struct block_device *bdev,
			       sector_t sector, struct blk_zone *zones,
			       unsigned int *nr_zones, gfp_t gfp_mask);
extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
			      sector_t nr_sectors, gfp_t gfp_mask);

S
Shaun Tancheff 已提交
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
				     unsigned int cmd, unsigned long arg);
extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
				    unsigned int cmd, unsigned long arg);

#else /* CONFIG_BLK_DEV_ZONED */

static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
					    fmode_t mode, unsigned int cmd,
					    unsigned long arg)
{
	return -ENOTTY;
}

static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
					   fmode_t mode, unsigned int cmd,
					   unsigned long arg)
{
	return -ENOTTY;
}

431 432
#endif /* CONFIG_BLK_DEV_ZONED */

433
struct request_queue {
L
Linus Torvalds 已提交
434 435 436 437 438
	/*
	 * Together with queue_head for cacheline sharing
	 */
	struct list_head	queue_head;
	struct request		*last_merge;
J
Jens Axboe 已提交
439
	struct elevator_queue	*elevator;
440 441
	int			nr_rqs[2];	/* # allocated [a]sync rqs */
	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
L
Linus Torvalds 已提交
442

443 444
	atomic_t		shared_hctx_restart;

445
	struct blk_queue_stats	*stats;
J
Jens Axboe 已提交
446 447
	struct rq_wb		*rq_wb;

L
Linus Torvalds 已提交
448
	/*
449 450 451 452
	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
	 * is used, root blkg allocates from @q->root_rl and all other
	 * blkgs from their own blkg->rl.  Which one to use should be
	 * determined using bio_request_list().
L
Linus Torvalds 已提交
453
	 */
454
	struct request_list	root_rl;
L
Linus Torvalds 已提交
455 456 457

	request_fn_proc		*request_fn;
	make_request_fn		*make_request_fn;
458
	poll_q_fn		*poll_fn;
L
Linus Torvalds 已提交
459
	prep_rq_fn		*prep_rq_fn;
460
	unprep_rq_fn		*unprep_rq_fn;
461
	softirq_done_fn		*softirq_done_fn;
J
Jens Axboe 已提交
462
	rq_timed_out_fn		*rq_timed_out_fn;
463
	dma_drain_needed_fn	*dma_drain_needed;
464
	lld_busy_fn		*lld_busy_fn;
465
	/* Called just after a request is allocated */
466
	init_rq_fn		*init_rq_fn;
467
	/* Called just before a request is freed */
468
	exit_rq_fn		*exit_rq_fn;
469 470
	/* Called from inside blk_get_request() */
	void (*initialize_rq_fn)(struct request *rq);
L
Linus Torvalds 已提交
471

J
Jens Axboe 已提交
472
	const struct blk_mq_ops	*mq_ops;
473 474 475 476

	unsigned int		*mq_map;

	/* sw queues */
477
	struct blk_mq_ctx __percpu	*queue_ctx;
478 479
	unsigned int		nr_queues;

480 481
	unsigned int		queue_depth;

482 483 484 485
	/* hw dispatch queues */
	struct blk_mq_hw_ctx	**queue_hw_ctx;
	unsigned int		nr_hw_queues;

486 487 488
	/*
	 * Dispatch queue sorting
	 */
J
Jens Axboe 已提交
489
	sector_t		end_sector;
490 491
	struct request		*boundary_rq;

L
Linus Torvalds 已提交
492
	/*
493
	 * Delayed queue handling
L
Linus Torvalds 已提交
494
	 */
495
	struct delayed_work	delay_work;
L
Linus Torvalds 已提交
496

497
	struct backing_dev_info	*backing_dev_info;
L
Linus Torvalds 已提交
498 499 500 501 502 503 504 505

	/*
	 * The queue owner gets to use this for whatever they like.
	 * ll_rw_blk doesn't touch it.
	 */
	void			*queuedata;

	/*
506
	 * various queue flags, see QUEUE_* below
L
Linus Torvalds 已提交
507
	 */
508
	unsigned long		queue_flags;
L
Linus Torvalds 已提交
509

510 511 512 513 514 515
	/*
	 * ida allocated id for this queue.  Used to index queues from
	 * ioctx.
	 */
	int			id;

L
Linus Torvalds 已提交
516
	/*
517
	 * queue needs bounce pages for pages above this limit
L
Linus Torvalds 已提交
518
	 */
519
	gfp_t			bounce_gfp;
L
Linus Torvalds 已提交
520 521

	/*
已提交
522 523 524
	 * protects queue structures from reentrancy. ->__queue_lock should
	 * _never_ be used directly, it is queue private. always use
	 * ->queue_lock.
L
Linus Torvalds 已提交
525
	 */
已提交
526
	spinlock_t		__queue_lock;
L
Linus Torvalds 已提交
527 528 529 530 531 532 533
	spinlock_t		*queue_lock;

	/*
	 * queue kobject
	 */
	struct kobject kobj;

534 535 536 537 538
	/*
	 * mq queue kobject
	 */
	struct kobject mq_kobj;

539 540 541 542
#ifdef  CONFIG_BLK_DEV_INTEGRITY
	struct blk_integrity integrity;
#endif	/* CONFIG_BLK_DEV_INTEGRITY */

543
#ifdef CONFIG_PM
L
Lin Ming 已提交
544 545 546 547 548
	struct device		*dev;
	int			rpm_status;
	unsigned int		nr_pending;
#endif

L
Linus Torvalds 已提交
549 550 551 552 553 554 555 556
	/*
	 * queue settings
	 */
	unsigned long		nr_requests;	/* Max # of requests */
	unsigned int		nr_congestion_on;
	unsigned int		nr_congestion_off;
	unsigned int		nr_batching;

J
James Bottomley 已提交
557
	unsigned int		dma_drain_size;
558
	void			*dma_drain_buffer;
559
	unsigned int		dma_pad_mask;
L
Linus Torvalds 已提交
560 561 562
	unsigned int		dma_alignment;

	struct blk_queue_tag	*queue_tags;
563
	struct list_head	tag_busy_list;
L
Linus Torvalds 已提交
564

565
	unsigned int		nr_sorted;
566
	unsigned int		in_flight[2];
567

568 569 570 571 572 573
	/*
	 * Number of active block driver functions for which blk_drain_queue()
	 * must wait. Must be incremented around functions that unlock the
	 * queue_lock internally, e.g. scsi_request_fn().
	 */
	unsigned int		request_fn_active;
L
Linus Torvalds 已提交
574

J
Jens Axboe 已提交
575
	unsigned int		rq_timeout;
576
	int			poll_nsec;
577 578

	struct blk_stat_callback	*poll_cb;
579
	struct blk_rq_stat	poll_stat[BLK_MQ_POLL_STATS_BKTS];
580

J
Jens Axboe 已提交
581
	struct timer_list	timeout;
582
	struct work_struct	timeout_work;
J
Jens Axboe 已提交
583 584
	struct list_head	timeout_list;

585
	struct list_head	icq_list;
586
#ifdef CONFIG_BLK_CGROUP
587
	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
T
Tejun Heo 已提交
588
	struct blkcg_gq		*root_blkg;
589
	struct list_head	blkg_list;
590
#endif
591

592 593
	struct queue_limits	limits;

594 595 596 597 598 599 600 601 602 603 604
	/*
	 * Zoned block device information for request dispatch control.
	 * nr_zones is the total number of zones of the device. This is always
	 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
	 * bits which indicates if a zone is conventional (bit clear) or
	 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
	 * bits which indicates if a zone is write locked, that is, if a write
	 * request targeting the zone was dispatched. All three fields are
	 * initialized by the low level device driver (e.g. scsi/sd.c).
	 * Stacking drivers (device mappers) may or may not initialize
	 * these fields.
605 606 607 608 609
	 *
	 * Reads of this information must be protected with blk_queue_enter() /
	 * blk_queue_exit(). Modifying this information is only allowed while
	 * no requests are being processed. See also blk_mq_freeze_queue() and
	 * blk_mq_unfreeze_queue().
610 611 612 613 614
	 */
	unsigned int		nr_zones;
	unsigned long		*seq_zones_bitmap;
	unsigned long		*seq_zones_wlock;

L
Linus Torvalds 已提交
615 616 617 618 619
	/*
	 * sg stuff
	 */
	unsigned int		sg_timeout;
	unsigned int		sg_reserved_size;
620
	int			node;
621
#ifdef CONFIG_BLK_DEV_IO_TRACE
622
	struct blk_trace	*blk_trace;
623
	struct mutex		blk_trace_mutex;
624
#endif
L
Linus Torvalds 已提交
625
	/*
626
	 * for flush operations
L
Linus Torvalds 已提交
627
	 */
628
	struct blk_flush_queue	*fq;
629

630 631
	struct list_head	requeue_list;
	spinlock_t		requeue_lock;
632
	struct delayed_work	requeue_work;
633

634
	struct mutex		sysfs_lock;
635

636
	int			bypass_depth;
637
	atomic_t		mq_freeze_depth;
638

639
#if defined(CONFIG_BLK_DEV_BSG)
M
Mike Christie 已提交
640
	bsg_job_fn		*bsg_job_fn;
641 642
	struct bsg_class_device bsg_dev;
#endif
643 644 645 646 647

#ifdef CONFIG_BLK_DEV_THROTTLING
	/* Throttle data */
	struct throtl_data *td;
#endif
T
Tejun Heo 已提交
648
	struct rcu_head		rcu_head;
649
	wait_queue_head_t	mq_freeze_wq;
650
	struct percpu_ref	q_usage_counter;
651
	struct list_head	all_q_node;
652 653 654

	struct blk_mq_tag_set	*tag_set;
	struct list_head	tag_set_list;
655
	struct bio_set		*bio_split;
656

657
#ifdef CONFIG_BLK_DEBUG_FS
658
	struct dentry		*debugfs_dir;
659
	struct dentry		*sched_debugfs_dir;
660 661
#endif

662
	bool			mq_sysfs_init_done;
663 664 665

	size_t			cmd_size;
	void			*rq_alloc_data;
666 667

	struct work_struct	release_work;
668 669 670

#define BLK_MAX_WRITE_HINTS	5
	u64			write_hints[BLK_MAX_WRITE_HINTS];
L
Linus Torvalds 已提交
671 672
};

673 674 675 676 677 678 679 680 681
#define QUEUE_FLAG_QUEUED	0	/* uses generic tag queueing */
#define QUEUE_FLAG_STOPPED	1	/* queue is stopped */
#define QUEUE_FLAG_DYING	2	/* queue being torn down */
#define QUEUE_FLAG_BYPASS	3	/* act as dumb FIFO queue */
#define QUEUE_FLAG_BIDI		4	/* queue supports bidi requests */
#define QUEUE_FLAG_NOMERGES     5	/* disable merge attempts */
#define QUEUE_FLAG_SAME_COMP	6	/* complete on same CPU-group */
#define QUEUE_FLAG_FAIL_IO	7	/* fake timeout */
#define QUEUE_FLAG_NONROT	9	/* non-rotational device (SSD) */
682
#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
#define QUEUE_FLAG_IO_STAT     10	/* do IO stats */
#define QUEUE_FLAG_DISCARD     11	/* supports DISCARD */
#define QUEUE_FLAG_NOXMERGES   12	/* No extended merges */
#define QUEUE_FLAG_ADD_RANDOM  13	/* Contributes to random pool */
#define QUEUE_FLAG_SECERASE    14	/* supports secure erase */
#define QUEUE_FLAG_SAME_FORCE  15	/* force complete on same CPU */
#define QUEUE_FLAG_DEAD        16	/* queue tear-down finished */
#define QUEUE_FLAG_INIT_DONE   17	/* queue is initialized */
#define QUEUE_FLAG_NO_SG_MERGE 18	/* don't attempt to merge SG segments*/
#define QUEUE_FLAG_POLL	       19	/* IO polling enabled if set */
#define QUEUE_FLAG_WC	       20	/* Write back caching */
#define QUEUE_FLAG_FUA	       21	/* device supports FUA writes */
#define QUEUE_FLAG_FLUSH_NQ    22	/* flush not queueuable */
#define QUEUE_FLAG_DAX         23	/* device supports DAX */
#define QUEUE_FLAG_STATS       24	/* track rq completion times */
#define QUEUE_FLAG_POLL_STATS  25	/* collecting stats for hybrid polling */
#define QUEUE_FLAG_REGISTERED  26	/* queue has been registered to a disk */
#define QUEUE_FLAG_SCSI_PASSTHROUGH 27	/* queue supports SCSI commands */
#define QUEUE_FLAG_QUIESCED    28	/* queue has been quiesced */
702
#define QUEUE_FLAG_PREEMPT_ONLY	29	/* only process REQ_PREEMPT requests */
703 704

#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
705 706
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_ADD_RANDOM))
707

708
#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
709 710
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_POLL))
711

712 713 714 715 716
void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);

L
Linus Torvalds 已提交
717 718
#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
B
Bart Van Assche 已提交
719
#define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
720
#define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
721
#define blk_queue_bypass(q)	test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
722
#define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
723
#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
724 725
#define blk_queue_noxmerges(q)	\
	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
726
#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
727
#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
728
#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
729
#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
730 731
#define blk_queue_secure_erase(q) \
	(test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
732
#define blk_queue_dax(q)	test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
733 734
#define blk_queue_scsi_passthrough(q)	\
	test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
L
Linus Torvalds 已提交
735

736 737 738
#define blk_noretry_request(rq) \
	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
			     REQ_FAILFAST_DRIVER))
739
#define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
740 741
#define blk_queue_preempt_only(q)				\
	test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)
742
#define blk_queue_fua(q)	test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
743 744 745

extern int blk_set_preempt_only(struct request_queue *q);
extern void blk_clear_preempt_only(struct request_queue *q);
746

747 748 749 750 751
static inline int queue_in_flight(struct request_queue *q)
{
	return q->in_flight[0] + q->in_flight[1];
}

752 753 754 755
static inline bool blk_account_rq(struct request *rq)
{
	return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
}
756

757
#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
758
#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
759 760
/* rq->queuelist of dequeued request must be list_empty() */
#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
L
Linus Torvalds 已提交
761 762 763

#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)

764
#define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)
L
Linus Torvalds 已提交
765

766 767 768 769 770 771 772 773 774
/*
 * Driver can handle struct request, if it either has an old style
 * request_fn defined, or is blk-mq based.
 */
static inline bool queue_is_rq_based(struct request_queue *q)
{
	return q->request_fn || q->mq_ops;
}

775 776 777 778 779
static inline unsigned int blk_queue_cluster(struct request_queue *q)
{
	return q->limits.cluster;
}

D
Damien Le Moal 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
static inline enum blk_zoned_model
blk_queue_zoned_model(struct request_queue *q)
{
	return q->limits.zoned;
}

static inline bool blk_queue_is_zoned(struct request_queue *q)
{
	switch (blk_queue_zoned_model(q)) {
	case BLK_ZONED_HA:
	case BLK_ZONED_HM:
		return true;
	default:
		return false;
	}
}

797
static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
798 799 800 801
{
	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
}

802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
{
	return q->nr_zones;
}

static inline unsigned int blk_queue_zone_no(struct request_queue *q,
					     sector_t sector)
{
	if (!blk_queue_is_zoned(q))
		return 0;
	return sector >> ilog2(q->limits.chunk_sectors);
}

static inline bool blk_queue_zone_is_seq(struct request_queue *q,
					 sector_t sector)
{
	if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
		return false;
	return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
}

823 824
static inline bool rq_is_sync(struct request *rq)
{
825
	return op_is_sync(rq->cmd_flags);
826 827
}

828
static inline bool blk_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
829
{
830 831 832
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	return rl->flags & flag;
L
Linus Torvalds 已提交
833 834
}

835
static inline void blk_set_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
836
{
837 838 839
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags |= flag;
L
Linus Torvalds 已提交
840 841
}

842
static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
843
{
844 845 846
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags &= ~flag;
L
Linus Torvalds 已提交
847 848
}

849 850
static inline bool rq_mergeable(struct request *rq)
{
851
	if (blk_rq_is_passthrough(rq))
852
		return false;
L
Linus Torvalds 已提交
853

854 855 856
	if (req_op(rq) == REQ_OP_FLUSH)
		return false;

857 858 859
	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
		return false;

860
	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
861 862
		return false;
	if (rq->rq_flags & RQF_NOMERGE_FLAGS)
863 864 865 866
		return false;

	return true;
}
L
Linus Torvalds 已提交
867

868 869
static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
{
870 871
	if (bio_page(a) == bio_page(b) &&
	    bio_offset(a) == bio_offset(b))
872 873 874 875 876
		return true;

	return false;
}

877 878 879 880 881 882 883 884
static inline unsigned int blk_queue_depth(struct request_queue *q)
{
	if (q->queue_depth)
		return q->queue_depth;

	return q->nr_requests;
}

L
Linus Torvalds 已提交
885 886 887
/*
 * q->prep_rq_fn return values
 */
888 889 890 891 892 893
enum {
	BLKPREP_OK,		/* serve it */
	BLKPREP_KILL,		/* fatal error, kill, return -EIO */
	BLKPREP_DEFER,		/* leave on queue */
	BLKPREP_INVALID,	/* invalid command, kill, return -EREMOTEIO */
};
L
Linus Torvalds 已提交
894 895 896 897 898 899 900 901 902 903

extern unsigned long blk_max_low_pfn, blk_max_pfn;

/*
 * standard bounce addresses:
 *
 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 * BLK_BOUNCE_ANY	: don't bounce anything
 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 */
904 905

#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
906
#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
907 908 909 910
#else
#define BLK_BOUNCE_HIGH		-1ULL
#endif
#define BLK_BOUNCE_ANY		(-1ULL)
911
#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
L
Linus Torvalds 已提交
912

913 914 915 916
/*
 * default timeout for SG_IO if none specified
 */
#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
L
Linus Torvalds 已提交
917
#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
918

919 920 921 922
struct rq_map_data {
	struct page **pages;
	int page_order;
	int nr_entries;
923
	unsigned long offset;
924
	int null_mapped;
925
	int from_user;
926 927
};

928
struct req_iterator {
929
	struct bvec_iter iter;
930 931 932 933
	struct bio *bio;
};

/* This should not be used directly - use rq_for_each_segment */
934 935
#define for_each_bio(_bio)		\
	for (; _bio; _bio = _bio->bi_next)
936
#define __rq_for_each_bio(_bio, rq)	\
L
Linus Torvalds 已提交
937 938 939
	if ((rq->bio))			\
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

940 941
#define rq_for_each_segment(bvl, _rq, _iter)			\
	__rq_for_each_bio(_iter.bio, _rq)			\
942
		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
943

K
Kent Overstreet 已提交
944
#define rq_iter_last(bvec, _iter)				\
945
		(_iter.bio->bi_next == NULL &&			\
K
Kent Overstreet 已提交
946
		 bio_iter_last(bvec, _iter.iter))
947

948 949 950 951 952 953 954 955 956 957 958
#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
#endif
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
extern void rq_flush_dcache_pages(struct request *rq);
#else
static inline void rq_flush_dcache_pages(struct request *rq)
{
}
#endif

L
Linus Torvalds 已提交
959 960
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
961
extern blk_qc_t generic_make_request(struct bio *bio);
962
extern blk_qc_t direct_make_request(struct bio *bio);
963
extern void blk_rq_init(struct request_queue *q, struct request *rq);
964
extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
L
Linus Torvalds 已提交
965
extern void blk_put_request(struct request *);
966
extern void __blk_put_request(struct request_queue *, struct request *);
967
extern struct request *blk_get_request(struct request_queue *, unsigned int op,
968
				       blk_mq_req_flags_t flags);
969
extern void blk_requeue_request(struct request_queue *, struct request *);
970
extern int blk_lld_busy(struct request_queue *q);
971 972 973 974 975
extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
			     struct bio_set *bs, gfp_t gfp_mask,
			     int (*bio_ctr)(struct bio *, struct bio *, void *),
			     void *data);
extern void blk_rq_unprep_clone(struct request *rq);
976
extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
977
				     struct request *rq);
J
Jens Axboe 已提交
978
extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
979
extern void blk_delay_queue(struct request_queue *, unsigned long);
980
extern void blk_queue_split(struct request_queue *, struct bio **);
981
extern void blk_recount_segments(struct request_queue *, struct bio *);
982
extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
983 984
extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
			      unsigned int, void __user *);
985 986
extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			  unsigned int, void __user *);
987 988
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			 struct scsi_ioctl_command __user *);
989

990
extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
991
extern void blk_queue_exit(struct request_queue *q);
992
extern void blk_start_queue(struct request_queue *q);
J
Jens Axboe 已提交
993
extern void blk_start_queue_async(struct request_queue *q);
994
extern void blk_stop_queue(struct request_queue *q);
L
Linus Torvalds 已提交
995
extern void blk_sync_queue(struct request_queue *q);
996
extern void __blk_stop_queue(struct request_queue *q);
997
extern void __blk_run_queue(struct request_queue *q);
998
extern void __blk_run_queue_uncond(struct request_queue *q);
999
extern void blk_run_queue(struct request_queue *);
1000
extern void blk_run_queue_async(struct request_queue *q);
1001
extern int blk_rq_map_user(struct request_queue *, struct request *,
1002 1003
			   struct rq_map_data *, void __user *, unsigned long,
			   gfp_t);
J
Jens Axboe 已提交
1004
extern int blk_rq_unmap_user(struct bio *);
1005 1006
extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
1007 1008
			       struct rq_map_data *, const struct iov_iter *,
			       gfp_t);
1009
extern void blk_execute_rq(struct request_queue *, struct gendisk *,
1010
			  struct request *, int);
1011
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
1012
				  struct request *, int, rq_end_io_fn *);
1013

1014 1015 1016
int blk_status_to_errno(blk_status_t status);
blk_status_t errno_to_blk_status(int errno);

1017
bool blk_poll(struct request_queue *q, blk_qc_t cookie);
J
Jens Axboe 已提交
1018

1019
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
L
Linus Torvalds 已提交
1020
{
1021
	return bdev->bd_disk->queue;	/* this is never NULL */
L
Linus Torvalds 已提交
1022 1023
}

1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
/*
 * The basic unit of block I/O is a sector. It is used in a number of contexts
 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
 * bytes. Variables of type sector_t represent an offset or size that is a
 * multiple of 512 bytes. Hence these two constants.
 */
#ifndef SECTOR_SHIFT
#define SECTOR_SHIFT 9
#endif
#ifndef SECTOR_SIZE
#define SECTOR_SIZE (1 << SECTOR_SHIFT)
#endif

1037
/*
1038 1039 1040 1041 1042 1043
 * blk_rq_pos()			: the current sector
 * blk_rq_bytes()		: bytes left in the entire request
 * blk_rq_cur_bytes()		: bytes left in the current segment
 * blk_rq_err_bytes()		: bytes left till the next error boundary
 * blk_rq_sectors()		: sectors left in the entire request
 * blk_rq_cur_sectors()		: sectors left in the current segment
1044
 */
1045 1046
static inline sector_t blk_rq_pos(const struct request *rq)
{
1047
	return rq->__sector;
1048 1049 1050 1051
}

static inline unsigned int blk_rq_bytes(const struct request *rq)
{
1052
	return rq->__data_len;
1053 1054
}

1055 1056 1057 1058
static inline int blk_rq_cur_bytes(const struct request *rq)
{
	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
}
1059

1060 1061
extern unsigned int blk_rq_err_bytes(const struct request *rq);

1062 1063
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
1064
	return blk_rq_bytes(rq) >> SECTOR_SHIFT;
1065 1066 1067 1068
}

static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
1069
	return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
1070 1071
}

1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
static inline unsigned int blk_rq_zone_no(struct request *rq)
{
	return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
}

static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
{
	return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
}

1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
/*
 * Some commands like WRITE SAME have a payload or data transfer size which
 * is different from the size of the request.  Any driver that supports such
 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
 * calculate the data transfer size.
 */
static inline unsigned int blk_rq_payload_bytes(struct request *rq)
{
	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
		return rq->special_vec.bv_len;
	return blk_rq_bytes(rq);
}

1095
static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1096
						     int op)
1097
{
A
Adrian Hunter 已提交
1098
	if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
1099 1100
		return min(q->limits.max_discard_sectors,
			   UINT_MAX >> SECTOR_SHIFT);
1101

1102
	if (unlikely(op == REQ_OP_WRITE_SAME))
1103 1104
		return q->limits.max_write_same_sectors;

1105 1106 1107
	if (unlikely(op == REQ_OP_WRITE_ZEROES))
		return q->limits.max_write_zeroes_sectors;

1108 1109 1110
	return q->limits.max_sectors;
}

1111 1112 1113 1114 1115 1116 1117 1118
/*
 * Return maximum size of a request at given offset. Only valid for
 * file system requests.
 */
static inline unsigned int blk_max_size_offset(struct request_queue *q,
					       sector_t offset)
{
	if (!q->limits.chunk_sectors)
1119
		return q->limits.max_sectors;
1120 1121 1122 1123 1124

	return q->limits.chunk_sectors -
			(offset & (q->limits.chunk_sectors - 1));
}

D
Damien Le Moal 已提交
1125 1126
static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
						  sector_t offset)
1127 1128 1129
{
	struct request_queue *q = rq->q;

1130
	if (blk_rq_is_passthrough(rq))
1131 1132
		return q->limits.max_hw_sectors;

A
Adrian Hunter 已提交
1133 1134 1135
	if (!q->limits.chunk_sectors ||
	    req_op(rq) == REQ_OP_DISCARD ||
	    req_op(rq) == REQ_OP_SECURE_ERASE)
1136
		return blk_queue_get_max_sectors(q, req_op(rq));
1137

D
Damien Le Moal 已提交
1138
	return min(blk_max_size_offset(q, offset),
1139
			blk_queue_get_max_sectors(q, req_op(rq)));
1140 1141
}

1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
static inline unsigned int blk_rq_count_bios(struct request *rq)
{
	unsigned int nr_bios = 0;
	struct bio *bio;

	__rq_for_each_bio(bio, rq)
		nr_bios++;

	return nr_bios;
}

1153 1154 1155 1156 1157 1158 1159
/*
 * Request issue related functions.
 */
extern struct request *blk_peek_request(struct request_queue *q);
extern void blk_start_request(struct request *rq);
extern struct request *blk_fetch_request(struct request_queue *q);

1160 1161
void blk_steal_bios(struct bio_list *list, struct request *rq);

L
Linus Torvalds 已提交
1162
/*
1163 1164 1165 1166 1167
 * Request completion related functions.
 *
 * blk_update_request() completes given number of bytes and updates
 * the request without completing it.
 *
1168 1169
 * blk_end_request() and friends.  __blk_end_request() must be called
 * with the request queue spinlock acquired.
L
Linus Torvalds 已提交
1170 1171
 *
 * Several drivers define their own end_request and call
1172 1173
 * blk_end_request() for parts of the original function.
 * This prevents code duplication in drivers.
L
Linus Torvalds 已提交
1174
 */
1175
extern bool blk_update_request(struct request *rq, blk_status_t error,
1176
			       unsigned int nr_bytes);
1177 1178
extern void blk_finish_request(struct request *rq, blk_status_t error);
extern bool blk_end_request(struct request *rq, blk_status_t error,
1179
			    unsigned int nr_bytes);
1180 1181
extern void blk_end_request_all(struct request *rq, blk_status_t error);
extern bool __blk_end_request(struct request *rq, blk_status_t error,
1182
			      unsigned int nr_bytes);
1183 1184
extern void __blk_end_request_all(struct request *rq, blk_status_t error);
extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1185

1186
extern void blk_complete_request(struct request *);
J
Jens Axboe 已提交
1187 1188
extern void __blk_complete_request(struct request *);
extern void blk_abort_request(struct request *);
1189
extern void blk_unprep_request(struct request *);
1190

L
Linus Torvalds 已提交
1191 1192 1193
/*
 * Access functions for manipulating queue properties
 */
1194
extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1195
					spinlock_t *lock, int node_id);
1196
extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
1197
extern int blk_init_allocated_queue(struct request_queue *);
1198 1199 1200
extern void blk_cleanup_queue(struct request_queue *);
extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
extern void blk_queue_bounce_limit(struct request_queue *, u64);
1201
extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1202
extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1203
extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1204 1205
extern void blk_queue_max_discard_segments(struct request_queue *,
		unsigned short);
1206
extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1207 1208
extern void blk_queue_max_discard_sectors(struct request_queue *q,
		unsigned int max_discard_sectors);
1209 1210
extern void blk_queue_max_write_same_sectors(struct request_queue *q,
		unsigned int max_write_same_sectors);
1211 1212
extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
		unsigned int max_write_same_sectors);
1213
extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
1214
extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1215 1216
extern void blk_queue_alignment_offset(struct request_queue *q,
				       unsigned int alignment);
1217
extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1218
extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1219
extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1220
extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1221
extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1222
extern void blk_set_default_limits(struct queue_limits *lim);
1223
extern void blk_set_stacking_limits(struct queue_limits *lim);
1224 1225
extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
			    sector_t offset);
1226 1227
extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
			    sector_t offset);
1228 1229
extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
			      sector_t offset);
1230
extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
1231
extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
1232
extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1233 1234 1235
extern int blk_queue_dma_drain(struct request_queue *q,
			       dma_drain_needed_fn *dma_drain_needed,
			       void *buf, unsigned int size);
1236
extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
1237
extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1238
extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1239
extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1240
extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1241
extern void blk_queue_dma_alignment(struct request_queue *, int);
1242
extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1243
extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
J
Jens Axboe 已提交
1244 1245
extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1246
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1247
extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
L
Linus Torvalds 已提交
1248

1249 1250 1251 1252 1253 1254 1255 1256 1257
/*
 * Number of physical segments as sent to the device.
 *
 * Normally this is the number of discontiguous data segments sent by the
 * submitter.  But for data-less command like discard we might have no
 * actual data segments submitted, but the driver might have to add it's
 * own special payload.  In that case we still return 1 here so that this
 * special payload will be mapped.
 */
1258 1259 1260 1261 1262 1263 1264
static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
{
	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
		return 1;
	return rq->nr_phys_segments;
}

1265 1266 1267 1268 1269 1270 1271 1272 1273
/*
 * Number of discard segments (or ranges) the driver needs to fill in.
 * Each discard bio merged into a request is counted as one segment.
 */
static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
{
	return max_t(unsigned short, rq->nr_phys_segments, 1);
}

1274
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
L
Linus Torvalds 已提交
1275 1276 1277
extern void blk_dump_rq_flags(struct request *, char *);
extern long nr_blockdev_pages(void);

T
Tejun Heo 已提交
1278
bool __must_check blk_get_queue(struct request_queue *);
1279
struct request_queue *blk_alloc_queue(gfp_t);
1280 1281
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
					   spinlock_t *lock);
1282
extern void blk_put_queue(struct request_queue *);
1283
extern void blk_set_queue_dying(struct request_queue *);
L
Linus Torvalds 已提交
1284

L
Lin Ming 已提交
1285 1286 1287
/*
 * block layer runtime pm functions
 */
1288
#ifdef CONFIG_PM
L
Lin Ming 已提交
1289 1290 1291 1292 1293
extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
extern int blk_pre_runtime_suspend(struct request_queue *q);
extern void blk_post_runtime_suspend(struct request_queue *q, int err);
extern void blk_pre_runtime_resume(struct request_queue *q);
extern void blk_post_runtime_resume(struct request_queue *q, int err);
1294
extern void blk_set_runtime_active(struct request_queue *q);
L
Lin Ming 已提交
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
#else
static inline void blk_pm_runtime_init(struct request_queue *q,
	struct device *dev) {}
static inline int blk_pre_runtime_suspend(struct request_queue *q)
{
	return -ENOSYS;
}
static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
static inline void blk_pre_runtime_resume(struct request_queue *q) {}
static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1305
static inline void blk_set_runtime_active(struct request_queue *q) {}
L
Lin Ming 已提交
1306 1307
#endif

1308
/*
S
Suresh Jayaraman 已提交
1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
 * blk_plug permits building a queue of related requests by holding the I/O
 * fragments for a short period. This allows merging of sequential requests
 * into single larger request. As the requests are moved from a per-task list to
 * the device's request_queue in a batch, this results in improved scalability
 * as the lock contention for request_queue lock is reduced.
 *
 * It is ok not to disable preemption when adding the request to the plug list
 * or when attempting a merge, because blk_schedule_flush_list() will only flush
 * the plug list when the task sleeps by itself. For details, please see
 * schedule() where blk_schedule_flush_plug() is called.
1319
 */
1320
struct blk_plug {
S
Suresh Jayaraman 已提交
1321
	struct list_head list; /* requests */
1322
	struct list_head mq_list; /* blk-mq requests */
S
Suresh Jayaraman 已提交
1323
	struct list_head cb_list; /* md requires an unplug callback */
1324
};
1325
#define BLK_MAX_REQUEST_COUNT 16
1326
#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1327

1328
struct blk_plug_cb;
1329
typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1330 1331
struct blk_plug_cb {
	struct list_head list;
1332 1333
	blk_plug_cb_fn callback;
	void *data;
1334
};
1335 1336
extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
					     void *data, int size);
1337 1338
extern void blk_start_plug(struct blk_plug *);
extern void blk_finish_plug(struct blk_plug *);
1339
extern void blk_flush_plug_list(struct blk_plug *, bool);
1340 1341 1342 1343 1344

static inline void blk_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1345 1346 1347 1348 1349 1350 1351 1352
	if (plug)
		blk_flush_plug_list(plug, false);
}

static inline void blk_schedule_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1353
	if (plug)
1354
		blk_flush_plug_list(plug, true);
1355 1356 1357 1358 1359 1360
}

static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1361 1362 1363 1364
	return plug &&
		(!list_empty(&plug->list) ||
		 !list_empty(&plug->mq_list) ||
		 !list_empty(&plug->cb_list));
1365 1366
}

L
Linus Torvalds 已提交
1367 1368 1369
/*
 * tag stuff
 */
1370 1371 1372
extern int blk_queue_start_tag(struct request_queue *, struct request *);
extern struct request *blk_queue_find_tag(struct request_queue *, int);
extern void blk_queue_end_tag(struct request_queue *, struct request *);
1373
extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
1374 1375 1376
extern void blk_queue_free_tags(struct request_queue *);
extern int blk_queue_resize_tags(struct request_queue *, int);
extern void blk_queue_invalidate_tags(struct request_queue *);
1377
extern struct blk_queue_tag *blk_init_tags(int, int);
1378
extern void blk_free_tags(struct blk_queue_tag *);
L
Linus Torvalds 已提交
1379

1380 1381 1382 1383 1384 1385 1386
static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
						int tag)
{
	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
		return NULL;
	return bqt->tag_index[tag];
}
1387

1388 1389 1390
extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1391 1392

#define BLKDEV_DISCARD_SECURE	(1 << 0)	/* issue a secure erase */
1393

1394 1395
extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1396
extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1397
		sector_t nr_sects, gfp_t gfp_mask, int flags,
1398
		struct bio **biop);
1399 1400

#define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
1401
#define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
1402

1403 1404
extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1405
		unsigned flags);
1406
extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1407 1408
		sector_t nr_sects, gfp_t gfp_mask, unsigned flags);

1409 1410
static inline int sb_issue_discard(struct super_block *sb, sector_t block,
		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
D
David Woodhouse 已提交
1411
{
1412 1413 1414 1415 1416
	return blkdev_issue_discard(sb->s_bdev,
				    block << (sb->s_blocksize_bits -
					      SECTOR_SHIFT),
				    nr_blocks << (sb->s_blocksize_bits -
						  SECTOR_SHIFT),
1417
				    gfp_mask, flags);
D
David Woodhouse 已提交
1418
}
1419
static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1420
		sector_t nr_blocks, gfp_t gfp_mask)
1421 1422
{
	return blkdev_issue_zeroout(sb->s_bdev,
1423 1424 1425 1426
				    block << (sb->s_blocksize_bits -
					      SECTOR_SHIFT),
				    nr_blocks << (sb->s_blocksize_bits -
						  SECTOR_SHIFT),
1427
				    gfp_mask, 0);
1428
}
L
Linus Torvalds 已提交
1429

1430
extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1431

1432 1433 1434
enum blk_default_limits {
	BLK_MAX_SEGMENTS	= 128,
	BLK_SAFE_MAX_SECTORS	= 255,
1435
	BLK_DEF_MAX_SECTORS	= 2560,
1436 1437 1438
	BLK_MAX_SEGMENT_SIZE	= 65536,
	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
};
1439

L
Linus Torvalds 已提交
1440 1441
#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)

1442 1443
static inline unsigned long queue_segment_boundary(struct request_queue *q)
{
1444
	return q->limits.seg_boundary_mask;
1445 1446
}

1447 1448 1449 1450 1451
static inline unsigned long queue_virt_boundary(struct request_queue *q)
{
	return q->limits.virt_boundary_mask;
}

1452 1453
static inline unsigned int queue_max_sectors(struct request_queue *q)
{
1454
	return q->limits.max_sectors;
1455 1456 1457 1458
}

static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
{
1459
	return q->limits.max_hw_sectors;
1460 1461
}

1462
static inline unsigned short queue_max_segments(struct request_queue *q)
1463
{
1464
	return q->limits.max_segments;
1465 1466
}

1467 1468 1469 1470 1471
static inline unsigned short queue_max_discard_segments(struct request_queue *q)
{
	return q->limits.max_discard_segments;
}

1472 1473
static inline unsigned int queue_max_segment_size(struct request_queue *q)
{
1474
	return q->limits.max_segment_size;
1475 1476
}

1477
static inline unsigned short queue_logical_block_size(struct request_queue *q)
L
Linus Torvalds 已提交
1478 1479 1480
{
	int retval = 512;

1481 1482
	if (q && q->limits.logical_block_size)
		retval = q->limits.logical_block_size;
L
Linus Torvalds 已提交
1483 1484 1485 1486

	return retval;
}

1487
static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1488
{
1489
	return queue_logical_block_size(bdev_get_queue(bdev));
L
Linus Torvalds 已提交
1490 1491
}

1492 1493 1494 1495 1496
static inline unsigned int queue_physical_block_size(struct request_queue *q)
{
	return q->limits.physical_block_size;
}

1497
static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
M
Martin K. Petersen 已提交
1498 1499 1500 1501
{
	return queue_physical_block_size(bdev_get_queue(bdev));
}

1502 1503 1504 1505 1506
static inline unsigned int queue_io_min(struct request_queue *q)
{
	return q->limits.io_min;
}

M
Martin K. Petersen 已提交
1507 1508 1509 1510 1511
static inline int bdev_io_min(struct block_device *bdev)
{
	return queue_io_min(bdev_get_queue(bdev));
}

1512 1513 1514 1515 1516
static inline unsigned int queue_io_opt(struct request_queue *q)
{
	return q->limits.io_opt;
}

M
Martin K. Petersen 已提交
1517 1518 1519 1520 1521
static inline int bdev_io_opt(struct block_device *bdev)
{
	return queue_io_opt(bdev_get_queue(bdev));
}

1522 1523
static inline int queue_alignment_offset(struct request_queue *q)
{
M
Martin K. Petersen 已提交
1524
	if (q->limits.misaligned)
1525 1526
		return -1;

M
Martin K. Petersen 已提交
1527
	return q->limits.alignment_offset;
1528 1529
}

1530
static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1531 1532
{
	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1533 1534
	unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
		<< SECTOR_SHIFT;
1535

1536
	return (granularity + lim->alignment_offset - alignment) % granularity;
1537 1538
}

M
Martin K. Petersen 已提交
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551
static inline int bdev_alignment_offset(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q->limits.misaligned)
		return -1;

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->alignment_offset;

	return q->limits.alignment_offset;
}

1552 1553 1554 1555 1556 1557 1558 1559
static inline int queue_discard_alignment(struct request_queue *q)
{
	if (q->limits.discard_misaligned)
		return -1;

	return q->limits.discard_alignment;
}

1560
static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1561
{
1562
	unsigned int alignment, granularity, offset;
1563

1564 1565 1566
	if (!lim->max_discard_sectors)
		return 0;

1567
	/* Why are these in bytes, not sectors? */
1568 1569
	alignment = lim->discard_alignment >> SECTOR_SHIFT;
	granularity = lim->discard_granularity >> SECTOR_SHIFT;
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
	if (!granularity)
		return 0;

	/* Offset of the partition start in 'granularity' sectors */
	offset = sector_div(sector, granularity);

	/* And why do we do this modulus *again* in blkdev_issue_discard()? */
	offset = (granularity + alignment - offset) % granularity;

	/* Turn it back into bytes, gaah */
1580
	return offset << SECTOR_SHIFT;
1581 1582
}

1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
static inline int bdev_discard_alignment(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->discard_alignment;

	return q->limits.discard_alignment;
}

1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
static inline unsigned int bdev_write_same(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return q->limits.max_write_same_sectors;

	return 0;
}

1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return q->limits.max_write_zeroes_sectors;

	return 0;
}

D
Damien Le Moal 已提交
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return blk_queue_zoned_model(q);

	return BLK_ZONED_NONE;
}

static inline bool bdev_is_zoned(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return blk_queue_is_zoned(q);

	return false;
}

1633
static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
1634 1635 1636 1637
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
1638
		return blk_queue_zone_sectors(q);
1639 1640
	return 0;
}
1641

1642 1643 1644
static inline unsigned int bdev_nr_zones(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);
1645

1646 1647
	if (q)
		return blk_queue_nr_zones(q);
1648 1649 1650
	return 0;
}

1651
static inline int queue_dma_alignment(struct request_queue *q)
L
Linus Torvalds 已提交
1652
{
1653
	return q ? q->dma_alignment : 511;
L
Linus Torvalds 已提交
1654 1655
}

1656
static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1657 1658 1659
				 unsigned int len)
{
	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1660
	return !(addr & alignment) && !(len & alignment);
1661 1662
}

L
Linus Torvalds 已提交
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673
/* assumes size > 256 */
static inline unsigned int blksize_bits(unsigned int size)
{
	unsigned int bits = 8;
	do {
		bits++;
		size >>= 1;
	} while (size > 256);
	return bits;
}

1674
static inline unsigned int block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1675 1676 1677 1678
{
	return bdev->bd_block_size;
}

1679 1680
static inline bool queue_flush_queueable(struct request_queue *q)
{
J
Jens Axboe 已提交
1681
	return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
1682 1683
}

L
Linus Torvalds 已提交
1684 1685 1686 1687 1688 1689
typedef struct {struct page *v;} Sector;

unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);

static inline void put_dev_sector(Sector p)
{
1690
	put_page(p.v);
L
Linus Torvalds 已提交
1691 1692
}

1693 1694 1695 1696 1697 1698 1699
static inline bool __bvec_gap_to_prev(struct request_queue *q,
				struct bio_vec *bprv, unsigned int offset)
{
	return offset ||
		((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
}

1700 1701 1702 1703 1704 1705 1706 1707 1708
/*
 * Check if adding a bio_vec after bprv with offset would create a gap in
 * the SG list. Most drivers don't care about this, but some do.
 */
static inline bool bvec_gap_to_prev(struct request_queue *q,
				struct bio_vec *bprv, unsigned int offset)
{
	if (!queue_virt_boundary(q))
		return false;
1709
	return __bvec_gap_to_prev(q, bprv, offset);
1710 1711
}

M
Ming Lei 已提交
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
/*
 * Check if the two bvecs from two bios can be merged to one segment.
 * If yes, no need to check gap between the two bios since the 1st bio
 * and the 1st bvec in the 2nd bio can be handled in one segment.
 */
static inline bool bios_segs_mergeable(struct request_queue *q,
		struct bio *prev, struct bio_vec *prev_last_bv,
		struct bio_vec *next_first_bv)
{
	if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
		return false;
	if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
		return false;
	if (prev->bi_seg_back_size + next_first_bv->bv_len >
			queue_max_segment_size(q))
		return false;
	return true;
}

1731 1732 1733 1734
static inline bool bio_will_gap(struct request_queue *q,
				struct request *prev_rq,
				struct bio *prev,
				struct bio *next)
1735
{
1736 1737 1738
	if (bio_has_data(prev) && queue_virt_boundary(q)) {
		struct bio_vec pb, nb;

1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
		/*
		 * don't merge if the 1st bio starts with non-zero
		 * offset, otherwise it is quite difficult to respect
		 * sg gap limit. We work hard to merge a huge number of small
		 * single bios in case of mkfs.
		 */
		if (prev_rq)
			bio_get_first_bvec(prev_rq->bio, &pb);
		else
			bio_get_first_bvec(prev, &pb);
		if (pb.bv_offset)
			return true;

		/*
		 * We don't need to worry about the situation that the
		 * merged segment ends in unaligned virt boundary:
		 *
		 * - if 'pb' ends aligned, the merged segment ends aligned
		 * - if 'pb' ends unaligned, the next bio must include
		 *   one single bvec of 'nb', otherwise the 'nb' can't
		 *   merge with 'pb'
		 */
1761 1762
		bio_get_last_bvec(prev, &pb);
		bio_get_first_bvec(next, &nb);
1763

M
Ming Lei 已提交
1764 1765
		if (!bios_segs_mergeable(q, prev, &pb, &nb))
			return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1766 1767 1768
	}

	return false;
1769 1770 1771 1772
}

static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
{
1773
	return bio_will_gap(req->q, req, req->biotail, bio);
1774 1775 1776 1777
}

static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
{
1778
	return bio_will_gap(req->q, NULL, bio, req->bio);
1779 1780
}

1781
int kblockd_schedule_work(struct work_struct *work);
1782
int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1783
int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
L
Linus Torvalds 已提交
1784 1785 1786 1787 1788 1789

#define MODULE_ALIAS_BLOCKDEV(major,minor) \
	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
	MODULE_ALIAS("block-major-" __stringify(major) "-*")

1790 1791
#if defined(CONFIG_BLK_DEV_INTEGRITY)

1792 1793 1794
enum blk_integrity_flags {
	BLK_INTEGRITY_VERIFY		= 1 << 0,
	BLK_INTEGRITY_GENERATE		= 1 << 1,
1795
	BLK_INTEGRITY_DEVICE_CAPABLE	= 1 << 2,
1796
	BLK_INTEGRITY_IP_CHECKSUM	= 1 << 3,
1797
};
1798

1799
struct blk_integrity_iter {
1800 1801
	void			*prot_buf;
	void			*data_buf;
1802
	sector_t		seed;
1803
	unsigned int		data_size;
1804
	unsigned short		interval;
1805 1806 1807
	const char		*disk_name;
};

1808
typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1809

1810 1811 1812 1813 1814
struct blk_integrity_profile {
	integrity_processing_fn		*generate_fn;
	integrity_processing_fn		*verify_fn;
	const char			*name;
};
1815

1816
extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1817
extern void blk_integrity_unregister(struct gendisk *);
1818
extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1819 1820 1821
extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
				   struct scatterlist *);
extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1822 1823 1824 1825
extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
				   struct request *);
extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
				    struct bio *);
1826

1827
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1828
{
1829
	struct blk_integrity *bi = &disk->queue->integrity;
1830 1831 1832 1833 1834

	if (!bi->profile)
		return NULL;

	return bi;
1835 1836
}

1837 1838
static inline
struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1839
{
1840
	return blk_get_integrity(bdev->bd_disk);
1841 1842
}

1843
static inline bool blk_integrity_rq(struct request *rq)
1844
{
1845
	return rq->cmd_flags & REQ_INTEGRITY;
1846 1847
}

1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
	q->limits.max_integrity_segments = segs;
}

static inline unsigned short
queue_max_integrity_segments(struct request_queue *q)
{
	return q->limits.max_integrity_segments;
}

1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
static inline bool integrity_req_gap_back_merge(struct request *req,
						struct bio *next)
{
	struct bio_integrity_payload *bip = bio_integrity(req->bio);
	struct bio_integrity_payload *bip_next = bio_integrity(next);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}

static inline bool integrity_req_gap_front_merge(struct request *req,
						 struct bio *bio)
{
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}

1880 1881
#else /* CONFIG_BLK_DEV_INTEGRITY */

1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
struct bio;
struct block_device;
struct gendisk;
struct blk_integrity;

static inline int blk_integrity_rq(struct request *rq)
{
	return 0;
}
static inline int blk_rq_count_integrity_sg(struct request_queue *q,
					    struct bio *b)
{
	return 0;
}
static inline int blk_rq_map_integrity_sg(struct request_queue *q,
					  struct bio *b,
					  struct scatterlist *s)
{
	return 0;
}
static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
{
1904
	return NULL;
1905 1906 1907 1908 1909 1910 1911 1912 1913
}
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
	return NULL;
}
static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
{
	return 0;
}
1914
static inline void blk_integrity_register(struct gendisk *d,
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
					 struct blk_integrity *b)
{
}
static inline void blk_integrity_unregister(struct gendisk *d)
{
}
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
}
static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
{
	return 0;
}
1929 1930 1931
static inline bool blk_integrity_merge_rq(struct request_queue *rq,
					  struct request *r1,
					  struct request *r2)
1932
{
1933
	return true;
1934
}
1935 1936 1937
static inline bool blk_integrity_merge_bio(struct request_queue *rq,
					   struct request *r,
					   struct bio *b)
1938
{
1939
	return true;
1940
}
1941

1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
static inline bool integrity_req_gap_back_merge(struct request *req,
						struct bio *next)
{
	return false;
}
static inline bool integrity_req_gap_front_merge(struct request *req,
						 struct bio *bio)
{
	return false;
}
1952 1953 1954

#endif /* CONFIG_BLK_DEV_INTEGRITY */

1955
struct block_device_operations {
A
Al Viro 已提交
1956
	int (*open) (struct block_device *, fmode_t);
1957
	void (*release) (struct gendisk *, fmode_t);
1958
	int (*rw_page)(struct block_device *, sector_t, struct page *, bool);
A
Al Viro 已提交
1959 1960
	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1961 1962 1963
	unsigned int (*check_events) (struct gendisk *disk,
				      unsigned int clearing);
	/* ->media_changed() is DEPRECATED, use ->check_events() instead */
1964
	int (*media_changed) (struct gendisk *);
1965
	void (*unlock_native_capacity) (struct gendisk *);
1966 1967
	int (*revalidate_disk) (struct gendisk *);
	int (*getgeo)(struct block_device *, struct hd_geometry *);
1968 1969
	/* this callback is with swap_lock and sometimes page table lock held */
	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1970
	struct module *owner;
1971
	const struct pr_ops *pr_ops;
1972 1973
};

1974 1975
extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
				 unsigned long);
1976 1977 1978
extern int bdev_read_page(struct block_device *, sector_t, struct page *);
extern int bdev_write_page(struct block_device *, sector_t, struct page *,
						struct writeback_control *);
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032

#ifdef CONFIG_BLK_DEV_ZONED
bool blk_req_needs_zone_write_lock(struct request *rq);
void __blk_req_zone_write_lock(struct request *rq);
void __blk_req_zone_write_unlock(struct request *rq);

static inline void blk_req_zone_write_lock(struct request *rq)
{
	if (blk_req_needs_zone_write_lock(rq))
		__blk_req_zone_write_lock(rq);
}

static inline void blk_req_zone_write_unlock(struct request *rq)
{
	if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
		__blk_req_zone_write_unlock(rq);
}

static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
	return rq->q->seq_zones_wlock &&
		test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
}

static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
	if (!blk_req_needs_zone_write_lock(rq))
		return true;
	return !blk_req_zone_is_write_locked(rq);
}
#else
static inline bool blk_req_needs_zone_write_lock(struct request *rq)
{
	return false;
}

static inline void blk_req_zone_write_lock(struct request *rq)
{
}

static inline void blk_req_zone_write_unlock(struct request *rq)
{
}
static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
	return false;
}

static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
	return true;
}
#endif /* CONFIG_BLK_DEV_ZONED */

2033
#else /* CONFIG_BLOCK */
2034 2035 2036

struct block_device;

2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
/*
 * stubs for when the block layer is configured out
 */
#define buffer_heads_over_limit 0

static inline long nr_blockdev_pages(void)
{
	return 0;
}

2047 2048 2049 2050
struct blk_plug {
};

static inline void blk_start_plug(struct blk_plug *plug)
2051 2052 2053
{
}

2054
static inline void blk_finish_plug(struct blk_plug *plug)
2055 2056 2057
{
}

2058
static inline void blk_flush_plug(struct task_struct *task)
2059 2060 2061
{
}

2062 2063 2064 2065 2066
static inline void blk_schedule_flush_plug(struct task_struct *task)
{
}


2067 2068 2069 2070 2071
static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	return false;
}

2072 2073 2074 2075 2076 2077
static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
				     sector_t *error_sector)
{
	return 0;
}

2078 2079
#endif /* CONFIG_BLOCK */

L
Linus Torvalds 已提交
2080
#endif