blkdev.h 62.2 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
L
Linus Torvalds 已提交
2 3 4
#ifndef _LINUX_BLKDEV_H
#define _LINUX_BLKDEV_H

5
#include <linux/sched.h>
6
#include <linux/sched/clock.h>
7

J
Jens Axboe 已提交
8 9
#ifdef CONFIG_BLOCK

L
Linus Torvalds 已提交
10 11 12
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/list.h>
13
#include <linux/llist.h>
L
Linus Torvalds 已提交
14 15 16
#include <linux/timer.h>
#include <linux/workqueue.h>
#include <linux/pagemap.h>
17
#include <linux/backing-dev-defs.h>
L
Linus Torvalds 已提交
18 19
#include <linux/wait.h>
#include <linux/mempool.h>
D
Dan Williams 已提交
20
#include <linux/pfn.h>
L
Linus Torvalds 已提交
21 22
#include <linux/bio.h>
#include <linux/stringify.h>
23
#include <linux/gfp.h>
24
#include <linux/bsg.h>
25
#include <linux/smp.h>
T
Tejun Heo 已提交
26
#include <linux/rcupdate.h>
27
#include <linux/percpu-refcount.h>
C
Christoph Hellwig 已提交
28
#include <linux/scatterlist.h>
29
#include <linux/blkzoned.h>
30 31
#include <linux/seqlock.h>
#include <linux/u64_stats_sync.h>
L
Linus Torvalds 已提交
32

33
struct module;
34 35
struct scsi_ioctl_command;

L
Linus Torvalds 已提交
36 37
struct request_queue;
struct elevator_queue;
38
struct blk_trace;
39 40
struct request;
struct sg_io_hdr;
M
Mike Christie 已提交
41
struct bsg_job;
T
Tejun Heo 已提交
42
struct blkcg_gq;
43
struct blk_flush_queue;
44
struct pr_ops;
J
Jens Axboe 已提交
45
struct rq_wb;
46 47
struct blk_queue_stats;
struct blk_stat_callback;
L
Linus Torvalds 已提交
48 49 50 51

#define BLKDEV_MIN_RQ	4
#define BLKDEV_MAX_RQ	128	/* Default maximum */

52
/* Must be consistent with blk_mq_poll_stats_bkt() */
53 54
#define BLK_MQ_POLL_STATS_BKTS 16

T
Tejun Heo 已提交
55 56 57 58
/*
 * Maximum number of blkcg policies allowed to be registered concurrently.
 * Defined here to simplify include dependency.
 */
59
#define BLKCG_MAX_POLS		3
T
Tejun Heo 已提交
60

61
typedef void (rq_end_io_fn)(struct request *, blk_status_t);
L
Linus Torvalds 已提交
62

63 64 65
#define BLK_RL_SYNCFULL		(1U << 0)
#define BLK_RL_ASYNCFULL	(1U << 1)

L
Linus Torvalds 已提交
66
struct request_list {
67
	struct request_queue	*q;	/* the queue this rl belongs to */
68 69 70
#ifdef CONFIG_BLK_CGROUP
	struct blkcg_gq		*blkg;	/* blkg this request pool belongs to */
#endif
71 72 73 74
	/*
	 * count[], starved[], and wait[] are indexed by
	 * BLK_RW_SYNC/BLK_RW_ASYNC
	 */
75 76 77 78
	int			count[2];
	int			starved[2];
	mempool_t		*rq_pool;
	wait_queue_head_t	wait[2];
79
	unsigned int		flags;
L
Linus Torvalds 已提交
80 81
};

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
/*
 * request flags */
typedef __u32 __bitwise req_flags_t;

/* elevator knows about this request */
#define RQF_SORTED		((__force req_flags_t)(1 << 0))
/* drive already may have started this one */
#define RQF_STARTED		((__force req_flags_t)(1 << 1))
/* uses tagged queueing */
#define RQF_QUEUED		((__force req_flags_t)(1 << 2))
/* may not be passed by ioscheduler */
#define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
/* request for flush sequence */
#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << 4))
/* merge of different types, fail separately */
#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << 5))
/* track inflight for MQ */
#define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
/* don't call prep for this one */
#define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
/* set for "ide_preempt" requests and also for requests for which the SCSI
   "quiesce" state must be ignored. */
#define RQF_PREEMPT		((__force req_flags_t)(1 << 8))
/* contains copies of user pages */
#define RQF_COPY_USER		((__force req_flags_t)(1 << 9))
/* vaguely specified driver internal error.  Ignored by the block layer */
#define RQF_FAILED		((__force req_flags_t)(1 << 10))
/* don't warn about errors */
#define RQF_QUIET		((__force req_flags_t)(1 << 11))
/* elevator private data attached */
#define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
/* account I/O stat */
#define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
/* request came from our alloc pool */
#define RQF_ALLOCED		((__force req_flags_t)(1 << 14))
/* runtime pm request */
#define RQF_PM			((__force req_flags_t)(1 << 15))
/* on IO scheduler merge hash */
#define RQF_HASHED		((__force req_flags_t)(1 << 16))
121 122
/* IO stats tracking on */
#define RQF_STATS		((__force req_flags_t)(1 << 17))
123 124 125
/* Look at ->special_vec for the actual data payload instead of the
   bio chain. */
#define RQF_SPECIAL_PAYLOAD	((__force req_flags_t)(1 << 18))
126 127
/* The per-zone write lock is held for this request */
#define RQF_ZONE_WRITE_LOCKED	((__force req_flags_t)(1 << 19))
128 129
/* timeout is expired */
#define RQF_MQ_TIMEOUT_EXPIRED	((__force req_flags_t)(1 << 20))
J
Jens Axboe 已提交
130 131
/* already slept for hybrid poll */
#define RQF_MQ_POLL_SLEPT	((__force req_flags_t)(1 << 21))
132 133 134

/* flags that prevent us from merging requests: */
#define RQF_NOMERGE_FLAGS \
135
	(RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
136

L
Linus Torvalds 已提交
137
/*
138 139 140 141
 * Try to put the fields that are referenced together in the same cacheline.
 *
 * If you modify this structure, make sure to update blk_rq_init() and
 * especially blk_mq_rq_ctx_init() to take care of the added fields.
L
Linus Torvalds 已提交
142 143
 */
struct request {
144
	struct request_queue *q;
145
	struct blk_mq_ctx *mq_ctx;
146

147
	int cpu;
148
	unsigned int cmd_flags;		/* op and common flags */
149
	req_flags_t rq_flags;
150 151 152

	int internal_tag;

153 154
	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
155
	int tag;
156
	sector_t __sector;		/* sector cursor */
L
Linus Torvalds 已提交
157 158 159 160

	struct bio *bio;
	struct bio *biotail;

161 162
	struct list_head queuelist;

163 164 165 166 167 168 169 170 171 172 173 174
	/*
	 * The hash is used inside the scheduler, and killed once the
	 * request reaches the dispatch list. The ipi_list is only used
	 * to queue the request for softirq completion, which is long
	 * after the request has been unhashed (and even removed from
	 * the dispatch list).
	 */
	union {
		struct hlist_node hash;	/* merge hash */
		struct list_head ipi_list;
	};

175 176 177
	/*
	 * The rb_node is only used inside the io scheduler, requests
	 * are pruned when moved to the dispatch queue. So let the
178
	 * completion_data share space with the rb_node.
179 180 181
	 */
	union {
		struct rb_node rb_node;	/* sort/lookup */
182
		struct bio_vec special_vec;
183
		void *completion_data;
184
		int error_count; /* for legacy drivers, don't use */
185
	};
186

187
	/*
188
	 * Three pointers are available for the IO schedulers, if they need
189 190
	 * more they have to dynamically allocate it.  Flush requests are
	 * never put on the IO scheduler. So let the flush fields share
191
	 * space with the elevator data.
192
	 */
193
	union {
194 195 196 197 198
		struct {
			struct io_cq		*icq;
			void			*priv[2];
		} elv;

199 200 201
		struct {
			unsigned int		seq;
			struct list_head	list;
202
			rq_end_io_fn		*saved_end_io;
203 204
		} flush;
	};
205

206
	struct gendisk *rq_disk;
207
	struct hd_struct *part;
L
Linus Torvalds 已提交
208
	unsigned long start_time;
209
	struct blk_issue_stat issue_stat;
L
Linus Torvalds 已提交
210 211 212 213
	/* Number of scatter-gather DMA addr+len pairs after
	 * physical address coalescing is performed.
	 */
	unsigned short nr_phys_segments;
214

215 216 217
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	unsigned short nr_integrity_segments;
#endif
L
Linus Torvalds 已提交
218

219
	unsigned short write_hint;
220 221
	unsigned short ioprio;

222
	unsigned int timeout;
L
Linus Torvalds 已提交
223

T
Tejun Heo 已提交
224
	void *special;		/* opaque pointer available for LLD use */
225

226
	unsigned int extra_len;	/* length of alignment and padding */
L
Linus Torvalds 已提交
227

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
	/*
	 * On blk-mq, the lower bits of ->gstate (generation number and
	 * state) carry the MQ_RQ_* state value and the upper bits the
	 * generation number which is monotonically incremented and used to
	 * distinguish the reuse instances.
	 *
	 * ->gstate_seq allows updates to ->gstate and other fields
	 * (currently ->deadline) during request start to be read
	 * atomically from the timeout path, so that it can operate on a
	 * coherent set of information.
	 */
	seqcount_t gstate_seq;
	u64 gstate;

	/*
	 * ->aborted_gstate is used by the timeout to claim a specific
	 * recycle instance of this request.  See blk_mq_timeout_work().
	 */
	struct u64_stats_sync aborted_gstate_sync;
	u64 aborted_gstate;

249 250
	/* access through blk_rq_set_deadline, blk_rq_deadline */
	unsigned long __deadline;
251

J
Jens Axboe 已提交
252
	struct list_head timeout_list;
L
Linus Torvalds 已提交
253

254
	union {
255
		struct __call_single_data csd;
256 257 258
		u64 fifo_time;
	};

L
Linus Torvalds 已提交
259
	/*
260
	 * completion callback.
L
Linus Torvalds 已提交
261 262 263
	 */
	rq_end_io_fn *end_io;
	void *end_io_data;
264 265 266

	/* for bidi */
	struct request *next_rq;
267 268 269 270 271 272

#ifdef CONFIG_BLK_CGROUP
	struct request_list *rl;		/* rl this rq is alloced from */
	unsigned long long start_time_ns;
	unsigned long long io_start_time_ns;    /* when passed to hardware */
#endif
L
Linus Torvalds 已提交
273 274
};

275 276 277 278 279 280 281 282 283 284
static inline bool blk_op_is_scsi(unsigned int op)
{
	return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
}

static inline bool blk_op_is_private(unsigned int op)
{
	return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
}

285 286
static inline bool blk_rq_is_scsi(struct request *rq)
{
287
	return blk_op_is_scsi(req_op(rq));
288 289 290 291
}

static inline bool blk_rq_is_private(struct request *rq)
{
292
	return blk_op_is_private(req_op(rq));
293 294
}

295 296
static inline bool blk_rq_is_passthrough(struct request *rq)
{
297
	return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
298 299
}

300 301 302 303 304 305 306
static inline bool bio_is_passthrough(struct bio *bio)
{
	unsigned op = bio_op(bio);

	return blk_op_is_scsi(op) || blk_op_is_private(op);
}

307 308 309 310 311
static inline unsigned short req_get_ioprio(struct request *req)
{
	return req->ioprio;
}

L
Linus Torvalds 已提交
312 313
#include <linux/elevator.h>

314 315
struct blk_queue_ctx;

316
typedef void (request_fn_proc) (struct request_queue *q);
317
typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
318
typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
319
typedef int (prep_rq_fn) (struct request_queue *, struct request *);
320
typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
L
Linus Torvalds 已提交
321 322

struct bio_vec;
323
typedef void (softirq_done_fn)(struct request *);
324
typedef int (dma_drain_needed_fn)(struct request *);
325
typedef int (lld_busy_fn) (struct request_queue *q);
M
Mike Christie 已提交
326
typedef int (bsg_job_fn) (struct bsg_job *);
327 328
typedef int (init_rq_fn)(struct request_queue *, struct request *, gfp_t);
typedef void (exit_rq_fn)(struct request_queue *, struct request *);
L
Linus Torvalds 已提交
329

J
Jens Axboe 已提交
330 331 332 333 334 335 336 337
enum blk_eh_timer_return {
	BLK_EH_NOT_HANDLED,
	BLK_EH_HANDLED,
	BLK_EH_RESET_TIMER,
};

typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);

L
Linus Torvalds 已提交
338 339 340 341 342 343 344 345 346
enum blk_queue_state {
	Queue_down,
	Queue_up,
};

struct blk_queue_tag {
	struct request **tag_index;	/* map of busy tags */
	unsigned long *tag_map;		/* bit map of free/busy tags */
	int max_depth;			/* what we will send to device */
347
	int real_max_depth;		/* what the array can hold */
L
Linus Torvalds 已提交
348
	atomic_t refcnt;		/* map can be shared */
349 350
	int alloc_policy;		/* tag allocation policy */
	int next_tag;			/* next tag */
L
Linus Torvalds 已提交
351
};
352 353
#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
L
Linus Torvalds 已提交
354

355 356 357
#define BLK_SCSI_MAX_CMDS	(256)
#define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))

D
Damien Le Moal 已提交
358 359 360 361 362 363 364 365 366
/*
 * Zoned block device models (zoned limit).
 */
enum blk_zoned_model {
	BLK_ZONED_NONE,	/* Regular block device */
	BLK_ZONED_HA,	/* Host-aware zoned block device */
	BLK_ZONED_HM,	/* Host-managed zoned block device */
};

367 368 369
struct queue_limits {
	unsigned long		bounce_pfn;
	unsigned long		seg_boundary_mask;
370
	unsigned long		virt_boundary_mask;
371 372

	unsigned int		max_hw_sectors;
373
	unsigned int		max_dev_sectors;
374
	unsigned int		chunk_sectors;
375 376
	unsigned int		max_sectors;
	unsigned int		max_segment_size;
377 378 379 380
	unsigned int		physical_block_size;
	unsigned int		alignment_offset;
	unsigned int		io_min;
	unsigned int		io_opt;
381
	unsigned int		max_discard_sectors;
382
	unsigned int		max_hw_discard_sectors;
383
	unsigned int		max_write_same_sectors;
384
	unsigned int		max_write_zeroes_sectors;
385 386
	unsigned int		discard_granularity;
	unsigned int		discard_alignment;
387 388

	unsigned short		logical_block_size;
389
	unsigned short		max_segments;
390
	unsigned short		max_integrity_segments;
391
	unsigned short		max_discard_segments;
392

393
	unsigned char		misaligned;
394
	unsigned char		discard_misaligned;
395
	unsigned char		cluster;
396
	unsigned char		raid_partial_stripes_expensive;
D
Damien Le Moal 已提交
397
	enum blk_zoned_model	zoned;
398 399
};

400 401 402 403 404 405 406 407 408 409 410 411 412
#ifdef CONFIG_BLK_DEV_ZONED

struct blk_zone_report_hdr {
	unsigned int	nr_zones;
	u8		padding[60];
};

extern int blkdev_report_zones(struct block_device *bdev,
			       sector_t sector, struct blk_zone *zones,
			       unsigned int *nr_zones, gfp_t gfp_mask);
extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
			      sector_t nr_sectors, gfp_t gfp_mask);

S
Shaun Tancheff 已提交
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
				     unsigned int cmd, unsigned long arg);
extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
				    unsigned int cmd, unsigned long arg);

#else /* CONFIG_BLK_DEV_ZONED */

static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
					    fmode_t mode, unsigned int cmd,
					    unsigned long arg)
{
	return -ENOTTY;
}

static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
					   fmode_t mode, unsigned int cmd,
					   unsigned long arg)
{
	return -ENOTTY;
}

434 435
#endif /* CONFIG_BLK_DEV_ZONED */

436
struct request_queue {
L
Linus Torvalds 已提交
437 438 439 440 441
	/*
	 * Together with queue_head for cacheline sharing
	 */
	struct list_head	queue_head;
	struct request		*last_merge;
J
Jens Axboe 已提交
442
	struct elevator_queue	*elevator;
443 444
	int			nr_rqs[2];	/* # allocated [a]sync rqs */
	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
L
Linus Torvalds 已提交
445

446 447
	atomic_t		shared_hctx_restart;

448
	struct blk_queue_stats	*stats;
J
Jens Axboe 已提交
449 450
	struct rq_wb		*rq_wb;

L
Linus Torvalds 已提交
451
	/*
452 453 454 455
	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
	 * is used, root blkg allocates from @q->root_rl and all other
	 * blkgs from their own blkg->rl.  Which one to use should be
	 * determined using bio_request_list().
L
Linus Torvalds 已提交
456
	 */
457
	struct request_list	root_rl;
L
Linus Torvalds 已提交
458 459 460

	request_fn_proc		*request_fn;
	make_request_fn		*make_request_fn;
461
	poll_q_fn		*poll_fn;
L
Linus Torvalds 已提交
462
	prep_rq_fn		*prep_rq_fn;
463
	unprep_rq_fn		*unprep_rq_fn;
464
	softirq_done_fn		*softirq_done_fn;
J
Jens Axboe 已提交
465
	rq_timed_out_fn		*rq_timed_out_fn;
466
	dma_drain_needed_fn	*dma_drain_needed;
467
	lld_busy_fn		*lld_busy_fn;
468
	/* Called just after a request is allocated */
469
	init_rq_fn		*init_rq_fn;
470
	/* Called just before a request is freed */
471
	exit_rq_fn		*exit_rq_fn;
472 473
	/* Called from inside blk_get_request() */
	void (*initialize_rq_fn)(struct request *rq);
L
Linus Torvalds 已提交
474

J
Jens Axboe 已提交
475
	const struct blk_mq_ops	*mq_ops;
476 477 478 479

	unsigned int		*mq_map;

	/* sw queues */
480
	struct blk_mq_ctx __percpu	*queue_ctx;
481 482
	unsigned int		nr_queues;

483 484
	unsigned int		queue_depth;

485 486 487 488
	/* hw dispatch queues */
	struct blk_mq_hw_ctx	**queue_hw_ctx;
	unsigned int		nr_hw_queues;

489 490 491
	/*
	 * Dispatch queue sorting
	 */
J
Jens Axboe 已提交
492
	sector_t		end_sector;
493 494
	struct request		*boundary_rq;

L
Linus Torvalds 已提交
495
	/*
496
	 * Delayed queue handling
L
Linus Torvalds 已提交
497
	 */
498
	struct delayed_work	delay_work;
L
Linus Torvalds 已提交
499

500
	struct backing_dev_info	*backing_dev_info;
L
Linus Torvalds 已提交
501 502 503 504 505 506 507 508

	/*
	 * The queue owner gets to use this for whatever they like.
	 * ll_rw_blk doesn't touch it.
	 */
	void			*queuedata;

	/*
509
	 * various queue flags, see QUEUE_* below
L
Linus Torvalds 已提交
510
	 */
511
	unsigned long		queue_flags;
L
Linus Torvalds 已提交
512

513 514 515 516 517 518
	/*
	 * ida allocated id for this queue.  Used to index queues from
	 * ioctx.
	 */
	int			id;

L
Linus Torvalds 已提交
519
	/*
520
	 * queue needs bounce pages for pages above this limit
L
Linus Torvalds 已提交
521
	 */
522
	gfp_t			bounce_gfp;
L
Linus Torvalds 已提交
523 524

	/*
已提交
525 526 527
	 * protects queue structures from reentrancy. ->__queue_lock should
	 * _never_ be used directly, it is queue private. always use
	 * ->queue_lock.
L
Linus Torvalds 已提交
528
	 */
已提交
529
	spinlock_t		__queue_lock;
L
Linus Torvalds 已提交
530 531 532 533 534 535 536
	spinlock_t		*queue_lock;

	/*
	 * queue kobject
	 */
	struct kobject kobj;

537 538 539 540 541
	/*
	 * mq queue kobject
	 */
	struct kobject mq_kobj;

542 543 544 545
#ifdef  CONFIG_BLK_DEV_INTEGRITY
	struct blk_integrity integrity;
#endif	/* CONFIG_BLK_DEV_INTEGRITY */

546
#ifdef CONFIG_PM
L
Lin Ming 已提交
547 548 549 550 551
	struct device		*dev;
	int			rpm_status;
	unsigned int		nr_pending;
#endif

L
Linus Torvalds 已提交
552 553 554 555 556 557 558 559
	/*
	 * queue settings
	 */
	unsigned long		nr_requests;	/* Max # of requests */
	unsigned int		nr_congestion_on;
	unsigned int		nr_congestion_off;
	unsigned int		nr_batching;

J
James Bottomley 已提交
560
	unsigned int		dma_drain_size;
561
	void			*dma_drain_buffer;
562
	unsigned int		dma_pad_mask;
L
Linus Torvalds 已提交
563 564 565
	unsigned int		dma_alignment;

	struct blk_queue_tag	*queue_tags;
566
	struct list_head	tag_busy_list;
L
Linus Torvalds 已提交
567

568
	unsigned int		nr_sorted;
569
	unsigned int		in_flight[2];
570

571 572 573 574 575 576
	/*
	 * Number of active block driver functions for which blk_drain_queue()
	 * must wait. Must be incremented around functions that unlock the
	 * queue_lock internally, e.g. scsi_request_fn().
	 */
	unsigned int		request_fn_active;
L
Linus Torvalds 已提交
577

J
Jens Axboe 已提交
578
	unsigned int		rq_timeout;
579
	int			poll_nsec;
580 581

	struct blk_stat_callback	*poll_cb;
582
	struct blk_rq_stat	poll_stat[BLK_MQ_POLL_STATS_BKTS];
583

J
Jens Axboe 已提交
584
	struct timer_list	timeout;
585
	struct work_struct	timeout_work;
J
Jens Axboe 已提交
586 587
	struct list_head	timeout_list;

588
	struct list_head	icq_list;
589
#ifdef CONFIG_BLK_CGROUP
590
	DECLARE_BITMAP		(blkcg_pols, BLKCG_MAX_POLS);
T
Tejun Heo 已提交
591
	struct blkcg_gq		*root_blkg;
592
	struct list_head	blkg_list;
593
#endif
594

595 596
	struct queue_limits	limits;

597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
	/*
	 * Zoned block device information for request dispatch control.
	 * nr_zones is the total number of zones of the device. This is always
	 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
	 * bits which indicates if a zone is conventional (bit clear) or
	 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
	 * bits which indicates if a zone is write locked, that is, if a write
	 * request targeting the zone was dispatched. All three fields are
	 * initialized by the low level device driver (e.g. scsi/sd.c).
	 * Stacking drivers (device mappers) may or may not initialize
	 * these fields.
	 */
	unsigned int		nr_zones;
	unsigned long		*seq_zones_bitmap;
	unsigned long		*seq_zones_wlock;

L
Linus Torvalds 已提交
613 614 615 616 617
	/*
	 * sg stuff
	 */
	unsigned int		sg_timeout;
	unsigned int		sg_reserved_size;
618
	int			node;
619
#ifdef CONFIG_BLK_DEV_IO_TRACE
620
	struct blk_trace	*blk_trace;
621
	struct mutex		blk_trace_mutex;
622
#endif
L
Linus Torvalds 已提交
623
	/*
624
	 * for flush operations
L
Linus Torvalds 已提交
625
	 */
626
	struct blk_flush_queue	*fq;
627

628 629
	struct list_head	requeue_list;
	spinlock_t		requeue_lock;
630
	struct delayed_work	requeue_work;
631

632
	struct mutex		sysfs_lock;
633

634
	int			bypass_depth;
635
	atomic_t		mq_freeze_depth;
636

637
#if defined(CONFIG_BLK_DEV_BSG)
M
Mike Christie 已提交
638
	bsg_job_fn		*bsg_job_fn;
639 640
	struct bsg_class_device bsg_dev;
#endif
641 642 643 644 645

#ifdef CONFIG_BLK_DEV_THROTTLING
	/* Throttle data */
	struct throtl_data *td;
#endif
T
Tejun Heo 已提交
646
	struct rcu_head		rcu_head;
647
	wait_queue_head_t	mq_freeze_wq;
648
	struct percpu_ref	q_usage_counter;
649
	struct list_head	all_q_node;
650 651 652

	struct blk_mq_tag_set	*tag_set;
	struct list_head	tag_set_list;
653
	struct bio_set		*bio_split;
654

655
#ifdef CONFIG_BLK_DEBUG_FS
656
	struct dentry		*debugfs_dir;
657
	struct dentry		*sched_debugfs_dir;
658 659
#endif

660
	bool			mq_sysfs_init_done;
661 662 663

	size_t			cmd_size;
	void			*rq_alloc_data;
664 665

	struct work_struct	release_work;
666 667 668

#define BLK_MAX_WRITE_HINTS	5
	u64			write_hints[BLK_MAX_WRITE_HINTS];
L
Linus Torvalds 已提交
669 670
};

671 672 673 674 675 676 677 678 679
#define QUEUE_FLAG_QUEUED	0	/* uses generic tag queueing */
#define QUEUE_FLAG_STOPPED	1	/* queue is stopped */
#define QUEUE_FLAG_DYING	2	/* queue being torn down */
#define QUEUE_FLAG_BYPASS	3	/* act as dumb FIFO queue */
#define QUEUE_FLAG_BIDI		4	/* queue supports bidi requests */
#define QUEUE_FLAG_NOMERGES     5	/* disable merge attempts */
#define QUEUE_FLAG_SAME_COMP	6	/* complete on same CPU-group */
#define QUEUE_FLAG_FAIL_IO	7	/* fake timeout */
#define QUEUE_FLAG_NONROT	9	/* non-rotational device (SSD) */
680
#define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
#define QUEUE_FLAG_IO_STAT     10	/* do IO stats */
#define QUEUE_FLAG_DISCARD     11	/* supports DISCARD */
#define QUEUE_FLAG_NOXMERGES   12	/* No extended merges */
#define QUEUE_FLAG_ADD_RANDOM  13	/* Contributes to random pool */
#define QUEUE_FLAG_SECERASE    14	/* supports secure erase */
#define QUEUE_FLAG_SAME_FORCE  15	/* force complete on same CPU */
#define QUEUE_FLAG_DEAD        16	/* queue tear-down finished */
#define QUEUE_FLAG_INIT_DONE   17	/* queue is initialized */
#define QUEUE_FLAG_NO_SG_MERGE 18	/* don't attempt to merge SG segments*/
#define QUEUE_FLAG_POLL	       19	/* IO polling enabled if set */
#define QUEUE_FLAG_WC	       20	/* Write back caching */
#define QUEUE_FLAG_FUA	       21	/* device supports FUA writes */
#define QUEUE_FLAG_FLUSH_NQ    22	/* flush not queueuable */
#define QUEUE_FLAG_DAX         23	/* device supports DAX */
#define QUEUE_FLAG_STATS       24	/* track rq completion times */
#define QUEUE_FLAG_POLL_STATS  25	/* collecting stats for hybrid polling */
#define QUEUE_FLAG_REGISTERED  26	/* queue has been registered to a disk */
#define QUEUE_FLAG_SCSI_PASSTHROUGH 27	/* queue supports SCSI commands */
#define QUEUE_FLAG_QUIESCED    28	/* queue has been quiesced */
700
#define QUEUE_FLAG_PREEMPT_ONLY	29	/* only process REQ_PREEMPT requests */
701 702

#define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
703 704
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_ADD_RANDOM))
705

706
#define QUEUE_FLAG_MQ_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
707 708
				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
				 (1 << QUEUE_FLAG_POLL))
709

710 711 712 713 714
void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);

715 716 717 718 719 720 721
/*
 * @q->queue_lock is set while a queue is being initialized. Since we know
 * that no other threads access the queue object before @q->queue_lock has
 * been set, it is safe to manipulate queue flags without holding the
 * queue_lock if @q->queue_lock == NULL. See also blk_alloc_queue_node() and
 * blk_init_allocated_queue().
 */
722
static inline void queue_lockdep_assert_held(struct request_queue *q)
723
{
724 725
	if (q->queue_lock)
		lockdep_assert_held(q->queue_lock);
726 727
}

N
Nick Piggin 已提交
728 729 730 731 732 733
static inline void queue_flag_set_unlocked(unsigned int flag,
					   struct request_queue *q)
{
	__set_bit(flag, &q->queue_flags);
}

734 735 736 737 738 739
static inline void queue_flag_clear_unlocked(unsigned int flag,
					     struct request_queue *q)
{
	__clear_bit(flag, &q->queue_flags);
}

J
Jens Axboe 已提交
740 741 742
static inline int queue_flag_test_and_clear(unsigned int flag,
					    struct request_queue *q)
{
743
	queue_lockdep_assert_held(q);
J
Jens Axboe 已提交
744 745 746 747 748 749 750 751 752 753 754 755

	if (test_bit(flag, &q->queue_flags)) {
		__clear_bit(flag, &q->queue_flags);
		return 1;
	}

	return 0;
}

static inline int queue_flag_test_and_set(unsigned int flag,
					  struct request_queue *q)
{
756
	queue_lockdep_assert_held(q);
J
Jens Axboe 已提交
757 758 759 760 761 762 763 764 765

	if (!test_bit(flag, &q->queue_flags)) {
		__set_bit(flag, &q->queue_flags);
		return 0;
	}

	return 1;
}

N
Nick Piggin 已提交
766 767
static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
{
768
	queue_lockdep_assert_held(q);
N
Nick Piggin 已提交
769 770 771 772 773
	__set_bit(flag, &q->queue_flags);
}

static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
{
774
	queue_lockdep_assert_held(q);
N
Nick Piggin 已提交
775 776 777
	__clear_bit(flag, &q->queue_flags);
}

L
Linus Torvalds 已提交
778 779
#define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
#define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
B
Bart Van Assche 已提交
780
#define blk_queue_dying(q)	test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
781
#define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
782
#define blk_queue_bypass(q)	test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
783
#define blk_queue_init_done(q)	test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
784
#define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
785 786
#define blk_queue_noxmerges(q)	\
	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
787
#define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
788
#define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
789
#define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
790
#define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
791 792
#define blk_queue_secure_erase(q) \
	(test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
793
#define blk_queue_dax(q)	test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
794 795
#define blk_queue_scsi_passthrough(q)	\
	test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
L
Linus Torvalds 已提交
796

797 798 799
#define blk_noretry_request(rq) \
	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
			     REQ_FAILFAST_DRIVER))
800
#define blk_queue_quiesced(q)	test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
801 802 803 804 805
#define blk_queue_preempt_only(q)				\
	test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)

extern int blk_set_preempt_only(struct request_queue *q);
extern void blk_clear_preempt_only(struct request_queue *q);
806

807 808 809 810 811
static inline int queue_in_flight(struct request_queue *q)
{
	return q->in_flight[0] + q->in_flight[1];
}

812 813 814 815
static inline bool blk_account_rq(struct request *rq)
{
	return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
}
816

817
#define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
818
#define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
819 820
/* rq->queuelist of dequeued request must be list_empty() */
#define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
L
Linus Torvalds 已提交
821 822 823

#define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)

824
#define rq_data_dir(rq)		(op_is_write(req_op(rq)) ? WRITE : READ)
L
Linus Torvalds 已提交
825

826 827 828 829 830 831 832 833 834
/*
 * Driver can handle struct request, if it either has an old style
 * request_fn defined, or is blk-mq based.
 */
static inline bool queue_is_rq_based(struct request_queue *q)
{
	return q->request_fn || q->mq_ops;
}

835 836 837 838 839
static inline unsigned int blk_queue_cluster(struct request_queue *q)
{
	return q->limits.cluster;
}

D
Damien Le Moal 已提交
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
static inline enum blk_zoned_model
blk_queue_zoned_model(struct request_queue *q)
{
	return q->limits.zoned;
}

static inline bool blk_queue_is_zoned(struct request_queue *q)
{
	switch (blk_queue_zoned_model(q)) {
	case BLK_ZONED_HA:
	case BLK_ZONED_HM:
		return true;
	default:
		return false;
	}
}

857
static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
858 859 860 861
{
	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
}

862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
{
	return q->nr_zones;
}

static inline unsigned int blk_queue_zone_no(struct request_queue *q,
					     sector_t sector)
{
	if (!blk_queue_is_zoned(q))
		return 0;
	return sector >> ilog2(q->limits.chunk_sectors);
}

static inline bool blk_queue_zone_is_seq(struct request_queue *q,
					 sector_t sector)
{
	if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
		return false;
	return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
}

883 884
static inline bool rq_is_sync(struct request *rq)
{
885
	return op_is_sync(rq->cmd_flags);
886 887
}

888
static inline bool blk_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
889
{
890 891 892
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	return rl->flags & flag;
L
Linus Torvalds 已提交
893 894
}

895
static inline void blk_set_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
896
{
897 898 899
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags |= flag;
L
Linus Torvalds 已提交
900 901
}

902
static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
L
Linus Torvalds 已提交
903
{
904 905 906
	unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;

	rl->flags &= ~flag;
L
Linus Torvalds 已提交
907 908
}

909 910
static inline bool rq_mergeable(struct request *rq)
{
911
	if (blk_rq_is_passthrough(rq))
912
		return false;
L
Linus Torvalds 已提交
913

914 915 916
	if (req_op(rq) == REQ_OP_FLUSH)
		return false;

917 918 919
	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
		return false;

920
	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
921 922
		return false;
	if (rq->rq_flags & RQF_NOMERGE_FLAGS)
923 924 925 926
		return false;

	return true;
}
L
Linus Torvalds 已提交
927

928 929
static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
{
930 931
	if (bio_page(a) == bio_page(b) &&
	    bio_offset(a) == bio_offset(b))
932 933 934 935 936
		return true;

	return false;
}

937 938 939 940 941 942 943 944
static inline unsigned int blk_queue_depth(struct request_queue *q)
{
	if (q->queue_depth)
		return q->queue_depth;

	return q->nr_requests;
}

L
Linus Torvalds 已提交
945 946 947
/*
 * q->prep_rq_fn return values
 */
948 949 950 951 952 953
enum {
	BLKPREP_OK,		/* serve it */
	BLKPREP_KILL,		/* fatal error, kill, return -EIO */
	BLKPREP_DEFER,		/* leave on queue */
	BLKPREP_INVALID,	/* invalid command, kill, return -EREMOTEIO */
};
L
Linus Torvalds 已提交
954 955 956 957 958 959 960 961 962 963

extern unsigned long blk_max_low_pfn, blk_max_pfn;

/*
 * standard bounce addresses:
 *
 * BLK_BOUNCE_HIGH	: bounce all highmem pages
 * BLK_BOUNCE_ANY	: don't bounce anything
 * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
 */
964 965

#if BITS_PER_LONG == 32
L
Linus Torvalds 已提交
966
#define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
967 968 969 970
#else
#define BLK_BOUNCE_HIGH		-1ULL
#endif
#define BLK_BOUNCE_ANY		(-1ULL)
971
#define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
L
Linus Torvalds 已提交
972

973 974 975 976
/*
 * default timeout for SG_IO if none specified
 */
#define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
L
Linus Torvalds 已提交
977
#define BLK_MIN_SG_TIMEOUT	(7 * HZ)
978

979 980 981 982
struct rq_map_data {
	struct page **pages;
	int page_order;
	int nr_entries;
983
	unsigned long offset;
984
	int null_mapped;
985
	int from_user;
986 987
};

988
struct req_iterator {
989
	struct bvec_iter iter;
990 991 992 993
	struct bio *bio;
};

/* This should not be used directly - use rq_for_each_segment */
994 995
#define for_each_bio(_bio)		\
	for (; _bio; _bio = _bio->bi_next)
996
#define __rq_for_each_bio(_bio, rq)	\
L
Linus Torvalds 已提交
997 998 999
	if ((rq->bio))			\
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

1000 1001
#define rq_for_each_segment(bvl, _rq, _iter)			\
	__rq_for_each_bio(_iter.bio, _rq)			\
1002
		bio_for_each_segment(bvl, _iter.bio, _iter.iter)
1003

K
Kent Overstreet 已提交
1004
#define rq_iter_last(bvec, _iter)				\
1005
		(_iter.bio->bi_next == NULL &&			\
K
Kent Overstreet 已提交
1006
		 bio_iter_last(bvec, _iter.iter))
1007

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
# error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
#endif
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
extern void rq_flush_dcache_pages(struct request *rq);
#else
static inline void rq_flush_dcache_pages(struct request *rq)
{
}
#endif

L
Linus Torvalds 已提交
1019 1020
extern int blk_register_queue(struct gendisk *disk);
extern void blk_unregister_queue(struct gendisk *disk);
1021
extern blk_qc_t generic_make_request(struct bio *bio);
1022
extern blk_qc_t direct_make_request(struct bio *bio);
1023
extern void blk_rq_init(struct request_queue *q, struct request *rq);
1024
extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
L
Linus Torvalds 已提交
1025
extern void blk_put_request(struct request *);
1026
extern void __blk_put_request(struct request_queue *, struct request *);
1027 1028
extern struct request *blk_get_request_flags(struct request_queue *,
					     unsigned int op,
1029
					     blk_mq_req_flags_t flags);
1030 1031
extern struct request *blk_get_request(struct request_queue *, unsigned int op,
				       gfp_t gfp_mask);
1032
extern void blk_requeue_request(struct request_queue *, struct request *);
1033
extern int blk_lld_busy(struct request_queue *q);
1034 1035 1036 1037 1038
extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
			     struct bio_set *bs, gfp_t gfp_mask,
			     int (*bio_ctr)(struct bio *, struct bio *, void *),
			     void *data);
extern void blk_rq_unprep_clone(struct request *rq);
1039
extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
1040
				     struct request *rq);
J
Jens Axboe 已提交
1041
extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
1042
extern void blk_delay_queue(struct request_queue *, unsigned long);
1043
extern void blk_queue_split(struct request_queue *, struct bio **);
1044
extern void blk_recount_segments(struct request_queue *, struct bio *);
1045
extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
1046 1047
extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
			      unsigned int, void __user *);
1048 1049
extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			  unsigned int, void __user *);
1050 1051
extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
			 struct scsi_ioctl_command __user *);
1052

1053
extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
1054
extern void blk_queue_exit(struct request_queue *q);
1055
extern void blk_start_queue(struct request_queue *q);
J
Jens Axboe 已提交
1056
extern void blk_start_queue_async(struct request_queue *q);
1057
extern void blk_stop_queue(struct request_queue *q);
L
Linus Torvalds 已提交
1058
extern void blk_sync_queue(struct request_queue *q);
1059
extern void __blk_stop_queue(struct request_queue *q);
1060
extern void __blk_run_queue(struct request_queue *q);
1061
extern void __blk_run_queue_uncond(struct request_queue *q);
1062
extern void blk_run_queue(struct request_queue *);
1063
extern void blk_run_queue_async(struct request_queue *q);
1064
extern int blk_rq_map_user(struct request_queue *, struct request *,
1065 1066
			   struct rq_map_data *, void __user *, unsigned long,
			   gfp_t);
J
Jens Axboe 已提交
1067
extern int blk_rq_unmap_user(struct bio *);
1068 1069
extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
1070 1071
			       struct rq_map_data *, const struct iov_iter *,
			       gfp_t);
1072
extern void blk_execute_rq(struct request_queue *, struct gendisk *,
1073
			  struct request *, int);
1074
extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
1075
				  struct request *, int, rq_end_io_fn *);
1076

1077 1078 1079
int blk_status_to_errno(blk_status_t status);
blk_status_t errno_to_blk_status(int errno);

1080
bool blk_poll(struct request_queue *q, blk_qc_t cookie);
J
Jens Axboe 已提交
1081

1082
static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
L
Linus Torvalds 已提交
1083
{
1084
	return bdev->bd_disk->queue;	/* this is never NULL */
L
Linus Torvalds 已提交
1085 1086
}

1087
/*
1088 1089 1090 1091 1092 1093
 * blk_rq_pos()			: the current sector
 * blk_rq_bytes()		: bytes left in the entire request
 * blk_rq_cur_bytes()		: bytes left in the current segment
 * blk_rq_err_bytes()		: bytes left till the next error boundary
 * blk_rq_sectors()		: sectors left in the entire request
 * blk_rq_cur_sectors()		: sectors left in the current segment
1094
 */
1095 1096
static inline sector_t blk_rq_pos(const struct request *rq)
{
1097
	return rq->__sector;
1098 1099 1100 1101
}

static inline unsigned int blk_rq_bytes(const struct request *rq)
{
1102
	return rq->__data_len;
1103 1104
}

1105 1106 1107 1108
static inline int blk_rq_cur_bytes(const struct request *rq)
{
	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
}
1109

1110 1111
extern unsigned int blk_rq_err_bytes(const struct request *rq);

1112 1113
static inline unsigned int blk_rq_sectors(const struct request *rq)
{
1114
	return blk_rq_bytes(rq) >> 9;
1115 1116 1117 1118
}

static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
{
1119
	return blk_rq_cur_bytes(rq) >> 9;
1120 1121
}

1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
static inline unsigned int blk_rq_zone_no(struct request *rq)
{
	return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
}

static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
{
	return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
}

1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
/*
 * Some commands like WRITE SAME have a payload or data transfer size which
 * is different from the size of the request.  Any driver that supports such
 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
 * calculate the data transfer size.
 */
static inline unsigned int blk_rq_payload_bytes(struct request *rq)
{
	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
		return rq->special_vec.bv_len;
	return blk_rq_bytes(rq);
}

1145
static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1146
						     int op)
1147
{
A
Adrian Hunter 已提交
1148
	if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
1149
		return min(q->limits.max_discard_sectors, UINT_MAX >> 9);
1150

1151
	if (unlikely(op == REQ_OP_WRITE_SAME))
1152 1153
		return q->limits.max_write_same_sectors;

1154 1155 1156
	if (unlikely(op == REQ_OP_WRITE_ZEROES))
		return q->limits.max_write_zeroes_sectors;

1157 1158 1159
	return q->limits.max_sectors;
}

1160 1161 1162 1163 1164 1165 1166 1167
/*
 * Return maximum size of a request at given offset. Only valid for
 * file system requests.
 */
static inline unsigned int blk_max_size_offset(struct request_queue *q,
					       sector_t offset)
{
	if (!q->limits.chunk_sectors)
1168
		return q->limits.max_sectors;
1169 1170 1171 1172 1173

	return q->limits.chunk_sectors -
			(offset & (q->limits.chunk_sectors - 1));
}

D
Damien Le Moal 已提交
1174 1175
static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
						  sector_t offset)
1176 1177 1178
{
	struct request_queue *q = rq->q;

1179
	if (blk_rq_is_passthrough(rq))
1180 1181
		return q->limits.max_hw_sectors;

A
Adrian Hunter 已提交
1182 1183 1184
	if (!q->limits.chunk_sectors ||
	    req_op(rq) == REQ_OP_DISCARD ||
	    req_op(rq) == REQ_OP_SECURE_ERASE)
1185
		return blk_queue_get_max_sectors(q, req_op(rq));
1186

D
Damien Le Moal 已提交
1187
	return min(blk_max_size_offset(q, offset),
1188
			blk_queue_get_max_sectors(q, req_op(rq)));
1189 1190
}

1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
static inline unsigned int blk_rq_count_bios(struct request *rq)
{
	unsigned int nr_bios = 0;
	struct bio *bio;

	__rq_for_each_bio(bio, rq)
		nr_bios++;

	return nr_bios;
}

1202 1203 1204 1205 1206 1207 1208
/*
 * Request issue related functions.
 */
extern struct request *blk_peek_request(struct request_queue *q);
extern void blk_start_request(struct request *rq);
extern struct request *blk_fetch_request(struct request_queue *q);

1209 1210
void blk_steal_bios(struct bio_list *list, struct request *rq);

L
Linus Torvalds 已提交
1211
/*
1212 1213 1214 1215 1216
 * Request completion related functions.
 *
 * blk_update_request() completes given number of bytes and updates
 * the request without completing it.
 *
1217 1218
 * blk_end_request() and friends.  __blk_end_request() must be called
 * with the request queue spinlock acquired.
L
Linus Torvalds 已提交
1219 1220
 *
 * Several drivers define their own end_request and call
1221 1222
 * blk_end_request() for parts of the original function.
 * This prevents code duplication in drivers.
L
Linus Torvalds 已提交
1223
 */
1224
extern bool blk_update_request(struct request *rq, blk_status_t error,
1225
			       unsigned int nr_bytes);
1226 1227
extern void blk_finish_request(struct request *rq, blk_status_t error);
extern bool blk_end_request(struct request *rq, blk_status_t error,
1228
			    unsigned int nr_bytes);
1229 1230
extern void blk_end_request_all(struct request *rq, blk_status_t error);
extern bool __blk_end_request(struct request *rq, blk_status_t error,
1231
			      unsigned int nr_bytes);
1232 1233
extern void __blk_end_request_all(struct request *rq, blk_status_t error);
extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1234

1235
extern void blk_complete_request(struct request *);
J
Jens Axboe 已提交
1236 1237
extern void __blk_complete_request(struct request *);
extern void blk_abort_request(struct request *);
1238
extern void blk_unprep_request(struct request *);
1239

L
Linus Torvalds 已提交
1240 1241 1242
/*
 * Access functions for manipulating queue properties
 */
1243
extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1244
					spinlock_t *lock, int node_id);
1245
extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
1246
extern int blk_init_allocated_queue(struct request_queue *);
1247 1248 1249
extern void blk_cleanup_queue(struct request_queue *);
extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
extern void blk_queue_bounce_limit(struct request_queue *, u64);
1250
extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1251
extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1252
extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1253 1254
extern void blk_queue_max_discard_segments(struct request_queue *,
		unsigned short);
1255
extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1256 1257
extern void blk_queue_max_discard_sectors(struct request_queue *q,
		unsigned int max_discard_sectors);
1258 1259
extern void blk_queue_max_write_same_sectors(struct request_queue *q,
		unsigned int max_write_same_sectors);
1260 1261
extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
		unsigned int max_write_same_sectors);
1262
extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
1263
extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1264 1265
extern void blk_queue_alignment_offset(struct request_queue *q,
				       unsigned int alignment);
1266
extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1267
extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1268
extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1269
extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1270
extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1271
extern void blk_set_default_limits(struct queue_limits *lim);
1272
extern void blk_set_stacking_limits(struct queue_limits *lim);
1273 1274
extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
			    sector_t offset);
1275 1276
extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
			    sector_t offset);
1277 1278
extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
			      sector_t offset);
1279
extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
1280
extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
1281
extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1282 1283 1284
extern int blk_queue_dma_drain(struct request_queue *q,
			       dma_drain_needed_fn *dma_drain_needed,
			       void *buf, unsigned int size);
1285
extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
1286
extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1287
extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1288
extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1289
extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1290
extern void blk_queue_dma_alignment(struct request_queue *, int);
1291
extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1292
extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
J
Jens Axboe 已提交
1293 1294
extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1295
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1296
extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
L
Linus Torvalds 已提交
1297

1298 1299 1300 1301 1302 1303 1304 1305 1306
/*
 * Number of physical segments as sent to the device.
 *
 * Normally this is the number of discontiguous data segments sent by the
 * submitter.  But for data-less command like discard we might have no
 * actual data segments submitted, but the driver might have to add it's
 * own special payload.  In that case we still return 1 here so that this
 * special payload will be mapped.
 */
1307 1308 1309 1310 1311 1312 1313
static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
{
	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
		return 1;
	return rq->nr_phys_segments;
}

1314 1315 1316 1317 1318 1319 1320 1321 1322
/*
 * Number of discard segments (or ranges) the driver needs to fill in.
 * Each discard bio merged into a request is counted as one segment.
 */
static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
{
	return max_t(unsigned short, rq->nr_phys_segments, 1);
}

1323
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
L
Linus Torvalds 已提交
1324 1325 1326
extern void blk_dump_rq_flags(struct request *, char *);
extern long nr_blockdev_pages(void);

T
Tejun Heo 已提交
1327
bool __must_check blk_get_queue(struct request_queue *);
1328
struct request_queue *blk_alloc_queue(gfp_t);
1329 1330
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
					   spinlock_t *lock);
1331
extern void blk_put_queue(struct request_queue *);
1332
extern void blk_set_queue_dying(struct request_queue *);
L
Linus Torvalds 已提交
1333

L
Lin Ming 已提交
1334 1335 1336
/*
 * block layer runtime pm functions
 */
1337
#ifdef CONFIG_PM
L
Lin Ming 已提交
1338 1339 1340 1341 1342
extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
extern int blk_pre_runtime_suspend(struct request_queue *q);
extern void blk_post_runtime_suspend(struct request_queue *q, int err);
extern void blk_pre_runtime_resume(struct request_queue *q);
extern void blk_post_runtime_resume(struct request_queue *q, int err);
1343
extern void blk_set_runtime_active(struct request_queue *q);
L
Lin Ming 已提交
1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
#else
static inline void blk_pm_runtime_init(struct request_queue *q,
	struct device *dev) {}
static inline int blk_pre_runtime_suspend(struct request_queue *q)
{
	return -ENOSYS;
}
static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
static inline void blk_pre_runtime_resume(struct request_queue *q) {}
static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1354
static inline void blk_set_runtime_active(struct request_queue *q) {}
L
Lin Ming 已提交
1355 1356
#endif

1357
/*
S
Suresh Jayaraman 已提交
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
 * blk_plug permits building a queue of related requests by holding the I/O
 * fragments for a short period. This allows merging of sequential requests
 * into single larger request. As the requests are moved from a per-task list to
 * the device's request_queue in a batch, this results in improved scalability
 * as the lock contention for request_queue lock is reduced.
 *
 * It is ok not to disable preemption when adding the request to the plug list
 * or when attempting a merge, because blk_schedule_flush_list() will only flush
 * the plug list when the task sleeps by itself. For details, please see
 * schedule() where blk_schedule_flush_plug() is called.
1368
 */
1369
struct blk_plug {
S
Suresh Jayaraman 已提交
1370
	struct list_head list; /* requests */
1371
	struct list_head mq_list; /* blk-mq requests */
S
Suresh Jayaraman 已提交
1372
	struct list_head cb_list; /* md requires an unplug callback */
1373
};
1374
#define BLK_MAX_REQUEST_COUNT 16
1375
#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1376

1377
struct blk_plug_cb;
1378
typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1379 1380
struct blk_plug_cb {
	struct list_head list;
1381 1382
	blk_plug_cb_fn callback;
	void *data;
1383
};
1384 1385
extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
					     void *data, int size);
1386 1387
extern void blk_start_plug(struct blk_plug *);
extern void blk_finish_plug(struct blk_plug *);
1388
extern void blk_flush_plug_list(struct blk_plug *, bool);
1389 1390 1391 1392 1393

static inline void blk_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1394 1395 1396 1397 1398 1399 1400 1401
	if (plug)
		blk_flush_plug_list(plug, false);
}

static inline void blk_schedule_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1402
	if (plug)
1403
		blk_flush_plug_list(plug, true);
1404 1405 1406 1407 1408 1409
}

static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	struct blk_plug *plug = tsk->plug;

1410 1411 1412 1413
	return plug &&
		(!list_empty(&plug->list) ||
		 !list_empty(&plug->mq_list) ||
		 !list_empty(&plug->cb_list));
1414 1415
}

L
Linus Torvalds 已提交
1416 1417 1418
/*
 * tag stuff
 */
1419 1420 1421
extern int blk_queue_start_tag(struct request_queue *, struct request *);
extern struct request *blk_queue_find_tag(struct request_queue *, int);
extern void blk_queue_end_tag(struct request_queue *, struct request *);
1422
extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
1423 1424 1425
extern void blk_queue_free_tags(struct request_queue *);
extern int blk_queue_resize_tags(struct request_queue *, int);
extern void blk_queue_invalidate_tags(struct request_queue *);
1426
extern struct blk_queue_tag *blk_init_tags(int, int);
1427
extern void blk_free_tags(struct blk_queue_tag *);
L
Linus Torvalds 已提交
1428

1429 1430 1431 1432 1433 1434 1435
static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
						int tag)
{
	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
		return NULL;
	return bqt->tag_index[tag];
}
1436

1437 1438 1439
extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1440 1441

#define BLKDEV_DISCARD_SECURE	(1 << 0)	/* issue a secure erase */
1442

1443 1444
extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1445
extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1446
		sector_t nr_sects, gfp_t gfp_mask, int flags,
1447
		struct bio **biop);
1448 1449

#define BLKDEV_ZERO_NOUNMAP	(1 << 0)  /* do not free blocks */
1450
#define BLKDEV_ZERO_NOFALLBACK	(1 << 1)  /* don't write explicit zeroes */
1451

1452 1453
extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1454
		unsigned flags);
1455
extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1456 1457
		sector_t nr_sects, gfp_t gfp_mask, unsigned flags);

1458 1459
static inline int sb_issue_discard(struct super_block *sb, sector_t block,
		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
D
David Woodhouse 已提交
1460
{
1461 1462 1463
	return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9),
				    nr_blocks << (sb->s_blocksize_bits - 9),
				    gfp_mask, flags);
D
David Woodhouse 已提交
1464
}
1465
static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1466
		sector_t nr_blocks, gfp_t gfp_mask)
1467 1468 1469 1470
{
	return blkdev_issue_zeroout(sb->s_bdev,
				    block << (sb->s_blocksize_bits - 9),
				    nr_blocks << (sb->s_blocksize_bits - 9),
1471
				    gfp_mask, 0);
1472
}
L
Linus Torvalds 已提交
1473

1474
extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1475

1476 1477 1478
enum blk_default_limits {
	BLK_MAX_SEGMENTS	= 128,
	BLK_SAFE_MAX_SECTORS	= 255,
1479
	BLK_DEF_MAX_SECTORS	= 2560,
1480 1481 1482
	BLK_MAX_SEGMENT_SIZE	= 65536,
	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
};
1483

L
Linus Torvalds 已提交
1484 1485
#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)

1486 1487
static inline unsigned long queue_segment_boundary(struct request_queue *q)
{
1488
	return q->limits.seg_boundary_mask;
1489 1490
}

1491 1492 1493 1494 1495
static inline unsigned long queue_virt_boundary(struct request_queue *q)
{
	return q->limits.virt_boundary_mask;
}

1496 1497
static inline unsigned int queue_max_sectors(struct request_queue *q)
{
1498
	return q->limits.max_sectors;
1499 1500 1501 1502
}

static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
{
1503
	return q->limits.max_hw_sectors;
1504 1505
}

1506
static inline unsigned short queue_max_segments(struct request_queue *q)
1507
{
1508
	return q->limits.max_segments;
1509 1510
}

1511 1512 1513 1514 1515
static inline unsigned short queue_max_discard_segments(struct request_queue *q)
{
	return q->limits.max_discard_segments;
}

1516 1517
static inline unsigned int queue_max_segment_size(struct request_queue *q)
{
1518
	return q->limits.max_segment_size;
1519 1520
}

1521
static inline unsigned short queue_logical_block_size(struct request_queue *q)
L
Linus Torvalds 已提交
1522 1523 1524
{
	int retval = 512;

1525 1526
	if (q && q->limits.logical_block_size)
		retval = q->limits.logical_block_size;
L
Linus Torvalds 已提交
1527 1528 1529 1530

	return retval;
}

1531
static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1532
{
1533
	return queue_logical_block_size(bdev_get_queue(bdev));
L
Linus Torvalds 已提交
1534 1535
}

1536 1537 1538 1539 1540
static inline unsigned int queue_physical_block_size(struct request_queue *q)
{
	return q->limits.physical_block_size;
}

1541
static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
M
Martin K. Petersen 已提交
1542 1543 1544 1545
{
	return queue_physical_block_size(bdev_get_queue(bdev));
}

1546 1547 1548 1549 1550
static inline unsigned int queue_io_min(struct request_queue *q)
{
	return q->limits.io_min;
}

M
Martin K. Petersen 已提交
1551 1552 1553 1554 1555
static inline int bdev_io_min(struct block_device *bdev)
{
	return queue_io_min(bdev_get_queue(bdev));
}

1556 1557 1558 1559 1560
static inline unsigned int queue_io_opt(struct request_queue *q)
{
	return q->limits.io_opt;
}

M
Martin K. Petersen 已提交
1561 1562 1563 1564 1565
static inline int bdev_io_opt(struct block_device *bdev)
{
	return queue_io_opt(bdev_get_queue(bdev));
}

1566 1567
static inline int queue_alignment_offset(struct request_queue *q)
{
M
Martin K. Petersen 已提交
1568
	if (q->limits.misaligned)
1569 1570
		return -1;

M
Martin K. Petersen 已提交
1571
	return q->limits.alignment_offset;
1572 1573
}

1574
static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1575 1576
{
	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1577
	unsigned int alignment = sector_div(sector, granularity >> 9) << 9;
1578

1579
	return (granularity + lim->alignment_offset - alignment) % granularity;
1580 1581
}

M
Martin K. Petersen 已提交
1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
static inline int bdev_alignment_offset(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q->limits.misaligned)
		return -1;

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->alignment_offset;

	return q->limits.alignment_offset;
}

1595 1596 1597 1598 1599 1600 1601 1602
static inline int queue_discard_alignment(struct request_queue *q)
{
	if (q->limits.discard_misaligned)
		return -1;

	return q->limits.discard_alignment;
}

1603
static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1604
{
1605
	unsigned int alignment, granularity, offset;
1606

1607 1608 1609
	if (!lim->max_discard_sectors)
		return 0;

1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
	/* Why are these in bytes, not sectors? */
	alignment = lim->discard_alignment >> 9;
	granularity = lim->discard_granularity >> 9;
	if (!granularity)
		return 0;

	/* Offset of the partition start in 'granularity' sectors */
	offset = sector_div(sector, granularity);

	/* And why do we do this modulus *again* in blkdev_issue_discard()? */
	offset = (granularity + alignment - offset) % granularity;

	/* Turn it back into bytes, gaah */
	return offset << 9;
1624 1625
}

1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
static inline int bdev_discard_alignment(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (bdev != bdev->bd_contains)
		return bdev->bd_part->discard_alignment;

	return q->limits.discard_alignment;
}

1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
static inline unsigned int bdev_write_same(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return q->limits.max_write_same_sectors;

	return 0;
}

1646 1647 1648 1649 1650 1651 1652 1653 1654 1655
static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return q->limits.max_write_zeroes_sectors;

	return 0;
}

D
Damien Le Moal 已提交
1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return blk_queue_zoned_model(q);

	return BLK_ZONED_NONE;
}

static inline bool bdev_is_zoned(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
		return blk_queue_is_zoned(q);

	return false;
}

1676
static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
1677 1678 1679 1680
{
	struct request_queue *q = bdev_get_queue(bdev);

	if (q)
1681
		return blk_queue_zone_sectors(q);
1682 1683
	return 0;
}
1684

1685 1686 1687
static inline unsigned int bdev_nr_zones(struct block_device *bdev)
{
	struct request_queue *q = bdev_get_queue(bdev);
1688

1689 1690
	if (q)
		return blk_queue_nr_zones(q);
1691 1692 1693
	return 0;
}

1694
static inline int queue_dma_alignment(struct request_queue *q)
L
Linus Torvalds 已提交
1695
{
1696
	return q ? q->dma_alignment : 511;
L
Linus Torvalds 已提交
1697 1698
}

1699
static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1700 1701 1702
				 unsigned int len)
{
	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1703
	return !(addr & alignment) && !(len & alignment);
1704 1705
}

L
Linus Torvalds 已提交
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
/* assumes size > 256 */
static inline unsigned int blksize_bits(unsigned int size)
{
	unsigned int bits = 8;
	do {
		bits++;
		size >>= 1;
	} while (size > 256);
	return bits;
}

1717
static inline unsigned int block_size(struct block_device *bdev)
L
Linus Torvalds 已提交
1718 1719 1720 1721
{
	return bdev->bd_block_size;
}

1722 1723
static inline bool queue_flush_queueable(struct request_queue *q)
{
J
Jens Axboe 已提交
1724
	return !test_bit(QUEUE_FLAG_FLUSH_NQ, &q->queue_flags);
1725 1726
}

L
Linus Torvalds 已提交
1727 1728 1729 1730 1731 1732
typedef struct {struct page *v;} Sector;

unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);

static inline void put_dev_sector(Sector p)
{
1733
	put_page(p.v);
L
Linus Torvalds 已提交
1734 1735
}

1736 1737 1738 1739 1740 1741 1742
static inline bool __bvec_gap_to_prev(struct request_queue *q,
				struct bio_vec *bprv, unsigned int offset)
{
	return offset ||
		((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
}

1743 1744 1745 1746 1747 1748 1749 1750 1751
/*
 * Check if adding a bio_vec after bprv with offset would create a gap in
 * the SG list. Most drivers don't care about this, but some do.
 */
static inline bool bvec_gap_to_prev(struct request_queue *q,
				struct bio_vec *bprv, unsigned int offset)
{
	if (!queue_virt_boundary(q))
		return false;
1752
	return __bvec_gap_to_prev(q, bprv, offset);
1753 1754
}

M
Ming Lei 已提交
1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
/*
 * Check if the two bvecs from two bios can be merged to one segment.
 * If yes, no need to check gap between the two bios since the 1st bio
 * and the 1st bvec in the 2nd bio can be handled in one segment.
 */
static inline bool bios_segs_mergeable(struct request_queue *q,
		struct bio *prev, struct bio_vec *prev_last_bv,
		struct bio_vec *next_first_bv)
{
	if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
		return false;
	if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
		return false;
	if (prev->bi_seg_back_size + next_first_bv->bv_len >
			queue_max_segment_size(q))
		return false;
	return true;
}

1774 1775 1776 1777
static inline bool bio_will_gap(struct request_queue *q,
				struct request *prev_rq,
				struct bio *prev,
				struct bio *next)
1778
{
1779 1780 1781
	if (bio_has_data(prev) && queue_virt_boundary(q)) {
		struct bio_vec pb, nb;

1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
		/*
		 * don't merge if the 1st bio starts with non-zero
		 * offset, otherwise it is quite difficult to respect
		 * sg gap limit. We work hard to merge a huge number of small
		 * single bios in case of mkfs.
		 */
		if (prev_rq)
			bio_get_first_bvec(prev_rq->bio, &pb);
		else
			bio_get_first_bvec(prev, &pb);
		if (pb.bv_offset)
			return true;

		/*
		 * We don't need to worry about the situation that the
		 * merged segment ends in unaligned virt boundary:
		 *
		 * - if 'pb' ends aligned, the merged segment ends aligned
		 * - if 'pb' ends unaligned, the next bio must include
		 *   one single bvec of 'nb', otherwise the 'nb' can't
		 *   merge with 'pb'
		 */
1804 1805
		bio_get_last_bvec(prev, &pb);
		bio_get_first_bvec(next, &nb);
1806

M
Ming Lei 已提交
1807 1808
		if (!bios_segs_mergeable(q, prev, &pb, &nb))
			return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1809 1810 1811
	}

	return false;
1812 1813 1814 1815
}

static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
{
1816
	return bio_will_gap(req->q, req, req->biotail, bio);
1817 1818 1819 1820
}

static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
{
1821
	return bio_will_gap(req->q, NULL, bio, req->bio);
1822 1823
}

1824
int kblockd_schedule_work(struct work_struct *work);
1825
int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1826
int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
L
Linus Torvalds 已提交
1827

1828
#ifdef CONFIG_BLK_CGROUP
1829 1830 1831 1832 1833
/*
 * This should not be using sched_clock(). A real patch is in progress
 * to fix this up, until that is in place we need to disable preemption
 * around sched_clock() in this function and set_io_start_time_ns().
 */
1834 1835
static inline void set_start_time_ns(struct request *req)
{
1836
	preempt_disable();
1837
	req->start_time_ns = sched_clock();
1838
	preempt_enable();
1839 1840 1841 1842
}

static inline void set_io_start_time_ns(struct request *req)
{
1843
	preempt_disable();
1844
	req->io_start_time_ns = sched_clock();
1845
	preempt_enable();
1846
}
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856

static inline uint64_t rq_start_time_ns(struct request *req)
{
        return req->start_time_ns;
}

static inline uint64_t rq_io_start_time_ns(struct request *req)
{
        return req->io_start_time_ns;
}
1857 1858 1859
#else
static inline void set_start_time_ns(struct request *req) {}
static inline void set_io_start_time_ns(struct request *req) {}
1860 1861 1862 1863 1864 1865 1866 1867
static inline uint64_t rq_start_time_ns(struct request *req)
{
	return 0;
}
static inline uint64_t rq_io_start_time_ns(struct request *req)
{
	return 0;
}
1868 1869
#endif

L
Linus Torvalds 已提交
1870 1871 1872 1873 1874
#define MODULE_ALIAS_BLOCKDEV(major,minor) \
	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
	MODULE_ALIAS("block-major-" __stringify(major) "-*")

1875 1876
#if defined(CONFIG_BLK_DEV_INTEGRITY)

1877 1878 1879
enum blk_integrity_flags {
	BLK_INTEGRITY_VERIFY		= 1 << 0,
	BLK_INTEGRITY_GENERATE		= 1 << 1,
1880
	BLK_INTEGRITY_DEVICE_CAPABLE	= 1 << 2,
1881
	BLK_INTEGRITY_IP_CHECKSUM	= 1 << 3,
1882
};
1883

1884
struct blk_integrity_iter {
1885 1886
	void			*prot_buf;
	void			*data_buf;
1887
	sector_t		seed;
1888
	unsigned int		data_size;
1889
	unsigned short		interval;
1890 1891 1892
	const char		*disk_name;
};

1893
typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1894

1895 1896 1897 1898 1899
struct blk_integrity_profile {
	integrity_processing_fn		*generate_fn;
	integrity_processing_fn		*verify_fn;
	const char			*name;
};
1900

1901
extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1902
extern void blk_integrity_unregister(struct gendisk *);
1903
extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1904 1905 1906
extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
				   struct scatterlist *);
extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1907 1908 1909 1910
extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
				   struct request *);
extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
				    struct bio *);
1911

1912
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1913
{
1914
	struct blk_integrity *bi = &disk->queue->integrity;
1915 1916 1917 1918 1919

	if (!bi->profile)
		return NULL;

	return bi;
1920 1921
}

1922 1923
static inline
struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1924
{
1925
	return blk_get_integrity(bdev->bd_disk);
1926 1927
}

1928
static inline bool blk_integrity_rq(struct request *rq)
1929
{
1930
	return rq->cmd_flags & REQ_INTEGRITY;
1931 1932
}

1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
	q->limits.max_integrity_segments = segs;
}

static inline unsigned short
queue_max_integrity_segments(struct request_queue *q)
{
	return q->limits.max_integrity_segments;
}

1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
static inline bool integrity_req_gap_back_merge(struct request *req,
						struct bio *next)
{
	struct bio_integrity_payload *bip = bio_integrity(req->bio);
	struct bio_integrity_payload *bip_next = bio_integrity(next);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}

static inline bool integrity_req_gap_front_merge(struct request *req,
						 struct bio *bio)
{
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}

1965 1966
#else /* CONFIG_BLK_DEV_INTEGRITY */

1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
struct bio;
struct block_device;
struct gendisk;
struct blk_integrity;

static inline int blk_integrity_rq(struct request *rq)
{
	return 0;
}
static inline int blk_rq_count_integrity_sg(struct request_queue *q,
					    struct bio *b)
{
	return 0;
}
static inline int blk_rq_map_integrity_sg(struct request_queue *q,
					  struct bio *b,
					  struct scatterlist *s)
{
	return 0;
}
static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
{
1989
	return NULL;
1990 1991 1992 1993 1994 1995 1996 1997 1998
}
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
	return NULL;
}
static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
{
	return 0;
}
1999
static inline void blk_integrity_register(struct gendisk *d,
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013
					 struct blk_integrity *b)
{
}
static inline void blk_integrity_unregister(struct gendisk *d)
{
}
static inline void blk_queue_max_integrity_segments(struct request_queue *q,
						    unsigned int segs)
{
}
static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
{
	return 0;
}
2014 2015 2016
static inline bool blk_integrity_merge_rq(struct request_queue *rq,
					  struct request *r1,
					  struct request *r2)
2017
{
2018
	return true;
2019
}
2020 2021 2022
static inline bool blk_integrity_merge_bio(struct request_queue *rq,
					   struct request *r,
					   struct bio *b)
2023
{
2024
	return true;
2025
}
2026

2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
static inline bool integrity_req_gap_back_merge(struct request *req,
						struct bio *next)
{
	return false;
}
static inline bool integrity_req_gap_front_merge(struct request *req,
						 struct bio *bio)
{
	return false;
}
2037 2038 2039

#endif /* CONFIG_BLK_DEV_INTEGRITY */

2040
struct block_device_operations {
A
Al Viro 已提交
2041
	int (*open) (struct block_device *, fmode_t);
2042
	void (*release) (struct gendisk *, fmode_t);
2043
	int (*rw_page)(struct block_device *, sector_t, struct page *, bool);
A
Al Viro 已提交
2044 2045
	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
2046 2047 2048
	unsigned int (*check_events) (struct gendisk *disk,
				      unsigned int clearing);
	/* ->media_changed() is DEPRECATED, use ->check_events() instead */
2049
	int (*media_changed) (struct gendisk *);
2050
	void (*unlock_native_capacity) (struct gendisk *);
2051 2052
	int (*revalidate_disk) (struct gendisk *);
	int (*getgeo)(struct block_device *, struct hd_geometry *);
2053 2054
	/* this callback is with swap_lock and sometimes page table lock held */
	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
2055
	struct module *owner;
2056
	const struct pr_ops *pr_ops;
2057 2058
};

2059 2060
extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
				 unsigned long);
2061 2062 2063
extern int bdev_read_page(struct block_device *, sector_t, struct page *);
extern int bdev_write_page(struct block_device *, sector_t, struct page *,
						struct writeback_control *);
2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117

#ifdef CONFIG_BLK_DEV_ZONED
bool blk_req_needs_zone_write_lock(struct request *rq);
void __blk_req_zone_write_lock(struct request *rq);
void __blk_req_zone_write_unlock(struct request *rq);

static inline void blk_req_zone_write_lock(struct request *rq)
{
	if (blk_req_needs_zone_write_lock(rq))
		__blk_req_zone_write_lock(rq);
}

static inline void blk_req_zone_write_unlock(struct request *rq)
{
	if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
		__blk_req_zone_write_unlock(rq);
}

static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
	return rq->q->seq_zones_wlock &&
		test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
}

static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
	if (!blk_req_needs_zone_write_lock(rq))
		return true;
	return !blk_req_zone_is_write_locked(rq);
}
#else
static inline bool blk_req_needs_zone_write_lock(struct request *rq)
{
	return false;
}

static inline void blk_req_zone_write_lock(struct request *rq)
{
}

static inline void blk_req_zone_write_unlock(struct request *rq)
{
}
static inline bool blk_req_zone_is_write_locked(struct request *rq)
{
	return false;
}

static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
{
	return true;
}
#endif /* CONFIG_BLK_DEV_ZONED */

2118
#else /* CONFIG_BLOCK */
2119 2120 2121

struct block_device;

2122 2123 2124 2125 2126 2127 2128 2129 2130 2131
/*
 * stubs for when the block layer is configured out
 */
#define buffer_heads_over_limit 0

static inline long nr_blockdev_pages(void)
{
	return 0;
}

2132 2133 2134 2135
struct blk_plug {
};

static inline void blk_start_plug(struct blk_plug *plug)
2136 2137 2138
{
}

2139
static inline void blk_finish_plug(struct blk_plug *plug)
2140 2141 2142
{
}

2143
static inline void blk_flush_plug(struct task_struct *task)
2144 2145 2146
{
}

2147 2148 2149 2150 2151
static inline void blk_schedule_flush_plug(struct task_struct *task)
{
}


2152 2153 2154 2155 2156
static inline bool blk_needs_flush_plug(struct task_struct *tsk)
{
	return false;
}

2157 2158 2159 2160 2161 2162
static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
				     sector_t *error_sector)
{
	return 0;
}

2163 2164
#endif /* CONFIG_BLOCK */

L
Linus Torvalds 已提交
2165
#endif