blk.h 13.3 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4
#ifndef BLK_INTERNAL_H
#define BLK_INTERNAL_H

5
#include <linux/idr.h>
6
#include <linux/blk-mq.h>
7
#include <linux/part_stat.h>
8
#include <linux/blk-crypto.h>
9
#include <xen/xen.h>
10
#include "blk-crypto-internal.h"
11
#include "blk-mq.h"
12
#include "blk-mq-sched.h"
13

14 15 16
/* Max future timer expiry for timeouts */
#define BLK_MAX_TIMEOUT		(5 * HZ)

17 18
extern struct dentry *blk_debugfs_root;

19 20 21
struct blk_flush_queue {
	unsigned int		flush_pending_idx:1;
	unsigned int		flush_running_idx:1;
22
	blk_status_t 		rq_status;
23 24 25 26
	unsigned long		flush_pending_since;
	struct list_head	flush_queue[2];
	struct list_head	flush_data_in_flight;
	struct request		*flush_rq;
27

28
	struct lock_class_key	key;
29 30 31
	spinlock_t		mq_flush_lock;
};

32 33
extern struct kmem_cache *blk_requestq_cachep;
extern struct kobj_type blk_queue_ktype;
34
extern struct ida blk_queue_ida;
35

36 37
static inline struct blk_flush_queue *
blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx)
38
{
39
	return blk_mq_map_queue(q, REQ_OP_FLUSH, ctx)->fq;
40 41
}

42 43 44 45 46
static inline void __blk_get_queue(struct request_queue *q)
{
	kobject_get(&q->kobj);
}

47 48 49 50 51 52
static inline bool
is_flush_rq(struct request *req, struct blk_mq_hw_ctx *hctx)
{
	return hctx->fq->flush_rq == req;
}

53 54
struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
					      gfp_t flags);
55
void blk_free_flush_queue(struct blk_flush_queue *q);
56

57 58
void blk_freeze_queue(struct request_queue *q);

59 60
static inline bool biovec_phys_mergeable(struct request_queue *q,
		struct bio_vec *vec1, struct bio_vec *vec2)
61
{
62
	unsigned long mask = queue_segment_boundary(q);
63 64
	phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset;
	phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset;
65 66

	if (addr1 + vec1->bv_len != addr2)
67
		return false;
68
	if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
69
		return false;
70 71
	if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
		return false;
72 73 74
	return true;
}

75 76 77
static inline bool __bvec_gap_to_prev(struct request_queue *q,
		struct bio_vec *bprv, unsigned int offset)
{
78
	return (offset & queue_virt_boundary(q)) ||
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
		((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
}

/*
 * Check if adding a bio_vec after bprv with offset would create a gap in
 * the SG list. Most drivers don't care about this, but some do.
 */
static inline bool bvec_gap_to_prev(struct request_queue *q,
		struct bio_vec *bprv, unsigned int offset)
{
	if (!queue_virt_boundary(q))
		return false;
	return __bvec_gap_to_prev(q, bprv, offset);
}

94 95 96 97 98 99 100 101 102 103 104 105
static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
		unsigned int nr_segs)
{
	rq->nr_phys_segments = nr_segs;
	rq->__data_len = bio->bi_iter.bi_size;
	rq->bio = rq->biotail = bio;
	rq->ioprio = bio_prio(bio);

	if (bio->bi_disk)
		rq->rq_disk = bio->bi_disk;
}

106 107
#ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void);
108
bool __bio_integrity_endio(struct bio *);
109
void bio_integrity_free(struct bio *bio);
110 111 112 113 114 115
static inline bool bio_integrity_endio(struct bio *bio)
{
	if (bio_integrity(bio))
		return __bio_integrity_endio(bio);
	return true;
}
116

117 118
bool blk_integrity_merge_rq(struct request_queue *, struct request *,
		struct request *);
119 120
bool blk_integrity_merge_bio(struct request_queue *, struct request *,
		struct bio *);
121

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
static inline bool integrity_req_gap_back_merge(struct request *req,
		struct bio *next)
{
	struct bio_integrity_payload *bip = bio_integrity(req->bio);
	struct bio_integrity_payload *bip_next = bio_integrity(next);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}

static inline bool integrity_req_gap_front_merge(struct request *req,
		struct bio *bio)
{
	struct bio_integrity_payload *bip = bio_integrity(bio);
	struct bio_integrity_payload *bip_next = bio_integrity(req->bio);

	return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
				bip_next->bip_vec[0].bv_offset);
}
141 142 143

void blk_integrity_add(struct gendisk *);
void blk_integrity_del(struct gendisk *);
144
#else /* CONFIG_BLK_DEV_INTEGRITY */
145 146 147 148 149
static inline bool blk_integrity_merge_rq(struct request_queue *rq,
		struct request *r1, struct request *r2)
{
	return true;
}
150 151 152 153 154
static inline bool blk_integrity_merge_bio(struct request_queue *rq,
		struct request *r, struct bio *b)
{
	return true;
}
155 156 157 158 159 160 161 162 163 164 165
static inline bool integrity_req_gap_back_merge(struct request *req,
		struct bio *next)
{
	return false;
}
static inline bool integrity_req_gap_front_merge(struct request *req,
		struct bio *bio)
{
	return false;
}

166 167 168
static inline void blk_flush_integrity(void)
{
}
169 170 171 172
static inline bool bio_integrity_endio(struct bio *bio)
{
	return true;
}
173 174 175
static inline void bio_integrity_free(struct bio *bio)
{
}
176 177 178 179 180 181
static inline void blk_integrity_add(struct gendisk *disk)
{
}
static inline void blk_integrity_del(struct gendisk *disk)
{
}
182
#endif /* CONFIG_BLK_DEV_INTEGRITY */
183

184
unsigned long blk_rq_timeout(unsigned long timeout);
185
void blk_add_timer(struct request *req);
186 187

bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
188
		unsigned int nr_segs, struct request **same_queue_rq);
189 190
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
			struct bio *bio, unsigned int nr_segs);
191

192
void blk_account_io_start(struct request *req);
193
void blk_account_io_done(struct request *req, u64 now);
194

195 196 197
/*
 * Internal elevator interface
 */
198
#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
199

200
void blk_insert_flush(struct request *rq);
201

202
void elevator_init_mq(struct request_queue *q);
203 204
int elevator_switch_mq(struct request_queue *q,
			      struct elevator_type *new_e);
205
void __elevator_exit(struct request_queue *, struct elevator_queue *);
206
int elv_register_queue(struct request_queue *q, bool uevent);
207 208
void elv_unregister_queue(struct request_queue *q);

209 210 211
static inline void elevator_exit(struct request_queue *q,
		struct elevator_queue *e)
{
212 213
	lockdep_assert_held(&q->sysfs_lock);

214 215 216 217
	blk_mq_sched_free_requests(q);
	__elevator_exit(q, e);
}

218 219
struct hd_struct *__disk_get_part(struct gendisk *disk, int partno);

220 221 222 223 224 225 226 227 228 229
ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
		char *buf);
ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
		char *buf);
ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,
		char *buf);
ssize_t part_fail_show(struct device *dev, struct device_attribute *attr,
		char *buf);
ssize_t part_fail_store(struct device *dev, struct device_attribute *attr,
		const char *buf, size_t count);
230 231 232 233
ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *,
				const char *, size_t);

234
void __blk_queue_split(struct bio **bio, unsigned int *nr_segs);
235 236
int ll_back_merge_fn(struct request *req, struct bio *bio,
		unsigned int nr_segs);
237 238
int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
				struct request *next);
239
unsigned int blk_recalc_rq_segments(struct request *rq);
240
void blk_rq_set_mixed_merge(struct request *rq);
241
bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
242
enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
243

244 245
int blk_dev_init(void);

246 247 248 249
/*
 * Contribute to IO statistics IFF:
 *
 *	a) it's attached to a gendisk, and
250
 *	b) the queue had IO stats enabled when this request was started
251
 */
252
static inline bool blk_do_io_stat(struct request *rq)
253
{
254
	return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT);
255 256
}

257 258 259 260 261 262 263
static inline void req_set_nomerge(struct request_queue *q, struct request *req)
{
	req->cmd_flags |= REQ_NOMERGE;
	if (req == q->last_merge)
		q->last_merge = NULL;
}

264 265 266 267 268 269 270 271 272 273
/*
 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size
 * is defined as 'unsigned int', meantime it has to aligned to with logical
 * block size which is the minimum accepted unit by hardware.
 */
static inline unsigned int bio_allowed_max_sectors(struct request_queue *q)
{
	return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9;
}

274 275 276 277 278 279 280 281 282 283 284 285 286 287
/*
 * The max bio size which is aligned to q->limits.discard_granularity. This
 * is a hint to split large discard bio in generic block layer, then if device
 * driver needs to split the discard bio into smaller ones, their bi_size can
 * be very probably and easily aligned to discard_granularity of the device's
 * queue.
 */
static inline unsigned int bio_aligned_discard_max_sectors(
					struct request_queue *q)
{
	return round_down(UINT_MAX, q->limits.discard_granularity) >>
			SECTOR_SHIFT;
}

288 289 290 291
/*
 * Internal io_context interface
 */
void get_io_context(struct io_context *ioc);
292
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
293 294
struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
			     gfp_t gfp_mask);
295
void ioc_clear_queue(struct request_queue *q);
296

297
int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node);
298 299 300 301

/*
 * Internal throttling interface
 */
302 303 304
#ifdef CONFIG_BLK_DEV_THROTTLING
extern int blk_throtl_init(struct request_queue *q);
extern void blk_throtl_exit(struct request_queue *q);
305
extern void blk_throtl_register_queue(struct request_queue *q);
306
bool blk_throtl_bio(struct bio *bio);
307 308 309
#else /* CONFIG_BLK_DEV_THROTTLING */
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
static inline void blk_throtl_exit(struct request_queue *q) { }
310
static inline void blk_throtl_register_queue(struct request_queue *q) { }
311
static inline bool blk_throtl_bio(struct bio *bio) { return false; }
312
#endif /* CONFIG_BLK_DEV_THROTTLING */
313 314 315 316
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page);
extern ssize_t blk_throtl_sample_time_store(struct request_queue *q,
	const char *page, size_t count);
317
extern void blk_throtl_bio_endio(struct bio *bio);
318
extern void blk_throtl_stat_add(struct request *rq, u64 time);
319 320
#else
static inline void blk_throtl_bio_endio(struct bio *bio) { }
321
static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
322
#endif
323

324 325 326 327 328 329 330 331 332 333 334 335 336
#ifdef CONFIG_BOUNCE
extern int init_emergency_isa_pool(void);
extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
#else
static inline int init_emergency_isa_pool(void)
{
	return 0;
}
static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
{
}
#endif /* CONFIG_BOUNCE */

337 338 339 340 341 342
#ifdef CONFIG_BLK_CGROUP_IOLATENCY
extern int blk_iolatency_init(struct request_queue *q);
#else
static inline int blk_iolatency_init(struct request_queue *q) { return 0; }
#endif

343 344
struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp);

345 346 347 348 349 350
#ifdef CONFIG_BLK_DEV_ZONED
void blk_queue_free_zone_bitmaps(struct request_queue *q);
#else
static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {}
#endif

351 352 353 354 355 356 357 358 359
struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector);

int blk_alloc_devt(struct hd_struct *part, dev_t *devt);
void blk_free_devt(dev_t devt);
void blk_invalidate_devt(dev_t devt);
char *disk_name(struct gendisk *hd, int partno, char *buf);
#define ADDPART_FLAG_NONE	0
#define ADDPART_FLAG_RAID	1
#define ADDPART_FLAG_WHOLEDISK	2
360
void delete_partition(struct hd_struct *part);
361 362 363 364 365
int bdev_add_partition(struct block_device *bdev, int partno,
		sector_t start, sector_t length);
int bdev_del_partition(struct block_device *bdev, int partno);
int bdev_resize_partition(struct block_device *bdev, int partno,
		sector_t start, sector_t length);
366
int disk_expand_part_tbl(struct gendisk *disk, int target);
367
int hd_ref_init(struct hd_struct *part);
368

369
/* no need to get/put refcount of part0 */
370 371
static inline int hd_struct_try_get(struct hd_struct *part)
{
372 373 374
	if (part->partno)
		return percpu_ref_tryget_live(&part->ref);
	return 1;
375 376 377 378
}

static inline void hd_struct_put(struct hd_struct *part)
{
379 380
	if (part->partno)
		percpu_ref_put(&part->ref);
381 382 383 384
}

static inline void hd_free_part(struct hd_struct *part)
{
385
	free_percpu(part->dkstats);
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
	kfree(part->info);
	percpu_ref_exit(&part->ref);
}

/*
 * Any access of part->nr_sects which is not protected by partition
 * bd_mutex or gendisk bdev bd_mutex, should be done using this
 * accessor function.
 *
 * Code written along the lines of i_size_read() and i_size_write().
 * CONFIG_PREEMPTION case optimizes the case of UP kernel with preemption
 * on.
 */
static inline sector_t part_nr_sects_read(struct hd_struct *part)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
	sector_t nr_sects;
	unsigned seq;
	do {
		seq = read_seqcount_begin(&part->nr_sects_seq);
		nr_sects = part->nr_sects;
	} while (read_seqcount_retry(&part->nr_sects_seq, seq));
	return nr_sects;
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
	sector_t nr_sects;

	preempt_disable();
	nr_sects = part->nr_sects;
	preempt_enable();
	return nr_sects;
#else
	return part->nr_sects;
#endif
}

/*
 * Should be called with mutex lock held (typically bd_mutex) of partition
 * to provide mutual exlusion among writers otherwise seqcount might be
 * left in wrong state leaving the readers spinning infinitely.
 */
static inline void part_nr_sects_write(struct hd_struct *part, sector_t size)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
429
	preempt_disable();
430 431 432
	write_seqcount_begin(&part->nr_sects_seq);
	part->nr_sects = size;
	write_seqcount_end(&part->nr_sects_seq);
433
	preempt_enable();
434 435 436 437 438 439 440 441 442
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
	preempt_disable();
	part->nr_sects = size;
	preempt_enable();
#else
	part->nr_sects = size;
#endif
}

443
int bio_add_hw_page(struct request_queue *q, struct bio *bio,
444
		struct page *page, unsigned int len, unsigned int offset,
445
		unsigned int max_sectors, bool *same_page);
446

447
#endif /* BLK_INTERNAL_H */
新手
引导
客服 返回
顶部