blk-mq.h 5.1 KB
Newer Older
1 2 3
#ifndef INT_BLK_MQ_H
#define INT_BLK_MQ_H

4 5
#include "blk-stat.h"

6 7
struct blk_mq_tag_set;

8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
struct blk_mq_ctx {
	struct {
		spinlock_t		lock;
		struct list_head	rq_list;
	}  ____cacheline_aligned_in_smp;

	unsigned int		cpu;
	unsigned int		index_hw;

	/* incremented at dispatch time */
	unsigned long		rq_dispatched[2];
	unsigned long		rq_merged;

	/* incremented at completion time */
	unsigned long		____cacheline_aligned_in_smp rq_completed[2];
23
	struct blk_rq_stat	stat[2];
24 25 26

	struct request_queue	*queue;
	struct kobject		kobj;
27
} ____cacheline_aligned_in_smp;
28 29

void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
30
void blk_mq_freeze_queue(struct request_queue *q);
31
void blk_mq_free_queue(struct request_queue *q);
32
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
33
void blk_mq_wake_waiters(struct request_queue *q);
34
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *, struct list_head *);
35
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
36
bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx);
37 38
bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
				bool wait);
39 40 41 42

/*
 * Internal helpers for allocating/freeing the request map
 */
43 44 45 46 47 48 49 50 51
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx);
void blk_mq_free_rq_map(struct blk_mq_tags *tags);
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags);
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth);
52 53 54 55 56 57

/*
 * Internal helpers for request insertion into sw queues
 */
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
				bool at_head);
58 59
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
				struct list_head *list);
60 61 62
/*
 * CPU hotplug helpers
 */
63 64
void blk_mq_enable_hotplug(void);
void blk_mq_disable_hotplug(void);
65 66 67 68

/*
 * CPU -> queue mappings
 */
69
extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
70

C
Christoph Hellwig 已提交
71 72 73 74 75 76
static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
		int cpu)
{
	return q->queue_hw_ctx[q->mq_map[cpu]];
}

77 78 79 80 81
/*
 * sysfs helpers
 */
extern int blk_mq_sysfs_register(struct request_queue *q);
extern void blk_mq_sysfs_unregister(struct request_queue *q);
K
Keith Busch 已提交
82
extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
83

84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
/*
 * debugfs helpers
 */
#ifdef CONFIG_DEBUG_FS
void blk_mq_debugfs_init(void);
int blk_mq_debugfs_register(struct request_queue *q, const char *name);
void blk_mq_debugfs_unregister(struct request_queue *q);
int blk_mq_debugfs_register_hctxs(struct request_queue *q);
void blk_mq_debugfs_unregister_hctxs(struct request_queue *q);
#else
static inline void blk_mq_debugfs_init(void)
{
}

int blk_mq_debugfs_register(struct request_queue *q, const char *name);
{
	return 0;
}

void blk_mq_debugfs_unregister(struct request_queue *q)
{
}

int blk_mq_debugfs_register_hctxs(struct request_queue *q)
{
	return 0;
}

void blk_mq_debugfs_unregister_hctxs(struct request_queue *q)
{
}
#endif

117 118
extern void blk_mq_rq_timed_out(struct request *req, bool reserved);

119 120
void blk_mq_release(struct request_queue *q);

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
					   unsigned int cpu)
{
	return per_cpu_ptr(q->queue_ctx, cpu);
}

/*
 * This assumes per-cpu software queueing queues. They could be per-node
 * as well, for instance. For now this is hardcoded as-is. Note that we don't
 * care about preemption, since we know the ctx's are persistent. This does
 * mean that we can't rely on ctx always matching the currently running CPU.
 */
static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
{
	return __blk_mq_get_ctx(q, get_cpu());
}

static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
{
	put_cpu();
}

143 144 145
struct blk_mq_alloc_data {
	/* input parameter */
	struct request_queue *q;
146
	unsigned int flags;
147 148 149 150 151 152 153

	/* input & output parameter */
	struct blk_mq_ctx *ctx;
	struct blk_mq_hw_ctx *hctx;
};

static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
154 155
		struct request_queue *q, unsigned int flags,
		struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx)
156 157
{
	data->q = q;
158
	data->flags = flags;
159 160 161 162
	data->ctx = ctx;
	data->hctx = hctx;
}

163 164
static inline struct blk_mq_tags *blk_mq_tags_from_data(struct blk_mq_alloc_data *data)
{
165 166 167
	if (data->flags & BLK_MQ_REQ_INTERNAL)
		return data->hctx->sched_tags;

168 169 170
	return data->hctx->tags;
}

171 172 173 174 175
/*
 * Internal helpers for request allocation/init/free
 */
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			struct request *rq, unsigned int op);
176
void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
177
				struct request *rq);
178
void blk_mq_finish_request(struct request *rq);
179 180 181
struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
					unsigned int op);

182 183 184 185 186
static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
{
	return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
}

187 188 189 190 191
static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
{
	return hctx->nr_ctx && hctx->tags;
}

192
#endif