blk.h 7.6 KB
Newer Older
1 2 3
#ifndef BLK_INTERNAL_H
#define BLK_INTERNAL_H

4 5
#include <linux/idr.h>

J
Jens Axboe 已提交
6 7 8 9 10 11
/* Amount of time in which a process may batch requests */
#define BLK_BATCH_TIME	(HZ/50UL)

/* Number of requests a "batching" process may submit */
#define BLK_BATCH_REQ	32

12 13
extern struct kmem_cache *blk_requestq_cachep;
extern struct kobj_type blk_queue_ktype;
14
extern struct ida blk_queue_ida;
15

T
Tejun Heo 已提交
16 17 18 19 20
static inline void __blk_get_queue(struct request_queue *q)
{
	kobject_get(&q->kobj);
}

J
Jens Axboe 已提交
21 22 23
void init_request_from_bio(struct request *req, struct bio *bio);
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
			struct bio *bio);
24 25
int blk_rq_append_bio(struct request_queue *q, struct request *rq,
		      struct bio *bio);
26 27
void blk_queue_bypass_start(struct request_queue *q);
void blk_queue_bypass_end(struct request_queue *q);
28
void blk_dequeue_request(struct request *rq);
29
void __blk_queue_free_tags(struct request_queue *q);
30 31
bool __blk_end_bidi_request(struct request *rq, int error,
			    unsigned int nr_bytes, unsigned int bidi_bytes);
32

J
Jens Axboe 已提交
33 34 35
void blk_rq_timed_out_timer(unsigned long data);
void blk_delete_timer(struct request *);
void blk_add_timer(struct request *);
36
void __generic_unplug_device(struct request_queue *);
J
Jens Axboe 已提交
37 38 39 40 41 42 43 44 45 46

/*
 * Internal atomic flags for request handling
 */
enum rq_atomic_flags {
	REQ_ATOM_COMPLETE = 0,
};

/*
 * EH timer and IO completion will both attempt to 'grab' the request, make
L
Lucas De Marchi 已提交
47
 * sure that only one of them succeeds
J
Jens Axboe 已提交
48 49 50 51 52 53 54 55 56 57
 */
static inline int blk_mark_rq_complete(struct request *rq)
{
	return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
}

static inline void blk_clear_rq_complete(struct request *rq)
{
	clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
}
J
Jens Axboe 已提交
58

59 60 61 62 63
/*
 * Internal elevator interface
 */
#define ELV_ON_HASH(rq)		(!hlist_unhashed(&(rq)->hash))

64 65
void blk_insert_flush(struct request *rq);
void blk_abort_flushes(struct request_queue *q);
T
Tejun Heo 已提交
66

67 68 69 70 71
static inline struct request *__elv_next_request(struct request_queue *q)
{
	struct request *rq;

	while (1) {
72
		if (!list_empty(&q->queue_head)) {
73
			rq = list_entry_rq(q->queue_head.next);
74
			return rq;
75 76
		}

77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
		/*
		 * Flush request is running and flush request isn't queueable
		 * in the drive, we can hold the queue till flush request is
		 * finished. Even we don't do this, driver can't dispatch next
		 * requests and will requeue them. And this can improve
		 * throughput too. For example, we have request flush1, write1,
		 * flush 2. flush1 is dispatched, then queue is hold, write1
		 * isn't inserted to queue. After flush1 is finished, flush2
		 * will be dispatched. Since disk cache is already clean,
		 * flush2 will be finished very soon, so looks like flush2 is
		 * folded to flush1.
		 * Since the queue is hold, a flag is set to indicate the queue
		 * should be restarted later. Please see flush_end_io() for
		 * details.
		 */
		if (q->flush_pending_idx != q->flush_running_idx &&
				!queue_flush_queueable(q)) {
			q->flush_queue_delayed = 1;
			return NULL;
		}
T
Tejun Heo 已提交
97
		if (unlikely(blk_queue_dead(q)) ||
T
Tejun Heo 已提交
98
		    !q->elevator->type->ops.elevator_dispatch_fn(q, 0))
99 100 101 102 103 104 105 106
			return NULL;
	}
}

static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

T
Tejun Heo 已提交
107 108
	if (e->type->ops.elevator_activate_req_fn)
		e->type->ops.elevator_activate_req_fn(q, rq);
109 110 111 112 113 114
}

static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

T
Tejun Heo 已提交
115 116
	if (e->type->ops.elevator_deactivate_req_fn)
		e->type->ops.elevator_deactivate_req_fn(q, rq);
117 118
}

119 120 121 122 123 124 125 126 127 128 129 130
#ifdef CONFIG_FAIL_IO_TIMEOUT
int blk_should_fake_timeout(struct request_queue *);
ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *,
				const char *, size_t);
#else
static inline int blk_should_fake_timeout(struct request_queue *q)
{
	return 0;
}
#endif

131 132 133 134 135 136
int ll_back_merge_fn(struct request_queue *q, struct request *req,
		     struct bio *bio);
int ll_front_merge_fn(struct request_queue *q, struct request *req, 
		      struct bio *bio);
int attempt_back_merge(struct request_queue *q, struct request *rq);
int attempt_front_merge(struct request_queue *q, struct request *rq);
137 138
int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
				struct request *next);
139
void blk_recalc_rq_segments(struct request *rq);
140
void blk_rq_set_mixed_merge(struct request *rq);
141 142
bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
int blk_try_merge(struct request *rq, struct bio *bio);
143

144 145
void blk_queue_congestion_threshold(struct request_queue *q);

146 147
int blk_dev_init(void);

148

149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
/*
 * Return the threshold (number of used requests) at which the queue is
 * considered to be congested.  It include a little hysteresis to keep the
 * context switch rate down.
 */
static inline int queue_congestion_on_threshold(struct request_queue *q)
{
	return q->nr_congestion_on;
}

/*
 * The threshold at which a queue is considered to be uncongested
 */
static inline int queue_congestion_off_threshold(struct request_queue *q)
{
	return q->nr_congestion_off;
}

167 168
static inline int blk_cpu_to_group(int cpu)
{
169
	int group = NR_CPUS;
170
#ifdef CONFIG_SCHED_MC
171
	const struct cpumask *mask = cpu_coregroup_mask(cpu);
172
	group = cpumask_first(mask);
173
#elif defined(CONFIG_SCHED_SMT)
174
	group = cpumask_first(topology_thread_cpumask(cpu));
175 176 177
#else
	return cpu;
#endif
178 179 180
	if (likely(group < NR_CPUS))
		return group;
	return cpu;
181 182
}

183 184 185 186 187
/*
 * Contribute to IO statistics IFF:
 *
 *	a) it's attached to a gendisk, and
 *	b) the queue had IO stats enabled when this request was started, and
K
Kiyoshi Ueda 已提交
188
 *	c) it's a file system request or a discard request
189
 */
190
static inline int blk_do_io_stat(struct request *rq)
191
{
192 193 194 195
	return rq->rq_disk &&
	       (rq->cmd_flags & REQ_IO_STAT) &&
	       (rq->cmd_type == REQ_TYPE_FS ||
	        (rq->cmd_flags & REQ_DISCARD));
196 197
}

198 199 200 201
/*
 * Internal io_context interface
 */
void get_io_context(struct io_context *ioc);
202
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
203
struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask);
204
void ioc_clear_queue(struct request_queue *q);
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233

void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_mask,
				int node);

/**
 * create_io_context - try to create task->io_context
 * @task: target task
 * @gfp_mask: allocation mask
 * @node: allocation node
 *
 * If @task->io_context is %NULL, allocate a new io_context and install it.
 * Returns the current @task->io_context which may be %NULL if allocation
 * failed.
 *
 * Note that this function can't be called with IRQ disabled because
 * task_lock which protects @task->io_context is IRQ-unsafe.
 */
static inline struct io_context *create_io_context(struct task_struct *task,
						   gfp_t gfp_mask, int node)
{
	WARN_ON_ONCE(irqs_disabled());
	if (unlikely(!task->io_context))
		create_io_context_slowpath(task, gfp_mask, node);
	return task->io_context;
}

/*
 * Internal throttling interface
 */
234
#ifdef CONFIG_BLK_DEV_THROTTLING
235
extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio);
236
extern void blk_throtl_drain(struct request_queue *q);
237 238
extern int blk_throtl_init(struct request_queue *q);
extern void blk_throtl_exit(struct request_queue *q);
239
extern void blk_throtl_release(struct request_queue *q);
240
#else /* CONFIG_BLK_DEV_THROTTLING */
241
static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
242
{
243
	return false;
244
}
245
static inline void blk_throtl_drain(struct request_queue *q) { }
246 247
static inline int blk_throtl_init(struct request_queue *q) { return 0; }
static inline void blk_throtl_exit(struct request_queue *q) { }
248
static inline void blk_throtl_release(struct request_queue *q) { }
249 250 251
#endif /* CONFIG_BLK_DEV_THROTTLING */

#endif /* BLK_INTERNAL_H */