blk-cgroup.h 21.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
#ifndef _BLK_CGROUP_H
#define _BLK_CGROUP_H
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
 */

#include <linux/cgroup.h>
T
Tejun Heo 已提交
17
#include <linux/percpu_counter.h>
18
#include <linux/seq_file.h>
19
#include <linux/radix-tree.h>
20
#include <linux/blkdev.h>
21
#include <linux/atomic.h>
22
#include <linux/kthread.h>
23

T
Tejun Heo 已提交
24 25 26
/* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
#define BLKG_STAT_CPU_BATCH	(INT_MAX / 2)

27 28 29
/* Max limits for throttle policy */
#define THROTL_IOPS_MAX		UINT_MAX

T
Tejun Heo 已提交
30 31
#ifdef CONFIG_BLK_CGROUP

32 33 34 35 36 37 38 39
enum blkg_rwstat_type {
	BLKG_RWSTAT_READ,
	BLKG_RWSTAT_WRITE,
	BLKG_RWSTAT_SYNC,
	BLKG_RWSTAT_ASYNC,

	BLKG_RWSTAT_NR,
	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
40 41
};

42 43
struct blkcg_gq;

T
Tejun Heo 已提交
44
struct blkcg {
45 46
	struct cgroup_subsys_state	css;
	spinlock_t			lock;
47 48

	struct radix_tree_root		blkg_tree;
49
	struct blkcg_gq	__rcu		*blkg_hint;
50
	struct hlist_head		blkg_list;
T
Tejun Heo 已提交
51

52
	struct blkcg_policy_data	*cpd[BLKCG_MAX_POLS];
53

T
Tejun Heo 已提交
54
	struct list_head		all_blkcgs_node;
55 56 57
#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head		cgwb_list;
#endif
58 59
};

60 61
/*
 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
T
Tejun Heo 已提交
62 63
 * recursive.  Used to carry stats of dead children, and, for blkg_rwstat,
 * to carry result values from read and sum operations.
64
 */
65
struct blkg_stat {
T
Tejun Heo 已提交
66
	struct percpu_counter		cpu_cnt;
67
	atomic64_t			aux_cnt;
68 69 70
};

struct blkg_rwstat {
T
Tejun Heo 已提交
71
	struct percpu_counter		cpu_cnt[BLKG_RWSTAT_NR];
72
	atomic64_t			aux_cnt[BLKG_RWSTAT_NR];
73 74
};

75 76 77 78 79
/*
 * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
 * request_queue (q).  This is used by blkcg policies which need to track
 * information per blkcg - q pair.
 *
80 81 82 83 84
 * There can be multiple active blkcg policies and each blkg:policy pair is
 * represented by a blkg_policy_data which is allocated and freed by each
 * policy's pd_alloc/free_fn() methods.  A policy can allocate private data
 * area by allocating larger data structure which embeds blkg_policy_data
 * at the beginning.
85
 */
86
struct blkg_policy_data {
T
Tejun Heo 已提交
87
	/* the blkg and policy id this per-policy data belongs to */
T
Tejun Heo 已提交
88
	struct blkcg_gq			*blkg;
T
Tejun Heo 已提交
89
	int				plid;
90 91
};

92
/*
93 94 95 96 97
 * Policies that need to keep per-blkcg data which is independent from any
 * request_queue associated to it should implement cpd_alloc/free_fn()
 * methods.  A policy can allocate private data area by allocating larger
 * data structure which embeds blkcg_policy_data at the beginning.
 * cpd_init() is invoked to let each policy handle per-blkcg data.
98 99
 */
struct blkcg_policy_data {
100 101
	/* the blkcg and policy id this per-policy data belongs to */
	struct blkcg			*blkcg;
102 103 104
	int				plid;
};

T
Tejun Heo 已提交
105 106
/* association between a blk cgroup and a request queue */
struct blkcg_gq {
T
Tejun Heo 已提交
107
	/* Pointer to the associated request_queue */
108 109 110
	struct request_queue		*q;
	struct list_head		q_node;
	struct hlist_node		blkcg_node;
T
Tejun Heo 已提交
111
	struct blkcg			*blkcg;
T
Tejun Heo 已提交
112

113 114 115 116 117 118
	/*
	 * Each blkg gets congested separately and the congestion state is
	 * propagated to the matching bdi_writeback_congested.
	 */
	struct bdi_writeback_congested	*wb_congested;

T
Tejun Heo 已提交
119 120 121
	/* all non-root blkcg_gq's are guaranteed to have access to parent */
	struct blkcg_gq			*parent;

122 123
	/* request allocation list for this blkcg-q pair */
	struct request_list		rl;
T
Tejun Heo 已提交
124

T
Tejun Heo 已提交
125
	/* reference count */
126
	atomic_t			refcnt;
127

128 129 130
	/* is this blkg online? protected by both blkcg and q locks */
	bool				online;

131 132 133
	struct blkg_rwstat		stat_bytes;
	struct blkg_rwstat		stat_ios;

134
	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];
T
Tejun Heo 已提交
135

136
	struct rcu_head			rcu_head;
137 138
};

139
typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp);
140
typedef void (blkcg_pol_init_cpd_fn)(struct blkcg_policy_data *cpd);
141
typedef void (blkcg_pol_free_cpd_fn)(struct blkcg_policy_data *cpd);
142
typedef void (blkcg_pol_bind_cpd_fn)(struct blkcg_policy_data *cpd);
143
typedef struct blkg_policy_data *(blkcg_pol_alloc_pd_fn)(gfp_t gfp, int node);
144 145 146
typedef void (blkcg_pol_init_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_online_pd_fn)(struct blkg_policy_data *pd);
typedef void (blkcg_pol_offline_pd_fn)(struct blkg_policy_data *pd);
147
typedef void (blkcg_pol_free_pd_fn)(struct blkg_policy_data *pd);
148
typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkg_policy_data *pd);
149

T
Tejun Heo 已提交
150
struct blkcg_policy {
151 152
	int				plid;
	/* cgroup files for the policy */
153
	struct cftype			*dfl_cftypes;
154
	struct cftype			*legacy_cftypes;
155 156

	/* operations */
157
	blkcg_pol_alloc_cpd_fn		*cpd_alloc_fn;
158
	blkcg_pol_init_cpd_fn		*cpd_init_fn;
159
	blkcg_pol_free_cpd_fn		*cpd_free_fn;
160
	blkcg_pol_bind_cpd_fn		*cpd_bind_fn;
161

162
	blkcg_pol_alloc_pd_fn		*pd_alloc_fn;
163
	blkcg_pol_init_pd_fn		*pd_init_fn;
164 165
	blkcg_pol_online_pd_fn		*pd_online_fn;
	blkcg_pol_offline_pd_fn		*pd_offline_fn;
166
	blkcg_pol_free_pd_fn		*pd_free_fn;
167
	blkcg_pol_reset_pd_stats_fn	*pd_reset_stats_fn;
168 169
};

T
Tejun Heo 已提交
170
extern struct blkcg blkcg_root;
T
Tejun Heo 已提交
171
extern struct cgroup_subsys_state * const blkcg_root_css;
172

T
Tejun Heo 已提交
173 174
struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
				      struct request_queue *q, bool update_hint);
T
Tejun Heo 已提交
175 176
struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
				    struct request_queue *q);
177 178 179
int blkcg_init_queue(struct request_queue *q);
void blkcg_drain_queue(struct request_queue *q);
void blkcg_exit_queue(struct request_queue *q);
180

181
/* Blkio controller policy registration */
182
int blkcg_policy_register(struct blkcg_policy *pol);
T
Tejun Heo 已提交
183
void blkcg_policy_unregister(struct blkcg_policy *pol);
184
int blkcg_activate_policy(struct request_queue *q,
T
Tejun Heo 已提交
185
			  const struct blkcg_policy *pol);
186
void blkcg_deactivate_policy(struct request_queue *q,
T
Tejun Heo 已提交
187
			     const struct blkcg_policy *pol);
188

189
const char *blkg_dev_name(struct blkcg_gq *blkg);
T
Tejun Heo 已提交
190
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
191 192
		       u64 (*prfill)(struct seq_file *,
				     struct blkg_policy_data *, int),
T
Tejun Heo 已提交
193
		       const struct blkcg_policy *pol, int data,
194
		       bool show_total);
195 196
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
197
			 const struct blkg_rwstat *rwstat);
198 199 200
u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
		       int off);
201 202 203 204
int blkg_print_stat_bytes(struct seq_file *sf, void *v);
int blkg_print_stat_ios(struct seq_file *sf, void *v);
int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v);
int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v);
205

206 207 208 209
u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
			    struct blkcg_policy *pol, int off);
struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
					     struct blkcg_policy *pol, int off);
210

211
struct blkg_conf_ctx {
212
	struct gendisk			*disk;
T
Tejun Heo 已提交
213
	struct blkcg_gq			*blkg;
214
	char				*body;
215 216
};

T
Tejun Heo 已提交
217
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
218
		   char *input, struct blkg_conf_ctx *ctx);
219 220 221
void blkg_conf_finish(struct blkg_conf_ctx *ctx);


222 223 224 225 226
static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
{
	return css ? container_of(css, struct blkcg, css) : NULL;
}

227 228
static inline struct blkcg *bio_blkcg(struct bio *bio)
{
229 230
	struct cgroup_subsys_state *css;

231
	if (bio && bio->bi_css)
232
		return css_to_blkcg(bio->bi_css);
233 234 235 236
	css = kthread_blkcg();
	if (css)
		return css_to_blkcg(css);
	return css_to_blkcg(task_css(current, io_cgrp_id));
237 238
}

T
Tejun Heo 已提交
239 240 241 242 243 244 245 246
/**
 * blkcg_parent - get the parent of a blkcg
 * @blkcg: blkcg of interest
 *
 * Return the parent blkcg of @blkcg.  Can be called anytime.
 */
static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
{
T
Tejun Heo 已提交
247
	return css_to_blkcg(blkcg->css.parent);
T
Tejun Heo 已提交
248 249
}

T
Tejun Heo 已提交
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
/**
 * __blkg_lookup - internal version of blkg_lookup()
 * @blkcg: blkcg of interest
 * @q: request_queue of interest
 * @update_hint: whether to update lookup hint with the result or not
 *
 * This is internal version and shouldn't be used by policy
 * implementations.  Looks up blkgs for the @blkcg - @q pair regardless of
 * @q's bypass state.  If @update_hint is %true, the caller should be
 * holding @q->queue_lock and lookup hint is updated on success.
 */
static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
					     struct request_queue *q,
					     bool update_hint)
{
	struct blkcg_gq *blkg;

267 268 269
	if (blkcg == &blkcg_root)
		return q->root_blkg;

T
Tejun Heo 已提交
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
	blkg = rcu_dereference(blkcg->blkg_hint);
	if (blkg && blkg->q == q)
		return blkg;

	return blkg_lookup_slowpath(blkcg, q, update_hint);
}

/**
 * blkg_lookup - lookup blkg for the specified blkcg - q pair
 * @blkcg: blkcg of interest
 * @q: request_queue of interest
 *
 * Lookup blkg for the @blkcg - @q pair.  This function should be called
 * under RCU read lock and is guaranteed to return %NULL if @q is bypassing
 * - see blk_queue_bypass_start() for details.
 */
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
					   struct request_queue *q)
{
	WARN_ON_ONCE(!rcu_read_lock_held());

	if (unlikely(blk_queue_bypass(q)))
		return NULL;
	return __blkg_lookup(blkcg, q, false);
}

296 297 298 299 300 301 302
/**
 * blkg_to_pdata - get policy private data
 * @blkg: blkg of interest
 * @pol: policy of interest
 *
 * Return pointer to private data associated with the @blkg-@pol pair.
 */
303 304
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
						  struct blkcg_policy *pol)
305
{
306
	return blkg ? blkg->pd[pol->plid] : NULL;
307 308
}

309 310 311
static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
						     struct blkcg_policy *pol)
{
312
	return blkcg ? blkcg->cpd[pol->plid] : NULL;
313 314
}

315 316
/**
 * pdata_to_blkg - get blkg associated with policy private data
317
 * @pd: policy private data of interest
318
 *
319
 * @pd is policy private data.  Determine the blkg it's associated with.
320
 */
321
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
322
{
323
	return pd ? pd->blkg : NULL;
324 325
}

326 327 328 329 330
static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
{
	return cpd ? cpd->blkcg : NULL;
}

T
Tejun Heo 已提交
331 332 333 334 335 336 337 338
/**
 * blkg_path - format cgroup path of blkg
 * @blkg: blkg of interest
 * @buf: target buffer
 * @buflen: target buffer length
 *
 * Format the path of the cgroup of @blkg into @buf.
 */
T
Tejun Heo 已提交
339
static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
340
{
341
	return cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
342 343
}

T
Tejun Heo 已提交
344 345 346 347
/**
 * blkg_get - get a blkg reference
 * @blkg: blkg to get
 *
348
 * The caller should be holding an existing reference.
T
Tejun Heo 已提交
349
 */
T
Tejun Heo 已提交
350
static inline void blkg_get(struct blkcg_gq *blkg)
T
Tejun Heo 已提交
351
{
352 353
	WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
	atomic_inc(&blkg->refcnt);
T
Tejun Heo 已提交
354 355
}

356
void __blkg_release_rcu(struct rcu_head *rcu);
T
Tejun Heo 已提交
357 358 359 360 361

/**
 * blkg_put - put a blkg reference
 * @blkg: blkg to put
 */
T
Tejun Heo 已提交
362
static inline void blkg_put(struct blkcg_gq *blkg)
T
Tejun Heo 已提交
363
{
364 365
	WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
	if (atomic_dec_and_test(&blkg->refcnt))
366
		call_rcu(&blkg->rcu_head, __blkg_release_rcu);
T
Tejun Heo 已提交
367 368
}

369 370 371
/**
 * blkg_for_each_descendant_pre - pre-order walk of a blkg's descendants
 * @d_blkg: loop cursor pointing to the current descendant
372
 * @pos_css: used for iteration
373 374 375 376 377
 * @p_blkg: target blkg to walk descendants of
 *
 * Walk @c_blkg through the descendants of @p_blkg.  Must be used with RCU
 * read locked.  If called under either blkcg or queue lock, the iteration
 * is guaranteed to include all and only online blkgs.  The caller may
378
 * update @pos_css by calling css_rightmost_descendant() to skip subtree.
379
 * @p_blkg is included in the iteration and the first node to be visited.
380
 */
381 382 383
#define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg)		\
	css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css)	\
		if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css),	\
384 385
					      (p_blkg)->q, false)))

386 387 388
/**
 * blkg_for_each_descendant_post - post-order walk of a blkg's descendants
 * @d_blkg: loop cursor pointing to the current descendant
389
 * @pos_css: used for iteration
390 391 392
 * @p_blkg: target blkg to walk descendants of
 *
 * Similar to blkg_for_each_descendant_pre() but performs post-order
393 394
 * traversal instead.  Synchronization rules are the same.  @p_blkg is
 * included in the iteration and the last node to be visited.
395
 */
396 397 398
#define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg)		\
	css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css)	\
		if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css),	\
399 400
					      (p_blkg)->q, false)))

401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
/**
 * blk_get_rl - get request_list to use
 * @q: request_queue of interest
 * @bio: bio which will be attached to the allocated request (may be %NULL)
 *
 * The caller wants to allocate a request from @q to use for @bio.  Find
 * the request_list to use and obtain a reference on it.  Should be called
 * under queue_lock.  This function is guaranteed to return non-%NULL
 * request_list.
 */
static inline struct request_list *blk_get_rl(struct request_queue *q,
					      struct bio *bio)
{
	struct blkcg *blkcg;
	struct blkcg_gq *blkg;

	rcu_read_lock();

	blkcg = bio_blkcg(bio);

	/* bypass blkg lookup and use @q->root_rl directly for root */
	if (blkcg == &blkcg_root)
		goto root_rl;

	/*
	 * Try to use blkg->rl.  blkg lookup may fail under memory pressure
	 * or if either the blkcg or queue is going away.  Fall back to
	 * root_rl in such cases.
	 */
430 431
	blkg = blkg_lookup(blkcg, q);
	if (unlikely(!blkg))
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
		goto root_rl;

	blkg_get(blkg);
	rcu_read_unlock();
	return &blkg->rl;
root_rl:
	rcu_read_unlock();
	return &q->root_rl;
}

/**
 * blk_put_rl - put request_list
 * @rl: request_list to put
 *
 * Put the reference acquired by blk_get_rl().  Should be called under
 * queue_lock.
 */
static inline void blk_put_rl(struct request_list *rl)
{
451
	if (rl->blkg->blkcg != &blkcg_root)
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
		blkg_put(rl->blkg);
}

/**
 * blk_rq_set_rl - associate a request with a request_list
 * @rq: request of interest
 * @rl: target request_list
 *
 * Associate @rq with @rl so that accounting and freeing can know the
 * request_list @rq came from.
 */
static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl)
{
	rq->rl = rl;
}

/**
 * blk_rq_rl - return the request_list a request came from
 * @rq: request of interest
 *
 * Return the request_list @rq is allocated from.
 */
static inline struct request_list *blk_rq_rl(struct request *rq)
{
	return rq->rl;
}

struct request_list *__blk_queue_next_rl(struct request_list *rl,
					 struct request_queue *q);
/**
 * blk_queue_for_each_rl - iterate through all request_lists of a request_queue
 *
 * Should be used under queue_lock.
 */
#define blk_queue_for_each_rl(rl, q)	\
	for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))

T
Tejun Heo 已提交
489
static inline int blkg_stat_init(struct blkg_stat *stat, gfp_t gfp)
490
{
T
Tejun Heo 已提交
491 492 493 494 495 496
	int ret;

	ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
	if (ret)
		return ret;

497
	atomic64_set(&stat->aux_cnt, 0);
T
Tejun Heo 已提交
498 499 500 501 502 503
	return 0;
}

static inline void blkg_stat_exit(struct blkg_stat *stat)
{
	percpu_counter_destroy(&stat->cpu_cnt);
504 505
}

506 507 508 509 510
/**
 * blkg_stat_add - add a value to a blkg_stat
 * @stat: target blkg_stat
 * @val: value to add
 *
T
Tejun Heo 已提交
511 512
 * Add @val to @stat.  The caller must ensure that IRQ on the same CPU
 * don't re-enter this function for the same counter.
513 514 515
 */
static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
{
516
	percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
517 518 519 520 521 522 523 524
}

/**
 * blkg_stat_read - read the current value of a blkg_stat
 * @stat: blkg_stat to read
 */
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
{
T
Tejun Heo 已提交
525
	return percpu_counter_sum_positive(&stat->cpu_cnt);
526 527 528 529 530 531 532 533
}

/**
 * blkg_stat_reset - reset a blkg_stat
 * @stat: blkg_stat to reset
 */
static inline void blkg_stat_reset(struct blkg_stat *stat)
{
T
Tejun Heo 已提交
534
	percpu_counter_set(&stat->cpu_cnt, 0);
535
	atomic64_set(&stat->aux_cnt, 0);
536 537
}

538
/**
539
 * blkg_stat_add_aux - add a blkg_stat into another's aux count
540 541 542
 * @to: the destination blkg_stat
 * @from: the source
 *
543
 * Add @from's count including the aux one to @to's aux count.
544
 */
545 546
static inline void blkg_stat_add_aux(struct blkg_stat *to,
				     struct blkg_stat *from)
547
{
548 549
	atomic64_add(blkg_stat_read(from) + atomic64_read(&from->aux_cnt),
		     &to->aux_cnt);
550 551
}

T
Tejun Heo 已提交
552
static inline int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp)
553
{
T
Tejun Heo 已提交
554 555 556 557 558 559 560 561 562 563 564 565 566
	int i, ret;

	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
		ret = percpu_counter_init(&rwstat->cpu_cnt[i], 0, gfp);
		if (ret) {
			while (--i >= 0)
				percpu_counter_destroy(&rwstat->cpu_cnt[i]);
			return ret;
		}
		atomic64_set(&rwstat->aux_cnt[i], 0);
	}
	return 0;
}
567

T
Tejun Heo 已提交
568 569 570
static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
{
	int i;
571 572

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
T
Tejun Heo 已提交
573
		percpu_counter_destroy(&rwstat->cpu_cnt[i]);
574 575
}

576 577 578
/**
 * blkg_rwstat_add - add a value to a blkg_rwstat
 * @rwstat: target blkg_rwstat
579
 * @op: REQ_OP and flags
580 581 582 583 584 585
 * @val: value to add
 *
 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 * caller is responsible for synchronizing calls to this function.
 */
static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
586
				   unsigned int op, uint64_t val)
587
{
T
Tejun Heo 已提交
588
	struct percpu_counter *cnt;
589

590
	if (op_is_write(op))
T
Tejun Heo 已提交
591
		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
592
	else
T
Tejun Heo 已提交
593 594
		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];

595
	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
T
Tejun Heo 已提交
596

597
	if (op_is_sync(op))
T
Tejun Heo 已提交
598
		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
599
	else
T
Tejun Heo 已提交
600
		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
601

602
	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
603 604 605 606 607 608
}

/**
 * blkg_rwstat_read - read the current values of a blkg_rwstat
 * @rwstat: blkg_rwstat to read
 *
T
Tejun Heo 已提交
609
 * Read the current snapshot of @rwstat and return it in the aux counts.
610
 */
611
static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
612
{
T
Tejun Heo 已提交
613 614
	struct blkg_rwstat result;
	int i;
615

T
Tejun Heo 已提交
616 617 618 619
	for (i = 0; i < BLKG_RWSTAT_NR; i++)
		atomic64_set(&result.aux_cnt[i],
			     percpu_counter_sum_positive(&rwstat->cpu_cnt[i]));
	return result;
620 621 622
}

/**
623
 * blkg_rwstat_total - read the total count of a blkg_rwstat
624 625 626 627 628 629
 * @rwstat: blkg_rwstat to read
 *
 * Return the total count of @rwstat regardless of the IO direction.  This
 * function can be called without synchronization and takes care of u64
 * atomicity.
 */
630
static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
631 632 633
{
	struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);

T
Tejun Heo 已提交
634 635
	return atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +
		atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);
636 637 638 639 640 641 642 643
}

/**
 * blkg_rwstat_reset - reset a blkg_rwstat
 * @rwstat: blkg_rwstat to reset
 */
static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
{
644 645
	int i;

T
Tejun Heo 已提交
646 647
	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
		percpu_counter_set(&rwstat->cpu_cnt[i], 0);
648
		atomic64_set(&rwstat->aux_cnt[i], 0);
T
Tejun Heo 已提交
649
	}
650 651
}

652
/**
653
 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
654 655 656
 * @to: the destination blkg_rwstat
 * @from: the source
 *
657
 * Add @from's count including the aux one to @to's aux count.
658
 */
659 660
static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
				       struct blkg_rwstat *from)
661 662 663 664 665
{
	struct blkg_rwstat v = blkg_rwstat_read(from);
	int i;

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
T
Tejun Heo 已提交
666 667
		atomic64_add(atomic64_read(&v.aux_cnt[i]) +
			     atomic64_read(&from->aux_cnt[i]),
668
			     &to->aux_cnt[i]);
669 670
}

671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
#ifdef CONFIG_BLK_DEV_THROTTLING
extern bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
			   struct bio *bio);
#else
static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
				  struct bio *bio) { return false; }
#endif

static inline bool blkcg_bio_issue_check(struct request_queue *q,
					 struct bio *bio)
{
	struct blkcg *blkcg;
	struct blkcg_gq *blkg;
	bool throtl = false;

	rcu_read_lock();
	blkcg = bio_blkcg(bio);

689 690 691
	/* associate blkcg if bio hasn't attached one */
	bio_associate_blkcg(bio, &blkcg->css);

692 693 694 695 696 697 698 699 700 701 702
	blkg = blkg_lookup(blkcg, q);
	if (unlikely(!blkg)) {
		spin_lock_irq(q->queue_lock);
		blkg = blkg_lookup_create(blkcg, q);
		if (IS_ERR(blkg))
			blkg = NULL;
		spin_unlock_irq(q->queue_lock);
	}

	throtl = blk_throtl_bio(q, blkg, bio);

703 704
	if (!throtl) {
		blkg = blkg ?: q->root_blkg;
705
		blkg_rwstat_add(&blkg->stat_bytes, bio->bi_opf,
706
				bio->bi_iter.bi_size);
707
		blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1);
708 709
	}

710 711 712 713
	rcu_read_unlock();
	return !throtl;
}

714 715
#else	/* CONFIG_BLK_CGROUP */

716 717
struct blkcg {
};
718

719 720 721
struct blkg_policy_data {
};

722 723 724
struct blkcg_policy_data {
};

T
Tejun Heo 已提交
725
struct blkcg_gq {
726 727
};

T
Tejun Heo 已提交
728
struct blkcg_policy {
729 730
};

T
Tejun Heo 已提交
731 732
#define blkcg_root_css	((struct cgroup_subsys_state *)ERR_PTR(-EINVAL))

733 734
#ifdef CONFIG_BLOCK

T
Tejun Heo 已提交
735
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
736 737 738
static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
static inline void blkcg_drain_queue(struct request_queue *q) { }
static inline void blkcg_exit_queue(struct request_queue *q) { }
739
static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
T
Tejun Heo 已提交
740
static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
741
static inline int blkcg_activate_policy(struct request_queue *q,
T
Tejun Heo 已提交
742
					const struct blkcg_policy *pol) { return 0; }
743
static inline void blkcg_deactivate_policy(struct request_queue *q,
T
Tejun Heo 已提交
744 745
					   const struct blkcg_policy *pol) { }

746
static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
747

748 749 750
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
						  struct blkcg_policy *pol) { return NULL; }
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
T
Tejun Heo 已提交
751 752 753
static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
static inline void blkg_get(struct blkcg_gq *blkg) { }
static inline void blkg_put(struct blkcg_gq *blkg) { }
754

755 756 757 758 759 760
static inline struct request_list *blk_get_rl(struct request_queue *q,
					      struct bio *bio) { return &q->root_rl; }
static inline void blk_put_rl(struct request_list *rl) { }
static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }

761 762 763
static inline bool blkcg_bio_issue_check(struct request_queue *q,
					 struct bio *bio) { return true; }

764 765 766
#define blk_queue_for_each_rl(rl, q)	\
	for ((rl) = &(q)->root_rl; (rl); (rl) = NULL)

767
#endif	/* CONFIG_BLOCK */
768 769
#endif	/* CONFIG_BLK_CGROUP */
#endif	/* _BLK_CGROUP_H */