blk-cgroup.c 35.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
12 13 14 15
 *
 * For policy-specific per-blkcg data:
 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
 *                    Arianna Avanzini <avanzini.arianna@gmail.com>
16 17
 */
#include <linux/ioprio.h>
18
#include <linux/kdev_t.h>
19
#include <linux/module.h>
20
#include <linux/err.h>
21
#include <linux/blkdev.h>
22
#include <linux/backing-dev.h>
23
#include <linux/slab.h>
24
#include <linux/genhd.h>
25
#include <linux/delay.h>
T
Tejun Heo 已提交
26
#include <linux/atomic.h>
27
#include <linux/ctype.h>
28
#include <linux/blk-cgroup.h>
29
#include "blk.h"
30

31 32
#define MAX_KEY_LEN 100

33 34 35 36 37 38 39 40
/*
 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
 * policy [un]register operations including cgroup file additions /
 * removals.  Putting cgroup file registration outside blkcg_pol_mutex
 * allows grabbing it from cgroup callbacks.
 */
static DEFINE_MUTEX(blkcg_pol_register_mutex);
41
static DEFINE_MUTEX(blkcg_pol_mutex);
42

43
struct blkcg blkcg_root;
T
Tejun Heo 已提交
44
EXPORT_SYMBOL_GPL(blkcg_root);
45

T
Tejun Heo 已提交
46 47
struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;

T
Tejun Heo 已提交
48
static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
49

T
Tejun Heo 已提交
50 51
static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */

52
static bool blkcg_policy_enabled(struct request_queue *q,
T
Tejun Heo 已提交
53
				 const struct blkcg_policy *pol)
54 55 56 57
{
	return pol && test_bit(pol->plid, q->blkcg_pols);
}

58 59 60 61 62 63
/**
 * blkg_free - free a blkg
 * @blkg: blkg to free
 *
 * Free @blkg which may be partially allocated.
 */
T
Tejun Heo 已提交
64
static void blkg_free(struct blkcg_gq *blkg)
65
{
66
	int i;
67 68 69 70

	if (!blkg)
		return;

71
	for (i = 0; i < BLKCG_MAX_POLS; i++)
72 73
		if (blkg->pd[i])
			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
74

75 76
	if (blkg->blkcg != &blkcg_root)
		blk_exit_rl(&blkg->rl);
77 78 79

	blkg_rwstat_exit(&blkg->stat_ios);
	blkg_rwstat_exit(&blkg->stat_bytes);
80
	kfree(blkg);
81 82 83 84 85 86
}

/**
 * blkg_alloc - allocate a blkg
 * @blkcg: block cgroup the new blkg is associated with
 * @q: request_queue the new blkg is associated with
87
 * @gfp_mask: allocation mask to use
88
 *
89
 * Allocate a new blkg assocating @blkcg and @q.
90
 */
91 92
static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
				   gfp_t gfp_mask)
93
{
T
Tejun Heo 已提交
94
	struct blkcg_gq *blkg;
95
	int i;
96 97

	/* alloc and init base part */
98
	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
99 100 101
	if (!blkg)
		return NULL;

102 103 104 105
	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
	    blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
		goto err_free;

T
Tejun Heo 已提交
106
	blkg->q = q;
107
	INIT_LIST_HEAD(&blkg->q_node);
108
	blkg->blkcg = blkcg;
109
	atomic_set(&blkg->refcnt, 1);
110

111 112 113 114 115 116 117
	/* root blkg uses @q->root_rl, init rl only for !root blkgs */
	if (blkcg != &blkcg_root) {
		if (blk_init_rl(&blkg->rl, q, gfp_mask))
			goto err_free;
		blkg->rl.blkg = blkg;
	}

T
Tejun Heo 已提交
118
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
T
Tejun Heo 已提交
119
		struct blkcg_policy *pol = blkcg_policy[i];
120
		struct blkg_policy_data *pd;
121

122
		if (!blkcg_policy_enabled(q, pol))
123 124 125
			continue;

		/* alloc per-policy data and attach it to blkg */
126
		pd = pol->pd_alloc_fn(gfp_mask, q->node);
127 128
		if (!pd)
			goto err_free;
129

130 131
		blkg->pd[i] = pd;
		pd->blkg = blkg;
T
Tejun Heo 已提交
132
		pd->plid = i;
133 134
	}

135
	return blkg;
136 137 138 139

err_free:
	blkg_free(blkg);
	return NULL;
140 141
}

T
Tejun Heo 已提交
142 143
struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
				      struct request_queue *q, bool update_hint)
144
{
T
Tejun Heo 已提交
145
	struct blkcg_gq *blkg;
146

147
	/*
148 149 150 151
	 * Hint didn't match.  Look up from the radix tree.  Note that the
	 * hint can only be updated under queue_lock as otherwise @blkg
	 * could have already been removed from blkg_tree.  The caller is
	 * responsible for grabbing queue_lock if @update_hint.
152 153
	 */
	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
154 155 156 157 158
	if (blkg && blkg->q == q) {
		if (update_hint) {
			lockdep_assert_held(q->queue_lock);
			rcu_assign_pointer(blkcg->blkg_hint, blkg);
		}
159
		return blkg;
160
	}
161

162 163
	return NULL;
}
164
EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
165

166 167
/*
 * If @new_blkg is %NULL, this function tries to allocate a new one as
168
 * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return.
169
 */
170 171 172
static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
				    struct request_queue *q,
				    struct blkcg_gq *new_blkg)
173
{
T
Tejun Heo 已提交
174
	struct blkcg_gq *blkg;
175
	struct bdi_writeback_congested *wb_congested;
176
	int i, ret;
177

178 179 180
	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

181
	/* blkg holds a reference to blkcg */
182
	if (!css_tryget_online(&blkcg->css)) {
183
		ret = -ENODEV;
184
		goto err_free_blkg;
185
	}
186

187
	wb_congested = wb_congested_get_create(&q->backing_dev_info,
188
					       blkcg->css.id, GFP_NOWAIT);
189 190 191 192 193
	if (!wb_congested) {
		ret = -ENOMEM;
		goto err_put_css;
	}

194
	/* allocate */
195
	if (!new_blkg) {
196
		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT);
197
		if (unlikely(!new_blkg)) {
198
			ret = -ENOMEM;
199
			goto err_put_congested;
200 201 202
		}
	}
	blkg = new_blkg;
203
	blkg->wb_congested = wb_congested;
204

205
	/* link parent */
T
Tejun Heo 已提交
206 207 208
	if (blkcg_parent(blkcg)) {
		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
		if (WARN_ON_ONCE(!blkg->parent)) {
209
			ret = -ENODEV;
210
			goto err_put_congested;
T
Tejun Heo 已提交
211 212 213 214
		}
		blkg_get(blkg->parent);
	}

215 216 217 218 219
	/* invoke per-policy init */
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && pol->pd_init_fn)
220
			pol->pd_init_fn(blkg->pd[i]);
221 222 223
	}

	/* insert */
224
	spin_lock(&blkcg->lock);
225 226 227 228
	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
	if (likely(!ret)) {
		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
		list_add(&blkg->q_node, &q->blkg_list);
229 230 231 232 233

		for (i = 0; i < BLKCG_MAX_POLS; i++) {
			struct blkcg_policy *pol = blkcg_policy[i];

			if (blkg->pd[i] && pol->pd_online_fn)
234
				pol->pd_online_fn(blkg->pd[i]);
235
		}
236
	}
237
	blkg->online = true;
238
	spin_unlock(&blkcg->lock);
239

240
	if (!ret)
241
		return blkg;
242

T
Tejun Heo 已提交
243 244 245 246
	/* @blkg failed fully initialized, use the usual release path */
	blkg_put(blkg);
	return ERR_PTR(ret);

247 248
err_put_congested:
	wb_congested_put(wb_congested);
249
err_put_css:
250
	css_put(&blkcg->css);
251
err_free_blkg:
252
	blkg_free(new_blkg);
253
	return ERR_PTR(ret);
254
}
255

256 257 258 259 260 261
/**
 * blkg_lookup_create - lookup blkg, try to create one if not there
 * @blkcg: blkcg of interest
 * @q: request_queue of interest
 *
 * Lookup blkg for the @blkcg - @q pair.  If it doesn't exist, try to
T
Tejun Heo 已提交
262 263 264
 * create one.  blkg creation is performed recursively from blkcg_root such
 * that all non-root blkg's have access to the parent blkg.  This function
 * should be called under RCU read lock and @q->queue_lock.
265 266 267 268 269
 *
 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
 * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not
 * dead and bypassing, returns ERR_PTR(-EBUSY).
 */
T
Tejun Heo 已提交
270 271
struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
				    struct request_queue *q)
272
{
273 274 275 276 277
	struct blkcg_gq *blkg;

	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

278 279 280 281 282
	/*
	 * This could be the first entry point of blkcg implementation and
	 * we shouldn't allow anything to go through for a bypassing queue.
	 */
	if (unlikely(blk_queue_bypass(q)))
283
		return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY);
284 285 286 287 288

	blkg = __blkg_lookup(blkcg, q, true);
	if (blkg)
		return blkg;

T
Tejun Heo 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
	/*
	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
	 * non-root blkgs have access to their parents.
	 */
	while (true) {
		struct blkcg *pos = blkcg;
		struct blkcg *parent = blkcg_parent(blkcg);

		while (parent && !__blkg_lookup(parent, q, false)) {
			pos = parent;
			parent = blkcg_parent(parent);
		}

		blkg = blkg_create(pos, q, NULL);
		if (pos == blkcg || IS_ERR(blkg))
			return blkg;
	}
306
}
307

T
Tejun Heo 已提交
308
static void blkg_destroy(struct blkcg_gq *blkg)
309
{
T
Tejun Heo 已提交
310
	struct blkcg *blkcg = blkg->blkcg;
311
	struct blkcg_gq *parent = blkg->parent;
312
	int i;
313

314
	lockdep_assert_held(blkg->q->queue_lock);
315
	lockdep_assert_held(&blkcg->lock);
316 317

	/* Something wrong if we are trying to remove same group twice */
318
	WARN_ON_ONCE(list_empty(&blkg->q_node));
319
	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
320

321 322 323 324
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && pol->pd_offline_fn)
325
			pol->pd_offline_fn(blkg->pd[i]);
326
	}
327 328 329 330 331 332

	if (parent) {
		blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
		blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
	}

333 334
	blkg->online = false;

335
	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
336
	list_del_init(&blkg->q_node);
337
	hlist_del_init_rcu(&blkg->blkcg_node);
338

339 340 341 342 343
	/*
	 * Both setting lookup hint to and clearing it from @blkg are done
	 * under queue_lock.  If it's not pointing to @blkg now, it never
	 * will.  Hint assignment itself can race safely.
	 */
344
	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
345 346
		rcu_assign_pointer(blkcg->blkg_hint, NULL);

347 348 349 350 351 352 353
	/*
	 * Put the reference taken at the time of creation so that when all
	 * queues are gone, group can be destroyed.
	 */
	blkg_put(blkg);
}

354 355 356 357
/**
 * blkg_destroy_all - destroy all blkgs associated with a request_queue
 * @q: request_queue of interest
 *
358
 * Destroy all blkgs associated with @q.
359
 */
360
static void blkg_destroy_all(struct request_queue *q)
361
{
T
Tejun Heo 已提交
362
	struct blkcg_gq *blkg, *n;
363

364
	lockdep_assert_held(q->queue_lock);
365

366
	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
T
Tejun Heo 已提交
367
		struct blkcg *blkcg = blkg->blkcg;
368

369 370 371
		spin_lock(&blkcg->lock);
		blkg_destroy(blkg);
		spin_unlock(&blkcg->lock);
372 373 374
	}
}

375 376 377 378 379 380 381 382 383
/*
 * A group is RCU protected, but having an rcu lock does not mean that one
 * can access all the fields of blkg and assume these are valid.  For
 * example, don't try to follow throtl_data and request queue links.
 *
 * Having a reference to blkg under an rcu allows accesses to only values
 * local to groups like group stats and group rate limits.
 */
void __blkg_release_rcu(struct rcu_head *rcu_head)
T
Tejun Heo 已提交
384
{
385
	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
386

T
Tejun Heo 已提交
387
	/* release the blkcg and parent blkg refs this blkg has been holding */
T
Tejun Heo 已提交
388
	css_put(&blkg->blkcg->css);
389
	if (blkg->parent)
T
Tejun Heo 已提交
390
		blkg_put(blkg->parent);
T
Tejun Heo 已提交
391

392 393
	wb_congested_put(blkg->wb_congested);

394
	blkg_free(blkg);
T
Tejun Heo 已提交
395
}
396
EXPORT_SYMBOL_GPL(__blkg_release_rcu);
T
Tejun Heo 已提交
397

398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
/*
 * The next function used by blk_queue_for_each_rl().  It's a bit tricky
 * because the root blkg uses @q->root_rl instead of its own rl.
 */
struct request_list *__blk_queue_next_rl(struct request_list *rl,
					 struct request_queue *q)
{
	struct list_head *ent;
	struct blkcg_gq *blkg;

	/*
	 * Determine the current blkg list_head.  The first entry is
	 * root_rl which is off @q->blkg_list and mapped to the head.
	 */
	if (rl == &q->root_rl) {
		ent = &q->blkg_list;
414 415 416
		/* There are no more block groups, hence no request lists */
		if (list_empty(ent))
			return NULL;
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
	} else {
		blkg = container_of(rl, struct blkcg_gq, rl);
		ent = &blkg->q_node;
	}

	/* walk to the next list_head, skip root blkcg */
	ent = ent->next;
	if (ent == &q->root_blkg->q_node)
		ent = ent->next;
	if (ent == &q->blkg_list)
		return NULL;

	blkg = container_of(ent, struct blkcg_gq, q_node);
	return &blkg->rl;
}

433 434
static int blkcg_reset_stats(struct cgroup_subsys_state *css,
			     struct cftype *cftype, u64 val)
435
{
436
	struct blkcg *blkcg = css_to_blkcg(css);
T
Tejun Heo 已提交
437
	struct blkcg_gq *blkg;
438
	int i;
439

440
	mutex_lock(&blkcg_pol_mutex);
441
	spin_lock_irq(&blkcg->lock);
T
Tejun Heo 已提交
442 443 444 445 446 447

	/*
	 * Note that stat reset is racy - it doesn't synchronize against
	 * stat updates.  This is a debug feature which shouldn't exist
	 * anyway.  If you get hit by a race, retry.
	 */
448
	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
449 450 451
		blkg_rwstat_reset(&blkg->stat_bytes);
		blkg_rwstat_reset(&blkg->stat_ios);

T
Tejun Heo 已提交
452
		for (i = 0; i < BLKCG_MAX_POLS; i++) {
T
Tejun Heo 已提交
453
			struct blkcg_policy *pol = blkcg_policy[i];
454

455 456
			if (blkg->pd[i] && pol->pd_reset_stats_fn)
				pol->pd_reset_stats_fn(blkg->pd[i]);
457
		}
458
	}
459

460
	spin_unlock_irq(&blkcg->lock);
461
	mutex_unlock(&blkcg_pol_mutex);
462 463 464
	return 0;
}

465
const char *blkg_dev_name(struct blkcg_gq *blkg)
466
{
467 468 469 470
	/* some drivers (floppy) instantiate a queue w/o disk registered */
	if (blkg->q->backing_dev_info.dev)
		return dev_name(blkg->q->backing_dev_info.dev);
	return NULL;
471
}
472
EXPORT_SYMBOL_GPL(blkg_dev_name);
473

474 475 476 477 478 479 480 481 482 483 484
/**
 * blkcg_print_blkgs - helper for printing per-blkg data
 * @sf: seq_file to print to
 * @blkcg: blkcg of interest
 * @prfill: fill function to print out a blkg
 * @pol: policy in question
 * @data: data to be passed to @prfill
 * @show_total: to print out sum of prfill return values or not
 *
 * This function invokes @prfill on each blkg of @blkcg if pd for the
 * policy specified by @pol exists.  @prfill is invoked with @sf, the
485 486 487
 * policy data and @data and the matching queue lock held.  If @show_total
 * is %true, the sum of the return values from @prfill is printed with
 * "Total" label at the end.
488 489 490 491
 *
 * This is to be used to construct print functions for
 * cftype->read_seq_string method.
 */
T
Tejun Heo 已提交
492
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
493 494
		       u64 (*prfill)(struct seq_file *,
				     struct blkg_policy_data *, int),
T
Tejun Heo 已提交
495
		       const struct blkcg_policy *pol, int data,
496
		       bool show_total)
497
{
T
Tejun Heo 已提交
498
	struct blkcg_gq *blkg;
499
	u64 total = 0;
500

501
	rcu_read_lock();
502
	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
503
		spin_lock_irq(blkg->q->queue_lock);
504
		if (blkcg_policy_enabled(blkg->q, pol))
505
			total += prfill(sf, blkg->pd[pol->plid], data);
506 507 508
		spin_unlock_irq(blkg->q->queue_lock);
	}
	rcu_read_unlock();
509 510 511 512

	if (show_total)
		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
}
513
EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
514 515 516 517

/**
 * __blkg_prfill_u64 - prfill helper for a single u64 value
 * @sf: seq_file to print to
518
 * @pd: policy private data of interest
519 520
 * @v: value to print
 *
521
 * Print @v to @sf for the device assocaited with @pd.
522
 */
523
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
524
{
525
	const char *dname = blkg_dev_name(pd->blkg);
526 527 528 529 530 531 532

	if (!dname)
		return 0;

	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
	return v;
}
533
EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
534 535 536 537

/**
 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
 * @sf: seq_file to print to
538
 * @pd: policy private data of interest
539 540
 * @rwstat: rwstat to print
 *
541
 * Print @rwstat to @sf for the device assocaited with @pd.
542
 */
543
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
544
			 const struct blkg_rwstat *rwstat)
545 546 547 548 549 550 551
{
	static const char *rwstr[] = {
		[BLKG_RWSTAT_READ]	= "Read",
		[BLKG_RWSTAT_WRITE]	= "Write",
		[BLKG_RWSTAT_SYNC]	= "Sync",
		[BLKG_RWSTAT_ASYNC]	= "Async",
	};
552
	const char *dname = blkg_dev_name(pd->blkg);
553 554 555 556 557 558 559 560
	u64 v;
	int i;

	if (!dname)
		return 0;

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
T
Tejun Heo 已提交
561
			   (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
562

T
Tejun Heo 已提交
563 564
	v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
565 566 567
	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
	return v;
}
T
Tejun Heo 已提交
568
EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
569

570 571 572
/**
 * blkg_prfill_stat - prfill callback for blkg_stat
 * @sf: seq_file to print to
573 574
 * @pd: policy private data of interest
 * @off: offset to the blkg_stat in @pd
575 576 577
 *
 * prfill callback for printing a blkg_stat.
 */
578
u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
579
{
580
	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
581
}
582
EXPORT_SYMBOL_GPL(blkg_prfill_stat);
583

584 585 586
/**
 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
 * @sf: seq_file to print to
587 588
 * @pd: policy private data of interest
 * @off: offset to the blkg_rwstat in @pd
589 590 591
 *
 * prfill callback for printing a blkg_rwstat.
 */
592 593
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
		       int off)
594
{
595
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
596

597
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
598
}
599
EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
600

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
				    struct blkg_policy_data *pd, int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_bytes.
 * cftype->private must be set to the blkcg_policy.
 */
int blkg_print_stat_bytes(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_ios.  cftype->private
 * must be set to the blkcg_policy.
 */
int blkg_print_stat_ios(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios);

static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
					      struct blkg_policy_data *pd,
					      int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
							      NULL, off);
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);

/**
 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);

682 683
/**
 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
684 685 686
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_stat
 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
687
 *
688 689 690 691 692 693
 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
 * at @off bytes into @blkg's blkg_policy_data of the policy.
694
 */
695 696
u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
			    struct blkcg_policy *pol, int off)
697 698
{
	struct blkcg_gq *pos_blkg;
699
	struct cgroup_subsys_state *pos_css;
700
	u64 sum = 0;
701

702
	lockdep_assert_held(blkg->q->queue_lock);
703 704

	rcu_read_lock();
705 706 707 708 709
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
		struct blkg_stat *stat;

		if (!pos_blkg->online)
			continue;
710

711 712 713 714 715 716
		if (pol)
			stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			stat = (void *)blkg + off;

		sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
717 718 719 720 721 722 723 724 725
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);

/**
 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
726 727 728
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_rwstat
 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
729
 *
730 731 732 733 734 735
 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
 * is at @off bytes into @blkg's blkg_policy_data of the policy.
736
 */
737 738
struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
					     struct blkcg_policy *pol, int off)
739 740
{
	struct blkcg_gq *pos_blkg;
741
	struct cgroup_subsys_state *pos_css;
742
	struct blkg_rwstat sum = { };
743 744
	int i;

745
	lockdep_assert_held(blkg->q->queue_lock);
746 747

	rcu_read_lock();
748
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
749
		struct blkg_rwstat *rwstat;
750 751 752 753

		if (!pos_blkg->online)
			continue;

754 755 756 757 758
		if (pol)
			rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			rwstat = (void *)pos_blkg + off;

759
		for (i = 0; i < BLKG_RWSTAT_NR; i++)
760 761 762
			atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
				percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
				&sum.aux_cnt[i]);
763 764 765 766 767 768 769
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);

770 771 772
/**
 * blkg_conf_prep - parse and prepare for per-blkg config update
 * @blkcg: target block cgroup
773
 * @pol: target policy
774 775 776 777
 * @input: input string
 * @ctx: blkg_conf_ctx to be filled
 *
 * Parse per-blkg config update from @input and initialize @ctx with the
778 779 780
 * result.  @ctx->blkg points to the blkg to be updated and @ctx->body the
 * part of @input following MAJ:MIN.  This function returns with RCU read
 * lock and queue lock held and must be paired with blkg_conf_finish().
781
 */
T
Tejun Heo 已提交
782
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
783
		   char *input, struct blkg_conf_ctx *ctx)
784
	__acquires(rcu) __acquires(disk->queue->queue_lock)
785
{
786
	struct gendisk *disk;
T
Tejun Heo 已提交
787
	struct blkcg_gq *blkg;
T
Tejun Heo 已提交
788
	unsigned int major, minor;
789 790
	int key_len, part, ret;
	char *body;
791

792
	if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
T
Tejun Heo 已提交
793
		return -EINVAL;
794

795 796 797 798 799
	body = input + key_len;
	if (!isspace(*body))
		return -EINVAL;
	body = skip_spaces(body);

T
Tejun Heo 已提交
800
	disk = get_gendisk(MKDEV(major, minor), &part);
801
	if (!disk)
802
		return -ENODEV;
803 804
	if (part) {
		put_disk(disk);
805
		return -ENODEV;
806
	}
807 808

	rcu_read_lock();
T
Tejun Heo 已提交
809
	spin_lock_irq(disk->queue->queue_lock);
810

811
	if (blkcg_policy_enabled(disk->queue, pol))
812
		blkg = blkg_lookup_create(blkcg, disk->queue);
813
	else
814
		blkg = ERR_PTR(-EOPNOTSUPP);
815

T
Tejun Heo 已提交
816 817
	if (IS_ERR(blkg)) {
		ret = PTR_ERR(blkg);
818
		rcu_read_unlock();
819
		spin_unlock_irq(disk->queue->queue_lock);
820 821 822 823 824 825 826 827 828 829
		put_disk(disk);
		/*
		 * If queue was bypassing, we should retry.  Do so after a
		 * short msleep().  It isn't strictly necessary but queue
		 * can be bypassing for some time and it's always nice to
		 * avoid busy looping.
		 */
		if (ret == -EBUSY) {
			msleep(10);
			ret = restart_syscall();
830
		}
T
Tejun Heo 已提交
831
		return ret;
832
	}
833 834 835

	ctx->disk = disk;
	ctx->blkg = blkg;
836
	ctx->body = body;
T
Tejun Heo 已提交
837
	return 0;
838
}
839
EXPORT_SYMBOL_GPL(blkg_conf_prep);
840

841 842 843 844 845 846 847
/**
 * blkg_conf_finish - finish up per-blkg config update
 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
 *
 * Finish up after per-blkg config update.  This function must be paired
 * with blkg_conf_prep().
 */
848
void blkg_conf_finish(struct blkg_conf_ctx *ctx)
849
	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
850
{
851
	spin_unlock_irq(ctx->disk->queue->queue_lock);
852 853
	rcu_read_unlock();
	put_disk(ctx->disk);
854
}
855
EXPORT_SYMBOL_GPL(blkg_conf_finish);
856

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
static int blkcg_print_stat(struct seq_file *sf, void *v)
{
	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
	struct blkcg_gq *blkg;

	rcu_read_lock();

	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
		const char *dname;
		struct blkg_rwstat rwstat;
		u64 rbytes, wbytes, rios, wios;

		dname = blkg_dev_name(blkg);
		if (!dname)
			continue;

		spin_lock_irq(blkg->q->queue_lock);

		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
					offsetof(struct blkcg_gq, stat_bytes));
		rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
		wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
					offsetof(struct blkcg_gq, stat_ios));
		rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
		wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);

		spin_unlock_irq(blkg->q->queue_lock);

		if (rbytes || wbytes || rios || wios)
			seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
				   dname, rbytes, wbytes, rios, wios);
	}

	rcu_read_unlock();
	return 0;
}

struct cftype blkcg_files[] = {
	{
		.name = "stat",
		.seq_show = blkcg_print_stat,
	},
	{ }	/* terminate */
};

904
struct cftype blkcg_legacy_files[] = {
905 906
	{
		.name = "reset_stats",
T
Tejun Heo 已提交
907
		.write_u64 = blkcg_reset_stats,
908
	},
909
	{ }	/* terminate */
910 911
};

912
/**
913
 * blkcg_css_offline - cgroup css_offline callback
914
 * @css: css of interest
915
 *
916 917
 * This function is called when @css is about to go away and responsible
 * for shooting down all blkgs associated with @css.  blkgs should be
918 919 920 921 922
 * removed while holding both q and blkcg locks.  As blkcg lock is nested
 * inside q lock, this function performs reverse double lock dancing.
 *
 * This is the blkcg counterpart of ioc_release_fn().
 */
923
static void blkcg_css_offline(struct cgroup_subsys_state *css)
924
{
925
	struct blkcg *blkcg = css_to_blkcg(css);
926

927
	spin_lock_irq(&blkcg->lock);
928

929
	while (!hlist_empty(&blkcg->blkg_list)) {
T
Tejun Heo 已提交
930 931
		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
						struct blkcg_gq, blkcg_node);
T
Tejun Heo 已提交
932
		struct request_queue *q = blkg->q;
933

934 935 936 937 938 939
		if (spin_trylock(q->queue_lock)) {
			blkg_destroy(blkg);
			spin_unlock(q->queue_lock);
		} else {
			spin_unlock_irq(&blkcg->lock);
			cpu_relax();
940
			spin_lock_irq(&blkcg->lock);
941
		}
942
	}
943

944
	spin_unlock_irq(&blkcg->lock);
945 946

	wb_blkcg_offline(blkcg);
947 948
}

949
static void blkcg_css_free(struct cgroup_subsys_state *css)
950
{
951
	struct blkcg *blkcg = css_to_blkcg(css);
952
	int i;
953

T
Tejun Heo 已提交
954
	mutex_lock(&blkcg_pol_mutex);
955

T
Tejun Heo 已提交
956 957
	list_del(&blkcg->all_blkcgs_node);

958
	for (i = 0; i < BLKCG_MAX_POLS; i++)
959 960 961 962 963
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);

	mutex_unlock(&blkcg_pol_mutex);

964
	kfree(blkcg);
965 966
}

967 968
static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
969
{
T
Tejun Heo 已提交
970
	struct blkcg *blkcg;
971 972
	struct cgroup_subsys_state *ret;
	int i;
973

T
Tejun Heo 已提交
974 975
	mutex_lock(&blkcg_pol_mutex);

976
	if (!parent_css) {
T
Tejun Heo 已提交
977
		blkcg = &blkcg_root;
978 979 980 981 982 983
	} else {
		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
		if (!blkcg) {
			ret = ERR_PTR(-ENOMEM);
			goto free_blkcg;
		}
984 985 986 987 988 989 990 991 992 993 994 995
	}

	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];
		struct blkcg_policy_data *cpd;

		/*
		 * If the policy hasn't been attached yet, wait for it
		 * to be attached before doing anything else. Otherwise,
		 * check if the policy requires any specific per-cgroup
		 * data: if it does, allocate and initialize it.
		 */
996
		if (!pol || !pol->cpd_alloc_fn)
997 998
			continue;

999
		cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1000 1001 1002 1003
		if (!cpd) {
			ret = ERR_PTR(-ENOMEM);
			goto free_pd_blkcg;
		}
1004 1005
		blkcg->cpd[i] = cpd;
		cpd->blkcg = blkcg;
1006
		cpd->plid = i;
1007 1008
		if (pol->cpd_init_fn)
			pol->cpd_init_fn(cpd);
1009
	}
1010 1011

	spin_lock_init(&blkcg->lock);
1012
	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);
1013
	INIT_HLIST_HEAD(&blkcg->blkg_list);
1014 1015 1016
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
T
Tejun Heo 已提交
1017 1018 1019
	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);

	mutex_unlock(&blkcg_pol_mutex);
1020
	return &blkcg->css;
1021 1022 1023

free_pd_blkcg:
	for (i--; i >= 0; i--)
1024 1025
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1026 1027
free_blkcg:
	kfree(blkcg);
T
Tejun Heo 已提交
1028
	mutex_unlock(&blkcg_pol_mutex);
1029
	return ret;
1030 1031
}

1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
/**
 * blkcg_init_queue - initialize blkcg part of request queue
 * @q: request_queue to initialize
 *
 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
 * part of new request_queue @q.
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
int blkcg_init_queue(struct request_queue *q)
{
1044 1045 1046 1047 1048 1049 1050 1051 1052
	struct blkcg_gq *new_blkg, *blkg;
	bool preloaded;
	int ret;

	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
	if (!new_blkg)
		return -ENOMEM;

	preloaded = !radix_tree_preload(GFP_KERNEL);
1053

1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
	/*
	 * Make sure the root blkg exists and count the existing blkgs.  As
	 * @q is bypassing at this point, blkg_lookup_create() can't be
	 * used.  Open code insertion.
	 */
	rcu_read_lock();
	spin_lock_irq(q->queue_lock);
	blkg = blkg_create(&blkcg_root, q, new_blkg);
	spin_unlock_irq(q->queue_lock);
	rcu_read_unlock();

	if (preloaded)
		radix_tree_preload_end();

	if (IS_ERR(blkg)) {
1069
		blkg_free(new_blkg);
1070 1071 1072 1073 1074
		return PTR_ERR(blkg);
	}

	q->root_blkg = blkg;
	q->root_rl.blkg = blkg;
1075

1076 1077 1078 1079 1080 1081 1082
	ret = blk_throtl_init(q);
	if (ret) {
		spin_lock_irq(q->queue_lock);
		blkg_destroy_all(q);
		spin_unlock_irq(q->queue_lock);
	}
	return ret;
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
}

/**
 * blkcg_drain_queue - drain blkcg part of request_queue
 * @q: request_queue to drain
 *
 * Called from blk_drain_queue().  Responsible for draining blkcg part.
 */
void blkcg_drain_queue(struct request_queue *q)
{
	lockdep_assert_held(q->queue_lock);

1095 1096 1097 1098 1099 1100 1101
	/*
	 * @q could be exiting and already have destroyed all blkgs as
	 * indicated by NULL root_blkg.  If so, don't confuse policies.
	 */
	if (!q->root_blkg)
		return;

1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
	blk_throtl_drain(q);
}

/**
 * blkcg_exit_queue - exit and release blkcg part of request_queue
 * @q: request_queue being released
 *
 * Called from blk_release_queue().  Responsible for exiting blkcg part.
 */
void blkcg_exit_queue(struct request_queue *q)
{
1113
	spin_lock_irq(q->queue_lock);
1114
	blkg_destroy_all(q);
1115 1116
	spin_unlock_irq(q->queue_lock);

1117 1118 1119
	blk_throtl_exit(q);
}

1120 1121 1122 1123 1124 1125
/*
 * We cannot support shared io contexts, as we have no mean to support
 * two tasks with the same ioc in two different groups without major rework
 * of the main cic data structures.  For now we allow a task to change
 * its cgroup only if it's the only owner of its ioc.
 */
1126 1127
static int blkcg_can_attach(struct cgroup_subsys_state *css,
			    struct cgroup_taskset *tset)
1128
{
1129
	struct task_struct *task;
1130 1131 1132 1133
	struct io_context *ioc;
	int ret = 0;

	/* task_lock() is needed to avoid races with exit_io_context() */
1134
	cgroup_taskset_for_each(task, tset) {
1135 1136 1137 1138 1139 1140 1141 1142
		task_lock(task);
		ioc = task->io_context;
		if (ioc && atomic_read(&ioc->nr_tasks) > 1)
			ret = -EINVAL;
		task_unlock(task);
		if (ret)
			break;
	}
1143 1144 1145
	return ret;
}

1146
struct cgroup_subsys io_cgrp_subsys = {
1147 1148 1149
	.css_alloc = blkcg_css_alloc,
	.css_offline = blkcg_css_offline,
	.css_free = blkcg_css_free,
T
Tejun Heo 已提交
1150
	.can_attach = blkcg_can_attach,
1151
	.dfl_cftypes = blkcg_files,
1152
	.legacy_cftypes = blkcg_legacy_files,
1153
	.legacy_name = "blkio",
1154 1155 1156 1157 1158 1159 1160 1161
#ifdef CONFIG_MEMCG
	/*
	 * This ensures that, if available, memcg is automatically enabled
	 * together on the default hierarchy so that the owner cgroup can
	 * be retrieved from writeback pages.
	 */
	.depends_on = 1 << memory_cgrp_id,
#endif
1162
};
1163
EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1164

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
/**
 * blkcg_activate_policy - activate a blkcg policy on a request_queue
 * @q: request_queue of interest
 * @pol: blkcg policy to activate
 *
 * Activate @pol on @q.  Requires %GFP_KERNEL context.  @q goes through
 * bypass mode to populate its blkgs with policy_data for @pol.
 *
 * Activation happens with @q bypassed, so nobody would be accessing blkgs
 * from IO path.  Update of each blkg is protected by both queue and blkcg
 * locks so that holding either lock and testing blkcg_policy_enabled() is
 * always enough for dereferencing policy data.
 *
 * The caller is responsible for synchronizing [de]activations and policy
 * [un]registerations.  Returns 0 on success, -errno on failure.
 */
int blkcg_activate_policy(struct request_queue *q,
T
Tejun Heo 已提交
1182
			  const struct blkcg_policy *pol)
1183
{
1184
	struct blkg_policy_data *pd_prealloc = NULL;
1185
	struct blkcg_gq *blkg;
1186
	int ret;
1187 1188 1189 1190 1191

	if (blkcg_policy_enabled(q, pol))
		return 0;

	blk_queue_bypass_start(q);
1192 1193
pd_prealloc:
	if (!pd_prealloc) {
1194
		pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1195
		if (!pd_prealloc) {
1196
			ret = -ENOMEM;
1197
			goto out_bypass_end;
1198 1199 1200 1201 1202 1203
		}
	}

	spin_lock_irq(q->queue_lock);

	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1204 1205 1206 1207
		struct blkg_policy_data *pd;

		if (blkg->pd[pol->plid])
			continue;
1208

1209
		pd = pol->pd_alloc_fn(GFP_NOWAIT, q->node);
1210 1211 1212 1213 1214 1215
		if (!pd)
			swap(pd, pd_prealloc);
		if (!pd) {
			spin_unlock_irq(q->queue_lock);
			goto pd_prealloc;
		}
1216 1217 1218

		blkg->pd[pol->plid] = pd;
		pd->blkg = blkg;
T
Tejun Heo 已提交
1219
		pd->plid = pol->plid;
1220
		if (pol->pd_init_fn)
1221
			pol->pd_init_fn(pd);
1222 1223 1224 1225
	}

	__set_bit(pol->plid, q->blkcg_pols);
	ret = 0;
1226

1227
	spin_unlock_irq(q->queue_lock);
1228
out_bypass_end:
1229
	blk_queue_bypass_end(q);
1230 1231
	if (pd_prealloc)
		pol->pd_free_fn(pd_prealloc);
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
	return ret;
}
EXPORT_SYMBOL_GPL(blkcg_activate_policy);

/**
 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
 * @q: request_queue of interest
 * @pol: blkcg policy to deactivate
 *
 * Deactivate @pol on @q.  Follows the same synchronization rules as
 * blkcg_activate_policy().
 */
void blkcg_deactivate_policy(struct request_queue *q,
T
Tejun Heo 已提交
1245
			     const struct blkcg_policy *pol)
1246
{
T
Tejun Heo 已提交
1247
	struct blkcg_gq *blkg;
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260

	if (!blkcg_policy_enabled(q, pol))
		return;

	blk_queue_bypass_start(q);
	spin_lock_irq(q->queue_lock);

	__clear_bit(pol->plid, q->blkcg_pols);

	list_for_each_entry(blkg, &q->blkg_list, q_node) {
		/* grab blkcg lock too while removing @pd from @blkg */
		spin_lock(&blkg->blkcg->lock);

1261
		if (blkg->pd[pol->plid]) {
1262 1263
			if (pol->pd_offline_fn)
				pol->pd_offline_fn(blkg->pd[pol->plid]);
1264 1265 1266
			pol->pd_free_fn(blkg->pd[pol->plid]);
			blkg->pd[pol->plid] = NULL;
		}
1267 1268 1269 1270 1271 1272 1273 1274 1275

		spin_unlock(&blkg->blkcg->lock);
	}

	spin_unlock_irq(q->queue_lock);
	blk_queue_bypass_end(q);
}
EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);

T
Tejun Heo 已提交
1276
/**
T
Tejun Heo 已提交
1277 1278
 * blkcg_policy_register - register a blkcg policy
 * @pol: blkcg policy to register
T
Tejun Heo 已提交
1279
 *
T
Tejun Heo 已提交
1280 1281
 * Register @pol with blkcg core.  Might sleep and @pol may be modified on
 * successful registration.  Returns 0 on success and -errno on failure.
T
Tejun Heo 已提交
1282
 */
1283
int blkcg_policy_register(struct blkcg_policy *pol)
1284
{
1285
	struct blkcg *blkcg;
T
Tejun Heo 已提交
1286
	int i, ret;
1287

1288
	mutex_lock(&blkcg_pol_register_mutex);
1289 1290
	mutex_lock(&blkcg_pol_mutex);

T
Tejun Heo 已提交
1291 1292 1293
	/* find an empty slot */
	ret = -ENOSPC;
	for (i = 0; i < BLKCG_MAX_POLS; i++)
T
Tejun Heo 已提交
1294
		if (!blkcg_policy[i])
T
Tejun Heo 已提交
1295 1296
			break;
	if (i >= BLKCG_MAX_POLS)
1297
		goto err_unlock;
1298

1299
	/* register @pol */
T
Tejun Heo 已提交
1300
	pol->plid = i;
1301 1302 1303
	blkcg_policy[pol->plid] = pol;

	/* allocate and install cpd's */
1304
	if (pol->cpd_alloc_fn) {
1305 1306 1307
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
			struct blkcg_policy_data *cpd;

1308
			cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1309 1310 1311 1312 1313
			if (!cpd) {
				mutex_unlock(&blkcg_pol_mutex);
				goto err_free_cpds;
			}

1314 1315
			blkcg->cpd[pol->plid] = cpd;
			cpd->blkcg = blkcg;
1316
			cpd->plid = pol->plid;
1317
			pol->cpd_init_fn(cpd);
1318 1319 1320
		}
	}

1321
	mutex_unlock(&blkcg_pol_mutex);
T
Tejun Heo 已提交
1322 1323

	/* everything is in place, add intf files for the new policy */
1324 1325 1326
	if (pol->dfl_cftypes)
		WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
					       pol->dfl_cftypes));
1327
	if (pol->legacy_cftypes)
1328
		WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1329
						  pol->legacy_cftypes));
1330 1331 1332
	mutex_unlock(&blkcg_pol_register_mutex);
	return 0;

1333
err_free_cpds:
1334
	if (pol->cpd_alloc_fn) {
1335
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1336 1337 1338 1339
			if (blkcg->cpd[pol->plid]) {
				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
				blkcg->cpd[pol->plid] = NULL;
			}
1340 1341 1342
		}
	}
	blkcg_policy[pol->plid] = NULL;
1343
err_unlock:
1344
	mutex_unlock(&blkcg_pol_mutex);
1345
	mutex_unlock(&blkcg_pol_register_mutex);
T
Tejun Heo 已提交
1346
	return ret;
1347
}
T
Tejun Heo 已提交
1348
EXPORT_SYMBOL_GPL(blkcg_policy_register);
1349

T
Tejun Heo 已提交
1350
/**
T
Tejun Heo 已提交
1351 1352
 * blkcg_policy_unregister - unregister a blkcg policy
 * @pol: blkcg policy to unregister
T
Tejun Heo 已提交
1353
 *
T
Tejun Heo 已提交
1354
 * Undo blkcg_policy_register(@pol).  Might sleep.
T
Tejun Heo 已提交
1355
 */
T
Tejun Heo 已提交
1356
void blkcg_policy_unregister(struct blkcg_policy *pol)
1357
{
1358 1359
	struct blkcg *blkcg;

1360
	mutex_lock(&blkcg_pol_register_mutex);
1361

T
Tejun Heo 已提交
1362
	if (WARN_ON(blkcg_policy[pol->plid] != pol))
T
Tejun Heo 已提交
1363 1364 1365
		goto out_unlock;

	/* kill the intf files first */
1366 1367
	if (pol->dfl_cftypes)
		cgroup_rm_cftypes(pol->dfl_cftypes);
1368 1369
	if (pol->legacy_cftypes)
		cgroup_rm_cftypes(pol->legacy_cftypes);
1370

1371
	/* remove cpds and unregister */
1372
	mutex_lock(&blkcg_pol_mutex);
1373

1374
	if (pol->cpd_alloc_fn) {
1375
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1376 1377 1378 1379
			if (blkcg->cpd[pol->plid]) {
				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
				blkcg->cpd[pol->plid] = NULL;
			}
1380 1381
		}
	}
T
Tejun Heo 已提交
1382
	blkcg_policy[pol->plid] = NULL;
1383

1384
	mutex_unlock(&blkcg_pol_mutex);
1385 1386
out_unlock:
	mutex_unlock(&blkcg_pol_register_mutex);
1387
}
T
Tejun Heo 已提交
1388
EXPORT_SYMBOL_GPL(blkcg_policy_unregister);