blk-cgroup.c 33.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
12 13 14 15
 *
 * For policy-specific per-blkcg data:
 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
 *                    Arianna Avanzini <avanzini.arianna@gmail.com>
16 17
 */
#include <linux/ioprio.h>
18
#include <linux/kdev_t.h>
19
#include <linux/module.h>
20
#include <linux/err.h>
21
#include <linux/blkdev.h>
22
#include <linux/backing-dev.h>
23
#include <linux/slab.h>
24
#include <linux/genhd.h>
25
#include <linux/delay.h>
T
Tejun Heo 已提交
26
#include <linux/atomic.h>
27
#include <linux/blk-cgroup.h>
28
#include "blk.h"
29

30 31
#define MAX_KEY_LEN 100

32 33 34 35 36 37 38 39
/*
 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
 * policy [un]register operations including cgroup file additions /
 * removals.  Putting cgroup file registration outside blkcg_pol_mutex
 * allows grabbing it from cgroup callbacks.
 */
static DEFINE_MUTEX(blkcg_pol_register_mutex);
40
static DEFINE_MUTEX(blkcg_pol_mutex);
41

42
struct blkcg blkcg_root;
T
Tejun Heo 已提交
43
EXPORT_SYMBOL_GPL(blkcg_root);
44

T
Tejun Heo 已提交
45 46
struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;

T
Tejun Heo 已提交
47
static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
48

T
Tejun Heo 已提交
49 50
static LIST_HEAD(all_blkcgs);		/* protected by blkcg_pol_mutex */

51
static bool blkcg_policy_enabled(struct request_queue *q,
T
Tejun Heo 已提交
52
				 const struct blkcg_policy *pol)
53 54 55 56
{
	return pol && test_bit(pol->plid, q->blkcg_pols);
}

57 58 59 60 61 62
/**
 * blkg_free - free a blkg
 * @blkg: blkg to free
 *
 * Free @blkg which may be partially allocated.
 */
T
Tejun Heo 已提交
63
static void blkg_free(struct blkcg_gq *blkg)
64
{
65
	int i;
66 67 68 69

	if (!blkg)
		return;

70
	for (i = 0; i < BLKCG_MAX_POLS; i++)
71 72
		if (blkg->pd[i])
			blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
73

74 75
	if (blkg->blkcg != &blkcg_root)
		blk_exit_rl(&blkg->rl);
76 77 78

	blkg_rwstat_exit(&blkg->stat_ios);
	blkg_rwstat_exit(&blkg->stat_bytes);
79
	kfree(blkg);
80 81 82 83 84 85
}

/**
 * blkg_alloc - allocate a blkg
 * @blkcg: block cgroup the new blkg is associated with
 * @q: request_queue the new blkg is associated with
86
 * @gfp_mask: allocation mask to use
87
 *
88
 * Allocate a new blkg assocating @blkcg and @q.
89
 */
90 91
static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
				   gfp_t gfp_mask)
92
{
T
Tejun Heo 已提交
93
	struct blkcg_gq *blkg;
94
	int i;
95 96

	/* alloc and init base part */
97
	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
98 99 100
	if (!blkg)
		return NULL;

101 102 103 104
	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
	    blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
		goto err_free;

T
Tejun Heo 已提交
105
	blkg->q = q;
106
	INIT_LIST_HEAD(&blkg->q_node);
107
	blkg->blkcg = blkcg;
108
	atomic_set(&blkg->refcnt, 1);
109

110 111 112 113 114 115 116
	/* root blkg uses @q->root_rl, init rl only for !root blkgs */
	if (blkcg != &blkcg_root) {
		if (blk_init_rl(&blkg->rl, q, gfp_mask))
			goto err_free;
		blkg->rl.blkg = blkg;
	}

T
Tejun Heo 已提交
117
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
T
Tejun Heo 已提交
118
		struct blkcg_policy *pol = blkcg_policy[i];
119
		struct blkg_policy_data *pd;
120

121
		if (!blkcg_policy_enabled(q, pol))
122 123 124
			continue;

		/* alloc per-policy data and attach it to blkg */
125
		pd = pol->pd_alloc_fn(gfp_mask, q->node);
126 127
		if (!pd)
			goto err_free;
128

129 130
		blkg->pd[i] = pd;
		pd->blkg = blkg;
T
Tejun Heo 已提交
131
		pd->plid = i;
132 133
	}

134
	return blkg;
135 136 137 138

err_free:
	blkg_free(blkg);
	return NULL;
139 140
}

T
Tejun Heo 已提交
141 142
struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
				      struct request_queue *q, bool update_hint)
143
{
T
Tejun Heo 已提交
144
	struct blkcg_gq *blkg;
145

146
	/*
147 148 149 150
	 * Hint didn't match.  Look up from the radix tree.  Note that the
	 * hint can only be updated under queue_lock as otherwise @blkg
	 * could have already been removed from blkg_tree.  The caller is
	 * responsible for grabbing queue_lock if @update_hint.
151 152
	 */
	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
153 154 155 156 157
	if (blkg && blkg->q == q) {
		if (update_hint) {
			lockdep_assert_held(q->queue_lock);
			rcu_assign_pointer(blkcg->blkg_hint, blkg);
		}
158
		return blkg;
159
	}
160

161 162
	return NULL;
}
163
EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
164

165 166
/*
 * If @new_blkg is %NULL, this function tries to allocate a new one as
167
 * necessary using %GFP_NOWAIT.  @new_blkg is always consumed on return.
168
 */
169 170 171
static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
				    struct request_queue *q,
				    struct blkcg_gq *new_blkg)
172
{
T
Tejun Heo 已提交
173
	struct blkcg_gq *blkg;
174
	struct bdi_writeback_congested *wb_congested;
175
	int i, ret;
176

177 178 179
	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

180
	/* blkg holds a reference to blkcg */
181
	if (!css_tryget_online(&blkcg->css)) {
182 183
		ret = -EINVAL;
		goto err_free_blkg;
184
	}
185

186
	wb_congested = wb_congested_get_create(&q->backing_dev_info,
187
					       blkcg->css.id, GFP_NOWAIT);
188 189 190 191 192
	if (!wb_congested) {
		ret = -ENOMEM;
		goto err_put_css;
	}

193
	/* allocate */
194
	if (!new_blkg) {
195
		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT);
196
		if (unlikely(!new_blkg)) {
197
			ret = -ENOMEM;
198
			goto err_put_congested;
199 200 201
		}
	}
	blkg = new_blkg;
202
	blkg->wb_congested = wb_congested;
203

204
	/* link parent */
T
Tejun Heo 已提交
205 206 207
	if (blkcg_parent(blkcg)) {
		blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
		if (WARN_ON_ONCE(!blkg->parent)) {
208
			ret = -EINVAL;
209
			goto err_put_congested;
T
Tejun Heo 已提交
210 211 212 213
		}
		blkg_get(blkg->parent);
	}

214 215 216 217 218
	/* invoke per-policy init */
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && pol->pd_init_fn)
219
			pol->pd_init_fn(blkg->pd[i]);
220 221 222
	}

	/* insert */
223
	spin_lock(&blkcg->lock);
224 225 226 227
	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
	if (likely(!ret)) {
		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
		list_add(&blkg->q_node, &q->blkg_list);
228 229 230 231 232

		for (i = 0; i < BLKCG_MAX_POLS; i++) {
			struct blkcg_policy *pol = blkcg_policy[i];

			if (blkg->pd[i] && pol->pd_online_fn)
233
				pol->pd_online_fn(blkg->pd[i]);
234
		}
235
	}
236
	blkg->online = true;
237
	spin_unlock(&blkcg->lock);
238

239
	if (!ret)
240
		return blkg;
241

T
Tejun Heo 已提交
242 243 244 245
	/* @blkg failed fully initialized, use the usual release path */
	blkg_put(blkg);
	return ERR_PTR(ret);

246 247
err_put_congested:
	wb_congested_put(wb_congested);
248
err_put_css:
249
	css_put(&blkcg->css);
250
err_free_blkg:
251
	blkg_free(new_blkg);
252
	return ERR_PTR(ret);
253
}
254

255 256 257 258 259 260
/**
 * blkg_lookup_create - lookup blkg, try to create one if not there
 * @blkcg: blkcg of interest
 * @q: request_queue of interest
 *
 * Lookup blkg for the @blkcg - @q pair.  If it doesn't exist, try to
T
Tejun Heo 已提交
261 262 263
 * create one.  blkg creation is performed recursively from blkcg_root such
 * that all non-root blkg's have access to the parent blkg.  This function
 * should be called under RCU read lock and @q->queue_lock.
264 265 266 267 268
 *
 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
 * value on error.  If @q is dead, returns ERR_PTR(-EINVAL).  If @q is not
 * dead and bypassing, returns ERR_PTR(-EBUSY).
 */
T
Tejun Heo 已提交
269 270
struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
				    struct request_queue *q)
271
{
272 273 274 275 276
	struct blkcg_gq *blkg;

	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

277 278 279 280 281
	/*
	 * This could be the first entry point of blkcg implementation and
	 * we shouldn't allow anything to go through for a bypassing queue.
	 */
	if (unlikely(blk_queue_bypass(q)))
B
Bart Van Assche 已提交
282
		return ERR_PTR(blk_queue_dying(q) ? -EINVAL : -EBUSY);
283 284 285 286 287

	blkg = __blkg_lookup(blkcg, q, true);
	if (blkg)
		return blkg;

T
Tejun Heo 已提交
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	/*
	 * Create blkgs walking down from blkcg_root to @blkcg, so that all
	 * non-root blkgs have access to their parents.
	 */
	while (true) {
		struct blkcg *pos = blkcg;
		struct blkcg *parent = blkcg_parent(blkcg);

		while (parent && !__blkg_lookup(parent, q, false)) {
			pos = parent;
			parent = blkcg_parent(parent);
		}

		blkg = blkg_create(pos, q, NULL);
		if (pos == blkcg || IS_ERR(blkg))
			return blkg;
	}
305
}
306

T
Tejun Heo 已提交
307
static void blkg_destroy(struct blkcg_gq *blkg)
308
{
T
Tejun Heo 已提交
309
	struct blkcg *blkcg = blkg->blkcg;
310
	struct blkcg_gq *parent = blkg->parent;
311
	int i;
312

313
	lockdep_assert_held(blkg->q->queue_lock);
314
	lockdep_assert_held(&blkcg->lock);
315 316

	/* Something wrong if we are trying to remove same group twice */
317
	WARN_ON_ONCE(list_empty(&blkg->q_node));
318
	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
319

320 321 322 323
	for (i = 0; i < BLKCG_MAX_POLS; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];

		if (blkg->pd[i] && pol->pd_offline_fn)
324
			pol->pd_offline_fn(blkg->pd[i]);
325
	}
326 327 328 329 330 331

	if (parent) {
		blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
		blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
	}

332 333
	blkg->online = false;

334
	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
335
	list_del_init(&blkg->q_node);
336
	hlist_del_init_rcu(&blkg->blkcg_node);
337

338 339 340 341 342
	/*
	 * Both setting lookup hint to and clearing it from @blkg are done
	 * under queue_lock.  If it's not pointing to @blkg now, it never
	 * will.  Hint assignment itself can race safely.
	 */
343
	if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
344 345
		rcu_assign_pointer(blkcg->blkg_hint, NULL);

346 347 348 349 350 351 352
	/*
	 * Put the reference taken at the time of creation so that when all
	 * queues are gone, group can be destroyed.
	 */
	blkg_put(blkg);
}

353 354 355 356
/**
 * blkg_destroy_all - destroy all blkgs associated with a request_queue
 * @q: request_queue of interest
 *
357
 * Destroy all blkgs associated with @q.
358
 */
359
static void blkg_destroy_all(struct request_queue *q)
360
{
T
Tejun Heo 已提交
361
	struct blkcg_gq *blkg, *n;
362

363
	lockdep_assert_held(q->queue_lock);
364

365
	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
T
Tejun Heo 已提交
366
		struct blkcg *blkcg = blkg->blkcg;
367

368 369 370
		spin_lock(&blkcg->lock);
		blkg_destroy(blkg);
		spin_unlock(&blkcg->lock);
371 372 373
	}
}

374 375 376 377 378 379 380 381 382
/*
 * A group is RCU protected, but having an rcu lock does not mean that one
 * can access all the fields of blkg and assume these are valid.  For
 * example, don't try to follow throtl_data and request queue links.
 *
 * Having a reference to blkg under an rcu allows accesses to only values
 * local to groups like group stats and group rate limits.
 */
void __blkg_release_rcu(struct rcu_head *rcu_head)
T
Tejun Heo 已提交
383
{
384
	struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
385

T
Tejun Heo 已提交
386
	/* release the blkcg and parent blkg refs this blkg has been holding */
T
Tejun Heo 已提交
387
	css_put(&blkg->blkcg->css);
388
	if (blkg->parent)
T
Tejun Heo 已提交
389
		blkg_put(blkg->parent);
T
Tejun Heo 已提交
390

391 392
	wb_congested_put(blkg->wb_congested);

393
	blkg_free(blkg);
T
Tejun Heo 已提交
394
}
395
EXPORT_SYMBOL_GPL(__blkg_release_rcu);
T
Tejun Heo 已提交
396

397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
/*
 * The next function used by blk_queue_for_each_rl().  It's a bit tricky
 * because the root blkg uses @q->root_rl instead of its own rl.
 */
struct request_list *__blk_queue_next_rl(struct request_list *rl,
					 struct request_queue *q)
{
	struct list_head *ent;
	struct blkcg_gq *blkg;

	/*
	 * Determine the current blkg list_head.  The first entry is
	 * root_rl which is off @q->blkg_list and mapped to the head.
	 */
	if (rl == &q->root_rl) {
		ent = &q->blkg_list;
413 414 415
		/* There are no more block groups, hence no request lists */
		if (list_empty(ent))
			return NULL;
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
	} else {
		blkg = container_of(rl, struct blkcg_gq, rl);
		ent = &blkg->q_node;
	}

	/* walk to the next list_head, skip root blkcg */
	ent = ent->next;
	if (ent == &q->root_blkg->q_node)
		ent = ent->next;
	if (ent == &q->blkg_list)
		return NULL;

	blkg = container_of(ent, struct blkcg_gq, q_node);
	return &blkg->rl;
}

432 433
static int blkcg_reset_stats(struct cgroup_subsys_state *css,
			     struct cftype *cftype, u64 val)
434
{
435
	struct blkcg *blkcg = css_to_blkcg(css);
T
Tejun Heo 已提交
436
	struct blkcg_gq *blkg;
437
	int i;
438

439
	mutex_lock(&blkcg_pol_mutex);
440
	spin_lock_irq(&blkcg->lock);
T
Tejun Heo 已提交
441 442 443 444 445 446

	/*
	 * Note that stat reset is racy - it doesn't synchronize against
	 * stat updates.  This is a debug feature which shouldn't exist
	 * anyway.  If you get hit by a race, retry.
	 */
447
	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
448 449 450
		blkg_rwstat_reset(&blkg->stat_bytes);
		blkg_rwstat_reset(&blkg->stat_ios);

T
Tejun Heo 已提交
451
		for (i = 0; i < BLKCG_MAX_POLS; i++) {
T
Tejun Heo 已提交
452
			struct blkcg_policy *pol = blkcg_policy[i];
453

454 455
			if (blkg->pd[i] && pol->pd_reset_stats_fn)
				pol->pd_reset_stats_fn(blkg->pd[i]);
456
		}
457
	}
458

459
	spin_unlock_irq(&blkcg->lock);
460
	mutex_unlock(&blkcg_pol_mutex);
461 462 463
	return 0;
}

T
Tejun Heo 已提交
464
static const char *blkg_dev_name(struct blkcg_gq *blkg)
465
{
466 467 468 469
	/* some drivers (floppy) instantiate a queue w/o disk registered */
	if (blkg->q->backing_dev_info.dev)
		return dev_name(blkg->q->backing_dev_info.dev);
	return NULL;
470 471
}

472 473 474 475 476 477 478 479 480 481 482
/**
 * blkcg_print_blkgs - helper for printing per-blkg data
 * @sf: seq_file to print to
 * @blkcg: blkcg of interest
 * @prfill: fill function to print out a blkg
 * @pol: policy in question
 * @data: data to be passed to @prfill
 * @show_total: to print out sum of prfill return values or not
 *
 * This function invokes @prfill on each blkg of @blkcg if pd for the
 * policy specified by @pol exists.  @prfill is invoked with @sf, the
483 484 485
 * policy data and @data and the matching queue lock held.  If @show_total
 * is %true, the sum of the return values from @prfill is printed with
 * "Total" label at the end.
486 487 488 489
 *
 * This is to be used to construct print functions for
 * cftype->read_seq_string method.
 */
T
Tejun Heo 已提交
490
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
491 492
		       u64 (*prfill)(struct seq_file *,
				     struct blkg_policy_data *, int),
T
Tejun Heo 已提交
493
		       const struct blkcg_policy *pol, int data,
494
		       bool show_total)
495
{
T
Tejun Heo 已提交
496
	struct blkcg_gq *blkg;
497
	u64 total = 0;
498

499
	rcu_read_lock();
500
	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
501
		spin_lock_irq(blkg->q->queue_lock);
502
		if (blkcg_policy_enabled(blkg->q, pol))
503
			total += prfill(sf, blkg->pd[pol->plid], data);
504 505 506
		spin_unlock_irq(blkg->q->queue_lock);
	}
	rcu_read_unlock();
507 508 509 510

	if (show_total)
		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
}
511
EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
512 513 514 515

/**
 * __blkg_prfill_u64 - prfill helper for a single u64 value
 * @sf: seq_file to print to
516
 * @pd: policy private data of interest
517 518
 * @v: value to print
 *
519
 * Print @v to @sf for the device assocaited with @pd.
520
 */
521
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
522
{
523
	const char *dname = blkg_dev_name(pd->blkg);
524 525 526 527 528 529 530

	if (!dname)
		return 0;

	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
	return v;
}
531
EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
532 533 534 535

/**
 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
 * @sf: seq_file to print to
536
 * @pd: policy private data of interest
537 538
 * @rwstat: rwstat to print
 *
539
 * Print @rwstat to @sf for the device assocaited with @pd.
540
 */
541
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
542
			 const struct blkg_rwstat *rwstat)
543 544 545 546 547 548 549
{
	static const char *rwstr[] = {
		[BLKG_RWSTAT_READ]	= "Read",
		[BLKG_RWSTAT_WRITE]	= "Write",
		[BLKG_RWSTAT_SYNC]	= "Sync",
		[BLKG_RWSTAT_ASYNC]	= "Async",
	};
550
	const char *dname = blkg_dev_name(pd->blkg);
551 552 553 554 555 556 557 558
	u64 v;
	int i;

	if (!dname)
		return 0;

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
T
Tejun Heo 已提交
559
			   (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
560

T
Tejun Heo 已提交
561 562
	v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
		atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
563 564 565
	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
	return v;
}
T
Tejun Heo 已提交
566
EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
567

568 569 570
/**
 * blkg_prfill_stat - prfill callback for blkg_stat
 * @sf: seq_file to print to
571 572
 * @pd: policy private data of interest
 * @off: offset to the blkg_stat in @pd
573 574 575
 *
 * prfill callback for printing a blkg_stat.
 */
576
u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
577
{
578
	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
579
}
580
EXPORT_SYMBOL_GPL(blkg_prfill_stat);
581

582 583 584
/**
 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
 * @sf: seq_file to print to
585 586
 * @pd: policy private data of interest
 * @off: offset to the blkg_rwstat in @pd
587 588 589
 *
 * prfill callback for printing a blkg_rwstat.
 */
590 591
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
		       int off)
592
{
593
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
594

595
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
596
}
597
EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
598

599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
				    struct blkg_policy_data *pd, int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_bytes.
 * cftype->private must be set to the blkcg_policy.
 */
int blkg_print_stat_bytes(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);

/**
 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
 * @sf: seq_file to print to
 * @v: unused
 *
 * To be used as cftype->seq_show to print blkg->stat_ios.  cftype->private
 * must be set to the blkcg_policy.
 */
int blkg_print_stat_ios(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios);

static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
					      struct blkg_policy_data *pd,
					      int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
							      NULL, off);
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/**
 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_bytes), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);

/**
 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
 * @sf: seq_file to print to
 * @v: unused
 */
int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
{
	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
			  blkg_prfill_rwstat_field_recursive,
			  (void *)seq_cft(sf)->private,
			  offsetof(struct blkcg_gq, stat_ios), true);
	return 0;
}
EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);

680 681
/**
 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
682 683 684
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_stat
 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
685
 *
686 687 688 689 690 691
 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
 * at @off bytes into @blkg's blkg_policy_data of the policy.
692
 */
693 694
u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
			    struct blkcg_policy *pol, int off)
695 696
{
	struct blkcg_gq *pos_blkg;
697
	struct cgroup_subsys_state *pos_css;
698
	u64 sum = 0;
699

700
	lockdep_assert_held(blkg->q->queue_lock);
701 702

	rcu_read_lock();
703 704 705 706 707
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
		struct blkg_stat *stat;

		if (!pos_blkg->online)
			continue;
708

709 710 711 712 713 714
		if (pol)
			stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			stat = (void *)blkg + off;

		sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
715 716 717 718 719 720 721 722 723
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);

/**
 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
724 725 726
 * @blkg: blkg of interest
 * @pol: blkcg_policy which contains the blkg_rwstat
 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
727
 *
728 729 730 731 732 733
 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
 * online descendants and their aux counts.  The caller must be holding the
 * queue lock for online tests.
 *
 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
 * is at @off bytes into @blkg's blkg_policy_data of the policy.
734
 */
735 736
struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
					     struct blkcg_policy *pol, int off)
737 738
{
	struct blkcg_gq *pos_blkg;
739
	struct cgroup_subsys_state *pos_css;
740
	struct blkg_rwstat sum = { };
741 742
	int i;

743
	lockdep_assert_held(blkg->q->queue_lock);
744 745

	rcu_read_lock();
746 747
	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
		struct blkg_rwstat *rwstat, tmp;
748 749 750 751

		if (!pos_blkg->online)
			continue;

752 753 754 755 756
		if (pol)
			rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
		else
			rwstat = (void *)pos_blkg + off;

757 758 759
		tmp = blkg_rwstat_read(rwstat);

		for (i = 0; i < BLKG_RWSTAT_NR; i++)
T
Tejun Heo 已提交
760 761 762
			atomic64_add(atomic64_read(&tmp.aux_cnt[i]) +
				     atomic64_read(&rwstat->aux_cnt[i]),
				     &sum.aux_cnt[i]);
763 764 765 766 767 768 769
	}
	rcu_read_unlock();

	return sum;
}
EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);

770 771 772
/**
 * blkg_conf_prep - parse and prepare for per-blkg config update
 * @blkcg: target block cgroup
773
 * @pol: target policy
774 775 776 777 778
 * @input: input string
 * @ctx: blkg_conf_ctx to be filled
 *
 * Parse per-blkg config update from @input and initialize @ctx with the
 * result.  @ctx->blkg points to the blkg to be updated and @ctx->v the new
779 780
 * value.  This function returns with RCU read lock and queue lock held and
 * must be paired with blkg_conf_finish().
781
 */
T
Tejun Heo 已提交
782 783
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
		   const char *input, struct blkg_conf_ctx *ctx)
784
	__acquires(rcu) __acquires(disk->queue->queue_lock)
785
{
786
	struct gendisk *disk;
T
Tejun Heo 已提交
787
	struct blkcg_gq *blkg;
T
Tejun Heo 已提交
788 789 790
	unsigned int major, minor;
	unsigned long long v;
	int part, ret;
791

T
Tejun Heo 已提交
792 793
	if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3)
		return -EINVAL;
794

T
Tejun Heo 已提交
795
	disk = get_gendisk(MKDEV(major, minor), &part);
796
	if (!disk)
T
Tejun Heo 已提交
797
		return -EINVAL;
798 799 800 801
	if (part) {
		put_disk(disk);
		return -EINVAL;
	}
802 803

	rcu_read_lock();
T
Tejun Heo 已提交
804
	spin_lock_irq(disk->queue->queue_lock);
805

806
	if (blkcg_policy_enabled(disk->queue, pol))
807
		blkg = blkg_lookup_create(blkcg, disk->queue);
808 809
	else
		blkg = ERR_PTR(-EINVAL);
810

T
Tejun Heo 已提交
811 812
	if (IS_ERR(blkg)) {
		ret = PTR_ERR(blkg);
813
		rcu_read_unlock();
814
		spin_unlock_irq(disk->queue->queue_lock);
815 816 817 818 819 820 821 822 823 824
		put_disk(disk);
		/*
		 * If queue was bypassing, we should retry.  Do so after a
		 * short msleep().  It isn't strictly necessary but queue
		 * can be bypassing for some time and it's always nice to
		 * avoid busy looping.
		 */
		if (ret == -EBUSY) {
			msleep(10);
			ret = restart_syscall();
825
		}
T
Tejun Heo 已提交
826
		return ret;
827
	}
828 829 830

	ctx->disk = disk;
	ctx->blkg = blkg;
T
Tejun Heo 已提交
831 832
	ctx->v = v;
	return 0;
833
}
834
EXPORT_SYMBOL_GPL(blkg_conf_prep);
835

836 837 838 839 840 841 842
/**
 * blkg_conf_finish - finish up per-blkg config update
 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
 *
 * Finish up after per-blkg config update.  This function must be paired
 * with blkg_conf_prep().
 */
843
void blkg_conf_finish(struct blkg_conf_ctx *ctx)
844
	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
845
{
846
	spin_unlock_irq(ctx->disk->queue->queue_lock);
847 848
	rcu_read_unlock();
	put_disk(ctx->disk);
849
}
850
EXPORT_SYMBOL_GPL(blkg_conf_finish);
851

T
Tejun Heo 已提交
852
struct cftype blkcg_files[] = {
853 854
	{
		.name = "reset_stats",
T
Tejun Heo 已提交
855
		.write_u64 = blkcg_reset_stats,
856
	},
857
	{ }	/* terminate */
858 859
};

860
/**
861
 * blkcg_css_offline - cgroup css_offline callback
862
 * @css: css of interest
863
 *
864 865
 * This function is called when @css is about to go away and responsible
 * for shooting down all blkgs associated with @css.  blkgs should be
866 867 868 869 870
 * removed while holding both q and blkcg locks.  As blkcg lock is nested
 * inside q lock, this function performs reverse double lock dancing.
 *
 * This is the blkcg counterpart of ioc_release_fn().
 */
871
static void blkcg_css_offline(struct cgroup_subsys_state *css)
872
{
873
	struct blkcg *blkcg = css_to_blkcg(css);
874

875
	spin_lock_irq(&blkcg->lock);
876

877
	while (!hlist_empty(&blkcg->blkg_list)) {
T
Tejun Heo 已提交
878 879
		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
						struct blkcg_gq, blkcg_node);
T
Tejun Heo 已提交
880
		struct request_queue *q = blkg->q;
881

882 883 884 885 886 887
		if (spin_trylock(q->queue_lock)) {
			blkg_destroy(blkg);
			spin_unlock(q->queue_lock);
		} else {
			spin_unlock_irq(&blkcg->lock);
			cpu_relax();
888
			spin_lock_irq(&blkcg->lock);
889
		}
890
	}
891

892
	spin_unlock_irq(&blkcg->lock);
893 894

	wb_blkcg_offline(blkcg);
895 896
}

897
static void blkcg_css_free(struct cgroup_subsys_state *css)
898
{
899
	struct blkcg *blkcg = css_to_blkcg(css);
900
	int i;
901

T
Tejun Heo 已提交
902
	mutex_lock(&blkcg_pol_mutex);
903

T
Tejun Heo 已提交
904 905
	list_del(&blkcg->all_blkcgs_node);

906
	for (i = 0; i < BLKCG_MAX_POLS; i++)
907 908 909 910 911
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);

	mutex_unlock(&blkcg_pol_mutex);

912
	kfree(blkcg);
913 914
}

915 916
static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
917
{
T
Tejun Heo 已提交
918
	struct blkcg *blkcg;
919 920
	struct cgroup_subsys_state *ret;
	int i;
921

T
Tejun Heo 已提交
922 923
	mutex_lock(&blkcg_pol_mutex);

924
	if (!parent_css) {
T
Tejun Heo 已提交
925
		blkcg = &blkcg_root;
926 927 928 929 930 931
	} else {
		blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
		if (!blkcg) {
			ret = ERR_PTR(-ENOMEM);
			goto free_blkcg;
		}
932 933 934 935 936 937 938 939 940 941 942 943
	}

	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
		struct blkcg_policy *pol = blkcg_policy[i];
		struct blkcg_policy_data *cpd;

		/*
		 * If the policy hasn't been attached yet, wait for it
		 * to be attached before doing anything else. Otherwise,
		 * check if the policy requires any specific per-cgroup
		 * data: if it does, allocate and initialize it.
		 */
944
		if (!pol || !pol->cpd_alloc_fn)
945 946
			continue;

947
		cpd = pol->cpd_alloc_fn(GFP_KERNEL);
948 949 950 951
		if (!cpd) {
			ret = ERR_PTR(-ENOMEM);
			goto free_pd_blkcg;
		}
952 953
		blkcg->cpd[i] = cpd;
		cpd->blkcg = blkcg;
954
		cpd->plid = i;
955 956
		if (pol->cpd_init_fn)
			pol->cpd_init_fn(cpd);
957
	}
958 959

	spin_lock_init(&blkcg->lock);
960
	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);
961
	INIT_HLIST_HEAD(&blkcg->blkg_list);
962 963 964
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&blkcg->cgwb_list);
#endif
T
Tejun Heo 已提交
965 966 967
	list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);

	mutex_unlock(&blkcg_pol_mutex);
968
	return &blkcg->css;
969 970 971

free_pd_blkcg:
	for (i--; i >= 0; i--)
972 973
		if (blkcg->cpd[i])
			blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
974 975
free_blkcg:
	kfree(blkcg);
T
Tejun Heo 已提交
976
	mutex_unlock(&blkcg_pol_mutex);
977
	return ret;
978 979
}

980 981 982 983 984 985 986 987 988 989 990 991
/**
 * blkcg_init_queue - initialize blkcg part of request queue
 * @q: request_queue to initialize
 *
 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
 * part of new request_queue @q.
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
int blkcg_init_queue(struct request_queue *q)
{
992 993 994 995 996 997 998 999 1000
	struct blkcg_gq *new_blkg, *blkg;
	bool preloaded;
	int ret;

	new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
	if (!new_blkg)
		return -ENOMEM;

	preloaded = !radix_tree_preload(GFP_KERNEL);
1001

1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
	/*
	 * Make sure the root blkg exists and count the existing blkgs.  As
	 * @q is bypassing at this point, blkg_lookup_create() can't be
	 * used.  Open code insertion.
	 */
	rcu_read_lock();
	spin_lock_irq(q->queue_lock);
	blkg = blkg_create(&blkcg_root, q, new_blkg);
	spin_unlock_irq(q->queue_lock);
	rcu_read_unlock();

	if (preloaded)
		radix_tree_preload_end();

	if (IS_ERR(blkg)) {
1017
		blkg_free(new_blkg);
1018 1019 1020 1021 1022
		return PTR_ERR(blkg);
	}

	q->root_blkg = blkg;
	q->root_rl.blkg = blkg;
1023

1024 1025 1026 1027 1028 1029 1030
	ret = blk_throtl_init(q);
	if (ret) {
		spin_lock_irq(q->queue_lock);
		blkg_destroy_all(q);
		spin_unlock_irq(q->queue_lock);
	}
	return ret;
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
}

/**
 * blkcg_drain_queue - drain blkcg part of request_queue
 * @q: request_queue to drain
 *
 * Called from blk_drain_queue().  Responsible for draining blkcg part.
 */
void blkcg_drain_queue(struct request_queue *q)
{
	lockdep_assert_held(q->queue_lock);

1043 1044 1045 1046 1047 1048 1049
	/*
	 * @q could be exiting and already have destroyed all blkgs as
	 * indicated by NULL root_blkg.  If so, don't confuse policies.
	 */
	if (!q->root_blkg)
		return;

1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	blk_throtl_drain(q);
}

/**
 * blkcg_exit_queue - exit and release blkcg part of request_queue
 * @q: request_queue being released
 *
 * Called from blk_release_queue().  Responsible for exiting blkcg part.
 */
void blkcg_exit_queue(struct request_queue *q)
{
1061
	spin_lock_irq(q->queue_lock);
1062
	blkg_destroy_all(q);
1063 1064
	spin_unlock_irq(q->queue_lock);

1065 1066 1067
	blk_throtl_exit(q);
}

1068 1069 1070 1071 1072 1073
/*
 * We cannot support shared io contexts, as we have no mean to support
 * two tasks with the same ioc in two different groups without major rework
 * of the main cic data structures.  For now we allow a task to change
 * its cgroup only if it's the only owner of its ioc.
 */
1074 1075
static int blkcg_can_attach(struct cgroup_subsys_state *css,
			    struct cgroup_taskset *tset)
1076
{
1077
	struct task_struct *task;
1078 1079 1080 1081
	struct io_context *ioc;
	int ret = 0;

	/* task_lock() is needed to avoid races with exit_io_context() */
1082
	cgroup_taskset_for_each(task, tset) {
1083 1084 1085 1086 1087 1088 1089 1090
		task_lock(task);
		ioc = task->io_context;
		if (ioc && atomic_read(&ioc->nr_tasks) > 1)
			ret = -EINVAL;
		task_unlock(task);
		if (ret)
			break;
	}
1091 1092 1093
	return ret;
}

1094
struct cgroup_subsys blkio_cgrp_subsys = {
1095 1096 1097
	.css_alloc = blkcg_css_alloc,
	.css_offline = blkcg_css_offline,
	.css_free = blkcg_css_free,
T
Tejun Heo 已提交
1098
	.can_attach = blkcg_can_attach,
1099
	.legacy_cftypes = blkcg_files,
1100 1101 1102 1103 1104 1105 1106 1107
#ifdef CONFIG_MEMCG
	/*
	 * This ensures that, if available, memcg is automatically enabled
	 * together on the default hierarchy so that the owner cgroup can
	 * be retrieved from writeback pages.
	 */
	.depends_on = 1 << memory_cgrp_id,
#endif
1108
};
1109
EXPORT_SYMBOL_GPL(blkio_cgrp_subsys);
1110

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
/**
 * blkcg_activate_policy - activate a blkcg policy on a request_queue
 * @q: request_queue of interest
 * @pol: blkcg policy to activate
 *
 * Activate @pol on @q.  Requires %GFP_KERNEL context.  @q goes through
 * bypass mode to populate its blkgs with policy_data for @pol.
 *
 * Activation happens with @q bypassed, so nobody would be accessing blkgs
 * from IO path.  Update of each blkg is protected by both queue and blkcg
 * locks so that holding either lock and testing blkcg_policy_enabled() is
 * always enough for dereferencing policy data.
 *
 * The caller is responsible for synchronizing [de]activations and policy
 * [un]registerations.  Returns 0 on success, -errno on failure.
 */
int blkcg_activate_policy(struct request_queue *q,
T
Tejun Heo 已提交
1128
			  const struct blkcg_policy *pol)
1129
{
1130
	struct blkg_policy_data *pd_prealloc = NULL;
1131
	struct blkcg_gq *blkg;
1132
	int ret;
1133 1134 1135 1136 1137

	if (blkcg_policy_enabled(q, pol))
		return 0;

	blk_queue_bypass_start(q);
1138 1139
pd_prealloc:
	if (!pd_prealloc) {
1140
		pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1141
		if (!pd_prealloc) {
1142
			ret = -ENOMEM;
1143
			goto out_bypass_end;
1144 1145 1146 1147 1148 1149
		}
	}

	spin_lock_irq(q->queue_lock);

	list_for_each_entry(blkg, &q->blkg_list, q_node) {
1150 1151 1152 1153
		struct blkg_policy_data *pd;

		if (blkg->pd[pol->plid])
			continue;
1154

1155
		pd = pol->pd_alloc_fn(GFP_NOWAIT, q->node);
1156 1157 1158 1159 1160 1161
		if (!pd)
			swap(pd, pd_prealloc);
		if (!pd) {
			spin_unlock_irq(q->queue_lock);
			goto pd_prealloc;
		}
1162 1163 1164

		blkg->pd[pol->plid] = pd;
		pd->blkg = blkg;
T
Tejun Heo 已提交
1165
		pd->plid = pol->plid;
1166
		if (pol->pd_init_fn)
1167
			pol->pd_init_fn(pd);
1168 1169 1170 1171
	}

	__set_bit(pol->plid, q->blkcg_pols);
	ret = 0;
1172

1173
	spin_unlock_irq(q->queue_lock);
1174
out_bypass_end:
1175
	blk_queue_bypass_end(q);
1176 1177
	if (pd_prealloc)
		pol->pd_free_fn(pd_prealloc);
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
	return ret;
}
EXPORT_SYMBOL_GPL(blkcg_activate_policy);

/**
 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
 * @q: request_queue of interest
 * @pol: blkcg policy to deactivate
 *
 * Deactivate @pol on @q.  Follows the same synchronization rules as
 * blkcg_activate_policy().
 */
void blkcg_deactivate_policy(struct request_queue *q,
T
Tejun Heo 已提交
1191
			     const struct blkcg_policy *pol)
1192
{
T
Tejun Heo 已提交
1193
	struct blkcg_gq *blkg;
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206

	if (!blkcg_policy_enabled(q, pol))
		return;

	blk_queue_bypass_start(q);
	spin_lock_irq(q->queue_lock);

	__clear_bit(pol->plid, q->blkcg_pols);

	list_for_each_entry(blkg, &q->blkg_list, q_node) {
		/* grab blkcg lock too while removing @pd from @blkg */
		spin_lock(&blkg->blkcg->lock);

1207
		if (blkg->pd[pol->plid]) {
1208 1209
			if (pol->pd_offline_fn)
				pol->pd_offline_fn(blkg->pd[pol->plid]);
1210 1211 1212
			pol->pd_free_fn(blkg->pd[pol->plid]);
			blkg->pd[pol->plid] = NULL;
		}
1213 1214 1215 1216 1217 1218 1219 1220 1221

		spin_unlock(&blkg->blkcg->lock);
	}

	spin_unlock_irq(q->queue_lock);
	blk_queue_bypass_end(q);
}
EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);

T
Tejun Heo 已提交
1222
/**
T
Tejun Heo 已提交
1223 1224
 * blkcg_policy_register - register a blkcg policy
 * @pol: blkcg policy to register
T
Tejun Heo 已提交
1225
 *
T
Tejun Heo 已提交
1226 1227
 * Register @pol with blkcg core.  Might sleep and @pol may be modified on
 * successful registration.  Returns 0 on success and -errno on failure.
T
Tejun Heo 已提交
1228
 */
1229
int blkcg_policy_register(struct blkcg_policy *pol)
1230
{
1231
	struct blkcg *blkcg;
T
Tejun Heo 已提交
1232
	int i, ret;
1233

1234
	mutex_lock(&blkcg_pol_register_mutex);
1235 1236
	mutex_lock(&blkcg_pol_mutex);

T
Tejun Heo 已提交
1237 1238 1239
	/* find an empty slot */
	ret = -ENOSPC;
	for (i = 0; i < BLKCG_MAX_POLS; i++)
T
Tejun Heo 已提交
1240
		if (!blkcg_policy[i])
T
Tejun Heo 已提交
1241 1242
			break;
	if (i >= BLKCG_MAX_POLS)
1243
		goto err_unlock;
1244

1245
	/* register @pol */
T
Tejun Heo 已提交
1246
	pol->plid = i;
1247 1248 1249
	blkcg_policy[pol->plid] = pol;

	/* allocate and install cpd's */
1250
	if (pol->cpd_alloc_fn) {
1251 1252 1253
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
			struct blkcg_policy_data *cpd;

1254
			cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1255 1256 1257 1258 1259
			if (!cpd) {
				mutex_unlock(&blkcg_pol_mutex);
				goto err_free_cpds;
			}

1260 1261
			blkcg->cpd[pol->plid] = cpd;
			cpd->blkcg = blkcg;
1262
			cpd->plid = pol->plid;
1263
			pol->cpd_init_fn(cpd);
1264 1265 1266
		}
	}

1267
	mutex_unlock(&blkcg_pol_mutex);
T
Tejun Heo 已提交
1268 1269

	/* everything is in place, add intf files for the new policy */
T
Tejun Heo 已提交
1270
	if (pol->cftypes)
1271 1272
		WARN_ON(cgroup_add_legacy_cftypes(&blkio_cgrp_subsys,
						  pol->cftypes));
1273 1274 1275
	mutex_unlock(&blkcg_pol_register_mutex);
	return 0;

1276
err_free_cpds:
1277
	if (pol->cpd_alloc_fn) {
1278
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1279 1280 1281 1282
			if (blkcg->cpd[pol->plid]) {
				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
				blkcg->cpd[pol->plid] = NULL;
			}
1283 1284 1285
		}
	}
	blkcg_policy[pol->plid] = NULL;
1286
err_unlock:
1287
	mutex_unlock(&blkcg_pol_mutex);
1288
	mutex_unlock(&blkcg_pol_register_mutex);
T
Tejun Heo 已提交
1289
	return ret;
1290
}
T
Tejun Heo 已提交
1291
EXPORT_SYMBOL_GPL(blkcg_policy_register);
1292

T
Tejun Heo 已提交
1293
/**
T
Tejun Heo 已提交
1294 1295
 * blkcg_policy_unregister - unregister a blkcg policy
 * @pol: blkcg policy to unregister
T
Tejun Heo 已提交
1296
 *
T
Tejun Heo 已提交
1297
 * Undo blkcg_policy_register(@pol).  Might sleep.
T
Tejun Heo 已提交
1298
 */
T
Tejun Heo 已提交
1299
void blkcg_policy_unregister(struct blkcg_policy *pol)
1300
{
1301 1302
	struct blkcg *blkcg;

1303
	mutex_lock(&blkcg_pol_register_mutex);
1304

T
Tejun Heo 已提交
1305
	if (WARN_ON(blkcg_policy[pol->plid] != pol))
T
Tejun Heo 已提交
1306 1307 1308
		goto out_unlock;

	/* kill the intf files first */
T
Tejun Heo 已提交
1309
	if (pol->cftypes)
1310
		cgroup_rm_cftypes(pol->cftypes);
1311

1312
	/* remove cpds and unregister */
1313
	mutex_lock(&blkcg_pol_mutex);
1314

1315
	if (pol->cpd_alloc_fn) {
1316
		list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1317 1318 1319 1320
			if (blkcg->cpd[pol->plid]) {
				pol->cpd_free_fn(blkcg->cpd[pol->plid]);
				blkcg->cpd[pol->plid] = NULL;
			}
1321 1322
		}
	}
T
Tejun Heo 已提交
1323
	blkcg_policy[pol->plid] = NULL;
1324

1325
	mutex_unlock(&blkcg_pol_mutex);
1326 1327
out_unlock:
	mutex_unlock(&blkcg_pol_register_mutex);
1328
}
T
Tejun Heo 已提交
1329
EXPORT_SYMBOL_GPL(blkcg_policy_unregister);