cgroup.c 132.4 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

#include <linux/cgroup.h>
30
#include <linux/cred.h>
31
#include <linux/ctype.h>
32
#include <linux/errno.h>
33
#include <linux/init_task.h>
34 35 36 37 38 39
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
40
#include <linux/proc_fs.h>
41 42 43 44
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
45
#include <linux/rwsem.h>
46
#include <linux/string.h>
47
#include <linux/sort.h>
48
#include <linux/kmod.h>
B
Balbir Singh 已提交
49 50
#include <linux/delayacct.h>
#include <linux/cgroupstats.h>
51
#include <linux/hashtable.h>
L
Li Zefan 已提交
52
#include <linux/pid_namespace.h>
53
#include <linux/idr.h>
54
#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
55
#include <linux/kthread.h>
T
Tejun Heo 已提交
56
#include <linux/delay.h>
B
Balbir Singh 已提交
57

A
Arun Sharma 已提交
58
#include <linux/atomic.h>
59

60 61 62 63 64 65 66 67
/*
 * pidlists linger the following amount before being destroyed.  The goal
 * is avoiding frequent destruction in the middle of consecutive read calls
 * Expiring in the middle is a performance problem not a correctness one.
 * 1 sec should be enough.
 */
#define CGROUP_PIDLIST_DESTROY_DELAY	HZ

T
Tejun Heo 已提交
68 69 70
#define CGROUP_FILE_NAME_MAX		(MAX_CGROUP_TYPE_NAMELEN +	\
					 MAX_CFTYPE_NAME + 2)

T
Tejun Heo 已提交
71 72 73 74 75 76 77 78 79
/*
 * cgroup_tree_mutex nests above cgroup_mutex and protects cftypes, file
 * creation/removal and hierarchy changing operations including cgroup
 * creation, removal, css association and controller rebinding.  This outer
 * lock is needed mainly to resolve the circular dependency between kernfs
 * active ref and cgroup_mutex.  cgroup_tree_mutex nests above both.
 */
static DEFINE_MUTEX(cgroup_tree_mutex);

T
Tejun Heo 已提交
80 81 82 83
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
84 85
 * css_set_rwsem protects task->cgroups pointer, the list of css_set
 * objects, and the chain of tasks off each css_set.
T
Tejun Heo 已提交
86
 *
87 88
 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 * cgroup.h can use them for lockdep annotations.
T
Tejun Heo 已提交
89
 */
T
Tejun Heo 已提交
90 91
#ifdef CONFIG_PROVE_RCU
DEFINE_MUTEX(cgroup_mutex);
92 93 94
DECLARE_RWSEM(css_set_rwsem);
EXPORT_SYMBOL_GPL(cgroup_mutex);
EXPORT_SYMBOL_GPL(css_set_rwsem);
T
Tejun Heo 已提交
95
#else
96
static DEFINE_MUTEX(cgroup_mutex);
97
static DECLARE_RWSEM(css_set_rwsem);
T
Tejun Heo 已提交
98 99
#endif

100 101 102 103 104
/*
 * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
 * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
 */
static DEFINE_SPINLOCK(release_agent_path_lock);
105

T
Tejun Heo 已提交
106
#define cgroup_assert_mutexes_or_rcu_locked()				\
107
	rcu_lockdep_assert(rcu_read_lock_held() ||			\
T
Tejun Heo 已提交
108
			   lockdep_is_held(&cgroup_tree_mutex) ||	\
109
			   lockdep_is_held(&cgroup_mutex),		\
T
Tejun Heo 已提交
110
			   "cgroup_[tree_]mutex or RCU read lock required");
111

112 113 114 115 116 117 118 119
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

120 121 122 123 124 125
/*
 * pidlist destructions need to be flushed on cgroup destruction.  Use a
 * separate workqueue as flush domain.
 */
static struct workqueue_struct *cgroup_pidlist_destroy_wq;

T
Tejun Heo 已提交
126
/* generate an array of cgroup subsystem pointers */
127
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
T
Tejun Heo 已提交
128
static struct cgroup_subsys *cgroup_subsys[] = {
129 130
#include <linux/cgroup_subsys.h>
};
131 132 133 134 135
#undef SUBSYS

/* array of cgroup subsystem names */
#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
static const char *cgroup_subsys_name[] = {
136 137
#include <linux/cgroup_subsys.h>
};
138
#undef SUBSYS
139 140

/*
141
 * The default hierarchy, reserved for the subsystems that are otherwise
142 143
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
144
 */
T
Tejun Heo 已提交
145
struct cgroup_root cgrp_dfl_root;
146

T
Tejun Heo 已提交
147 148 149 150 151
/*
 * The default hierarchy always exists but is hidden until mounted for the
 * first time.  This is for backward compatibility.
 */
static bool cgrp_dfl_root_visible;
152 153 154

/* The list of hierarchy roots */

155 156
static LIST_HEAD(cgroup_roots);
static int cgroup_root_count;
157

T
Tejun Heo 已提交
158
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
159
static DEFINE_IDR(cgroup_hierarchy_idr);
160

161 162 163 164 165
/*
 * Assign a monotonically increasing serial number to cgroups.  It
 * guarantees cgroups with bigger numbers are newer than those with smaller
 * numbers.  Also, as cgroups are always appended to the parent's
 * ->children list, it guarantees that sibling cgroups are always sorted in
166 167
 * the ascending serial number order on the list.  Protected by
 * cgroup_mutex.
168
 */
169
static u64 cgroup_serial_nr_next = 1;
170

171
/* This flag indicates whether tasks in the fork and exit paths should
L
Li Zefan 已提交
172 173 174
 * check for fork/exit handlers to call. This avoids us having to do
 * extra work in the fork/exit path if none of the subsystems need to
 * be called.
175
 */
176
static int need_forkexit_callback __read_mostly;
177

178 179
static struct cftype cgroup_base_files[];

180
static void cgroup_put(struct cgroup *cgrp);
181
static int rebind_subsystems(struct cgroup_root *dst_root,
182
			     unsigned long ss_mask);
183
static void cgroup_destroy_css_killed(struct cgroup *cgrp);
184
static int cgroup_destroy_locked(struct cgroup *cgrp);
185 186
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add);
187
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
188

T
Tejun Heo 已提交
189 190 191
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
192
 * @ss: the subsystem of interest (%NULL returns the dummy_css)
T
Tejun Heo 已提交
193
 *
194 195 196 197 198
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
199 200
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
201
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
202
{
203
	if (ss)
204
		return rcu_dereference_check(cgrp->subsys[ss->id],
T
Tejun Heo 已提交
205 206
					lockdep_is_held(&cgroup_tree_mutex) ||
					lockdep_is_held(&cgroup_mutex));
207 208
	else
		return &cgrp->dummy_css;
T
Tejun Heo 已提交
209
}
210

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
/**
 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
 * @ss: the subsystem of interest (%NULL returns the dummy_css)
 *
 * Similar to cgroup_css() but returns the effctive css, which is defined
 * as the matching css of the nearest ancestor including self which has @ss
 * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
 * function is guaranteed to return non-NULL css.
 */
static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
						struct cgroup_subsys *ss)
{
	lockdep_assert_held(&cgroup_mutex);

	if (!ss)
		return &cgrp->dummy_css;

	if (!(cgrp->root->subsys_mask & (1 << ss->id)))
		return NULL;

	while (cgrp->parent &&
	       !(cgrp->parent->child_subsys_mask & (1 << ss->id)))
		cgrp = cgrp->parent;

	return cgroup_css(cgrp, ss);
}

239
/* convenient tests for these bits */
240
static inline bool cgroup_is_dead(const struct cgroup *cgrp)
241
{
242
	return test_bit(CGRP_DEAD, &cgrp->flags);
243 244
}

245 246
struct cgroup_subsys_state *seq_css(struct seq_file *seq)
{
T
Tejun Heo 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
	struct kernfs_open_file *of = seq->private;
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = seq_cft(seq);

	/*
	 * This is open and unprotected implementation of cgroup_css().
	 * seq_css() is only called from a kernfs file operation which has
	 * an active reference on the file.  Because all the subsystem
	 * files are drained before a css is disassociated with a cgroup,
	 * the matching css from the cgroup's subsys table is guaranteed to
	 * be and stay valid until the enclosing operation is complete.
	 */
	if (cft->ss)
		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
	else
		return &cgrp->dummy_css;
263 264 265
}
EXPORT_SYMBOL_GPL(seq_css);

266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
/**
 * cgroup_is_descendant - test ancestry
 * @cgrp: the cgroup to be tested
 * @ancestor: possible ancestor of @cgrp
 *
 * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
 * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
 * and @ancestor are accessible.
 */
bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
{
	while (cgrp) {
		if (cgrp == ancestor)
			return true;
		cgrp = cgrp->parent;
	}
	return false;
}
284

285
static int cgroup_is_releasable(const struct cgroup *cgrp)
286 287
{
	const int bits =
288 289 290
		(1 << CGRP_RELEASABLE) |
		(1 << CGRP_NOTIFY_ON_RELEASE);
	return (cgrp->flags & bits) == bits;
291 292
}

293
static int notify_on_release(const struct cgroup *cgrp)
294
{
295
	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
296 297
}

T
Tejun Heo 已提交
298 299 300 301 302 303
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
304
 * Should be called under cgroup_[tree_]mutex.
T
Tejun Heo 已提交
305 306 307 308 309
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
T
Tejun Heo 已提交
310
				lockdep_is_held(&cgroup_tree_mutex) ||	\
T
Tejun Heo 已提交
311 312 313
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

314 315 316 317 318 319 320 321 322 323 324 325 326 327
/**
 * for_each_e_css - iterate all effective css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_[tree_]mutex.
 */
#define for_each_e_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
			;						\
		else

328
/**
T
Tejun Heo 已提交
329
 * for_each_subsys - iterate all enabled cgroup subsystems
330
 * @ss: the iteration cursor
331
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
332
 */
333
#define for_each_subsys(ss, ssid)					\
T
Tejun Heo 已提交
334 335
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&		\
	     (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
336

337 338
/* iterate across the hierarchies */
#define for_each_root(root)						\
339
	list_for_each_entry((root), &cgroup_roots, root_list)
340

341 342 343 344
/**
 * cgroup_lock_live_group - take cgroup_mutex and check that cgrp is alive.
 * @cgrp: the cgroup to be checked for liveness
 *
T
Tejun Heo 已提交
345 346
 * On success, returns true; the mutex should be later unlocked.  On
 * failure returns false with no lock held.
347
 */
348
static bool cgroup_lock_live_group(struct cgroup *cgrp)
349 350
{
	mutex_lock(&cgroup_mutex);
351
	if (cgroup_is_dead(cgrp)) {
352 353 354 355 356 357
		mutex_unlock(&cgroup_mutex);
		return false;
	}
	return true;
}

358 359 360
/* the list of cgroups eligible for automatic release. Protected by
 * release_list_lock */
static LIST_HEAD(release_list);
361
static DEFINE_RAW_SPINLOCK(release_list_lock);
362 363
static void cgroup_release_agent(struct work_struct *work);
static DECLARE_WORK(release_agent_work, cgroup_release_agent);
364
static void check_for_release(struct cgroup *cgrp);
365

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
/*
 * A cgroup can be associated with multiple css_sets as different tasks may
 * belong to different cgroups on different hierarchies.  In the other
 * direction, a css_set is naturally associated with multiple cgroups.
 * This M:N relationship is represented by the following link structure
 * which exists for each association and allows traversing the associations
 * from both sides.
 */
struct cgrp_cset_link {
	/* the cgroup and css_set this link associates */
	struct cgroup		*cgrp;
	struct css_set		*cset;

	/* list of cgrp_cset_links anchored at cgrp->cset_links */
	struct list_head	cset_link;

	/* list of cgrp_cset_links anchored at css_set->cgrp_links */
	struct list_head	cgrp_link;
384 385
};

386 387
/*
 * The default css_set - used by init and its children prior to any
388 389 390 391 392
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */
393 394 395 396 397 398 399 400
static struct css_set init_css_set = {
	.refcount		= ATOMIC_INIT(1),
	.cgrp_links		= LIST_HEAD_INIT(init_css_set.cgrp_links),
	.tasks			= LIST_HEAD_INIT(init_css_set.tasks),
	.mg_tasks		= LIST_HEAD_INIT(init_css_set.mg_tasks),
	.mg_preload_node	= LIST_HEAD_INIT(init_css_set.mg_preload_node),
	.mg_node		= LIST_HEAD_INIT(init_css_set.mg_node),
};
401

402
static int css_set_count	= 1;	/* 1 for init_css_set */
403

404 405 406 407 408
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
409
#define CSS_SET_HASH_BITS	7
410
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
411

412
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
413
{
414
	unsigned long key = 0UL;
415 416
	struct cgroup_subsys *ss;
	int i;
417

418
	for_each_subsys(ss, i)
419 420
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
421

422
	return key;
423 424
}

425
static void put_css_set_locked(struct css_set *cset, bool taskexit)
426
{
427
	struct cgrp_cset_link *link, *tmp_link;
T
Tejun Heo 已提交
428 429
	struct cgroup_subsys *ss;
	int ssid;
430

431 432 433
	lockdep_assert_held(&css_set_rwsem);

	if (!atomic_dec_and_test(&cset->refcount))
434
		return;
435

436
	/* This css_set is dead. unlink it and release cgroup refcounts */
T
Tejun Heo 已提交
437 438
	for_each_subsys(ss, ssid)
		list_del(&cset->e_cset_node[ssid]);
439
	hash_del(&cset->hlist);
440 441
	css_set_count--;

442
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
443
		struct cgroup *cgrp = link->cgrp;
444

445 446
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
L
Li Zefan 已提交
447

448
		/* @cgrp can't go away while we're holding css_set_rwsem */
T
Tejun Heo 已提交
449
		if (list_empty(&cgrp->cset_links) && notify_on_release(cgrp)) {
450
			if (taskexit)
451 452
				set_bit(CGRP_RELEASABLE, &cgrp->flags);
			check_for_release(cgrp);
453
		}
454 455

		kfree(link);
456
	}
457

458
	kfree_rcu(cset, rcu_head);
459 460
}

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
static void put_css_set(struct css_set *cset, bool taskexit)
{
	/*
	 * Ensure that the refcount doesn't hit zero while any readers
	 * can see it. Similar to atomic_dec_and_lock(), but for an
	 * rwlock
	 */
	if (atomic_add_unless(&cset->refcount, -1, 1))
		return;

	down_write(&css_set_rwsem);
	put_css_set_locked(cset, taskexit);
	up_write(&css_set_rwsem);
}

476 477 478
/*
 * refcounted get/put for css_set objects
 */
479
static inline void get_css_set(struct css_set *cset)
480
{
481
	atomic_inc(&cset->refcount);
482 483
}

484
/**
485
 * compare_css_sets - helper function for find_existing_css_set().
486 487
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
488 489 490
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
491
 * Returns true if "cset" matches "old_cset" except for the hierarchy
492 493
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
494 495
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
496 497 498 499 500
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
	struct list_head *l1, *l2;

501 502 503 504 505 506
	/*
	 * On the default hierarchy, there can be csets which are
	 * associated with the same set of cgroups but different csses.
	 * Let's first ensure that csses match.
	 */
	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
507 508 509 510
		return false;

	/*
	 * Compare cgroup pointers in order to distinguish between
511 512 513
	 * different cgroups in hierarchies.  As different cgroups may
	 * share the same effective css, this comparison is always
	 * necessary.
514
	 */
515 516
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
517
	while (1) {
518
		struct cgrp_cset_link *link1, *link2;
519
		struct cgroup *cgrp1, *cgrp2;
520 521 522 523

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
524 525
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
526 527
			break;
		} else {
528
			BUG_ON(l2 == &old_cset->cgrp_links);
529 530
		}
		/* Locate the cgroups associated with these links. */
531 532 533 534
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
535
		/* Hierarchies should be linked in the same order. */
536
		BUG_ON(cgrp1->root != cgrp2->root);
537 538 539 540 541 542 543 544

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
545 546
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
547 548
				return false;
		} else {
549
			if (cgrp1 != cgrp2)
550 551 552 553 554 555
				return false;
		}
	}
	return true;
}

556 557 558 559 560
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
561
 */
562 563 564
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
565
{
566
	struct cgroup_root *root = cgrp->root;
567
	struct cgroup_subsys *ss;
568
	struct css_set *cset;
569
	unsigned long key;
570
	int i;
571

B
Ben Blum 已提交
572 573 574 575 576
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
577
	for_each_subsys(ss, i) {
578
		if (root->subsys_mask & (1UL << i)) {
579 580 581 582 583
			/*
			 * @ss is in this hierarchy, so we want the
			 * effective css from @cgrp.
			 */
			template[i] = cgroup_e_css(cgrp, ss);
584
		} else {
585 586 587 588
			/*
			 * @ss is not in this hierarchy, so we don't want
			 * to change the css.
			 */
589
			template[i] = old_cset->subsys[i];
590 591 592
		}
	}

593
	key = css_set_hash(template);
594 595
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
596 597 598
			continue;

		/* This css_set matches what we need */
599
		return cset;
600
	}
601 602 603 604 605

	/* No existing cgroup group matched */
	return NULL;
}

606
static void free_cgrp_cset_links(struct list_head *links_to_free)
607
{
608
	struct cgrp_cset_link *link, *tmp_link;
609

610 611
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
612 613 614 615
		kfree(link);
	}
}

616 617 618 619 620 621 622
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
623
 */
624
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
625
{
626
	struct cgrp_cset_link *link;
627
	int i;
628 629 630

	INIT_LIST_HEAD(tmp_links);

631
	for (i = 0; i < count; i++) {
632
		link = kzalloc(sizeof(*link), GFP_KERNEL);
633
		if (!link) {
634
			free_cgrp_cset_links(tmp_links);
635 636
			return -ENOMEM;
		}
637
		list_add(&link->cset_link, tmp_links);
638 639 640 641
	}
	return 0;
}

642 643
/**
 * link_css_set - a helper function to link a css_set to a cgroup
644
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
645
 * @cset: the css_set to be linked
646 647
 * @cgrp: the destination cgroup
 */
648 649
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
650
{
651
	struct cgrp_cset_link *link;
652

653 654 655
	BUG_ON(list_empty(tmp_links));
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
656
	link->cgrp = cgrp;
657
	list_move(&link->cset_link, &cgrp->cset_links);
658 659 660 661
	/*
	 * Always add links to the tail of the list so that the list
	 * is sorted by order of hierarchy creation
	 */
662
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
663 664
}

665 666 667 668 669 670 671
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
672
 */
673 674
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
675
{
676
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
677
	struct css_set *cset;
678 679
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
680
	struct cgroup_subsys *ss;
681
	unsigned long key;
T
Tejun Heo 已提交
682
	int ssid;
683

684 685
	lockdep_assert_held(&cgroup_mutex);

686 687
	/* First see if we already have a cgroup group that matches
	 * the desired set */
688
	down_read(&css_set_rwsem);
689 690 691
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
692
	up_read(&css_set_rwsem);
693

694 695
	if (cset)
		return cset;
696

697
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
698
	if (!cset)
699 700
		return NULL;

701
	/* Allocate all the cgrp_cset_link objects that we'll need */
702
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
703
		kfree(cset);
704 705 706
		return NULL;
	}

707
	atomic_set(&cset->refcount, 1);
708
	INIT_LIST_HEAD(&cset->cgrp_links);
709
	INIT_LIST_HEAD(&cset->tasks);
T
Tejun Heo 已提交
710
	INIT_LIST_HEAD(&cset->mg_tasks);
711
	INIT_LIST_HEAD(&cset->mg_preload_node);
712
	INIT_LIST_HEAD(&cset->mg_node);
713
	INIT_HLIST_NODE(&cset->hlist);
714 715 716

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
717
	memcpy(cset->subsys, template, sizeof(cset->subsys));
718

719
	down_write(&css_set_rwsem);
720
	/* Add reference counts and links from the new css_set. */
721
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
722
		struct cgroup *c = link->cgrp;
723

724 725
		if (c->root == cgrp->root)
			c = cgrp;
726
		link_css_set(&tmp_links, cset, c);
727
	}
728

729
	BUG_ON(!list_empty(&tmp_links));
730 731

	css_set_count++;
732

T
Tejun Heo 已提交
733
	/* Add @cset to the hash table */
734 735
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
736

T
Tejun Heo 已提交
737 738 739 740
	for_each_subsys(ss, ssid)
		list_add_tail(&cset->e_cset_node[ssid],
			      &cset->subsys[ssid]->cgroup->e_csets[ssid]);

741
	up_write(&css_set_rwsem);
742

743
	return cset;
744 745
}

746
static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
747
{
748
	struct cgroup *root_cgrp = kf_root->kn->priv;
T
Tejun Heo 已提交
749

750
	return root_cgrp->root;
T
Tejun Heo 已提交
751 752
}

753
static int cgroup_init_root_id(struct cgroup_root *root)
754 755 756 757 758
{
	int id;

	lockdep_assert_held(&cgroup_mutex);

759
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
760 761 762 763 764 765 766
	if (id < 0)
		return id;

	root->hierarchy_id = id;
	return 0;
}

767
static void cgroup_exit_root_id(struct cgroup_root *root)
768 769 770 771 772 773 774 775 776
{
	lockdep_assert_held(&cgroup_mutex);

	if (root->hierarchy_id) {
		idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
		root->hierarchy_id = 0;
	}
}

777
static void cgroup_free_root(struct cgroup_root *root)
778 779 780 781 782 783 784 785 786 787
{
	if (root) {
		/* hierarhcy ID shoulid already have been released */
		WARN_ON_ONCE(root->hierarchy_id);

		idr_destroy(&root->cgroup_idr);
		kfree(root);
	}
}

788
static void cgroup_destroy_root(struct cgroup_root *root)
789
{
790
	struct cgroup *cgrp = &root->cgrp;
791 792
	struct cgrp_cset_link *link, *tmp_link;

T
Tejun Heo 已提交
793 794
	mutex_lock(&cgroup_tree_mutex);
	mutex_lock(&cgroup_mutex);
795

T
Tejun Heo 已提交
796
	BUG_ON(atomic_read(&root->nr_cgrps));
797 798 799
	BUG_ON(!list_empty(&cgrp->children));

	/* Rebind all subsystems back to the default hierarchy */
800
	rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
801 802

	/*
803 804
	 * Release all the links from cset_links to this hierarchy's
	 * root cgroup
805
	 */
806
	down_write(&css_set_rwsem);
807 808 809 810 811 812

	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
		kfree(link);
	}
813
	up_write(&css_set_rwsem);
814 815 816 817 818 819 820 821 822 823 824

	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
		cgroup_root_count--;
	}

	cgroup_exit_root_id(root);

	mutex_unlock(&cgroup_mutex);
	mutex_unlock(&cgroup_tree_mutex);

T
Tejun Heo 已提交
825
	kernfs_destroy_root(root->kf_root);
826 827 828
	cgroup_free_root(root);
}

829 830
/* look up cgroup associated with given css_set on the specified hierarchy */
static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
831
					    struct cgroup_root *root)
832 833 834
{
	struct cgroup *res = NULL;

835 836 837
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

838
	if (cset == &init_css_set) {
839
		res = &root->cgrp;
840
	} else {
841 842 843
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
844
			struct cgroup *c = link->cgrp;
845

846 847 848 849 850 851
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
852

853 854 855 856
	BUG_ON(!res);
	return res;
}

857
/*
858 859 860 861
 * Return the cgroup for "task" from the given hierarchy. Must be
 * called with cgroup_mutex and css_set_rwsem held.
 */
static struct cgroup *task_cgroup_from_root(struct task_struct *task,
862
					    struct cgroup_root *root)
863 864 865 866 867 868 869 870 871
{
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
	return cset_cgroup_from_root(task_css_set(task), root);
}

872 873 874 875 876 877
/*
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
878
 * cgroup_attach_task() can increment it again.  Because a count of zero
879 880 881 882 883 884 885 886 887 888 889 890 891
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
 * (usually) take cgroup_mutex.  These are the two most performance
 * critical pieces of code here.  The exception occurs on cgroup_exit(),
 * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
 * is taken, and if the cgroup count is zero, a usermode call made
L
Li Zefan 已提交
892 893
 * to the release agent with the name of the cgroup (path relative to
 * the root of cgroup file system) as the argument.
894 895 896 897
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
898
 * least one task in the system (init, pid == 1), therefore, root cgroup
899
 * always has either children cgroups and/or using tasks.  So we don't
900
 * need a special hack to ensure that root cgroup cannot be deleted.
901 902
 *
 * P.S.  One more locking exception.  RCU is used to guard the
903
 * update of a tasks cgroup pointer by cgroup_attach_task()
904 905
 */

906
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask);
T
Tejun Heo 已提交
907
static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
908
static const struct file_operations proc_cgroupstats_operations;
909

T
Tejun Heo 已提交
910 911
static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
			      char *buf)
912
{
T
Tejun Heo 已提交
913 914 915 916 917 918 919
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
		snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
			 cft->ss->name, cft->name);
	else
		strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
	return buf;
920 921
}

922 923 924 925 926 927 928 929 930 931
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
 * returns cft->mode if ->mode is not 0
 * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
 * returns S_IRUGO if it has only a read handler
 * returns S_IWUSR if it has only a write hander
 */
static umode_t cgroup_file_mode(const struct cftype *cft)
932
{
933
	umode_t mode = 0;
934

935 936 937 938 939 940 941 942 943 944 945
	if (cft->mode)
		return cft->mode;

	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
		mode |= S_IRUGO;

	if (cft->write_u64 || cft->write_s64 || cft->write_string ||
	    cft->trigger)
		mode |= S_IWUSR;

	return mode;
946 947
}

948 949
static void cgroup_free_fn(struct work_struct *work)
{
950
	struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
951

952
	atomic_dec(&cgrp->root->nr_cgrps);
953
	cgroup_pidlist_destroy_all(cgrp);
954

T
Tejun Heo 已提交
955 956 957 958 959 960 961 962 963 964 965
	if (cgrp->parent) {
		/*
		 * We get a ref to the parent, and put the ref when this
		 * cgroup is being freed, so it's guaranteed that the
		 * parent won't be destroyed before its children.
		 */
		cgroup_put(cgrp->parent);
		kernfs_put(cgrp->kn);
		kfree(cgrp);
	} else {
		/*
966
		 * This is root cgroup's refcnt reaching zero, which
T
Tejun Heo 已提交
967 968 969 970
		 * indicates that the root should be released.
		 */
		cgroup_destroy_root(cgrp->root);
	}
971 972 973 974 975 976
}

static void cgroup_free_rcu(struct rcu_head *head)
{
	struct cgroup *cgrp = container_of(head, struct cgroup, rcu_head);

977
	INIT_WORK(&cgrp->destroy_work, cgroup_free_fn);
978
	queue_work(cgroup_destroy_wq, &cgrp->destroy_work);
979 980
}

981
static void cgroup_get(struct cgroup *cgrp)
982
{
T
Tejun Heo 已提交
983 984 985
	WARN_ON_ONCE(cgroup_is_dead(cgrp));
	WARN_ON_ONCE(atomic_read(&cgrp->refcnt) <= 0);
	atomic_inc(&cgrp->refcnt);
986 987
}

988
static void cgroup_put(struct cgroup *cgrp)
T
Tejun Heo 已提交
989
{
T
Tejun Heo 已提交
990 991
	if (!atomic_dec_and_test(&cgrp->refcnt))
		return;
T
Tejun Heo 已提交
992
	if (WARN_ON_ONCE(cgrp->parent && !cgroup_is_dead(cgrp)))
T
Tejun Heo 已提交
993
		return;
T
Tejun Heo 已提交
994

995
	/*
T
Tejun Heo 已提交
996 997 998 999
	 * XXX: cgrp->id is only used to look up css's.  As cgroup and
	 * css's lifetimes will be decoupled, it should be made
	 * per-subsystem and moved to css->id so that lookups are
	 * successful until the target css is released.
1000
	 */
T
Tejun Heo 已提交
1001 1002 1003 1004
	mutex_lock(&cgroup_mutex);
	idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
	mutex_unlock(&cgroup_mutex);
	cgrp->id = -1;
T
Tejun Heo 已提交
1005

T
Tejun Heo 已提交
1006
	call_rcu(&cgrp->rcu_head, cgroup_free_rcu);
1007
}
T
Tejun Heo 已提交
1008

1009
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
1010
{
T
Tejun Heo 已提交
1011
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
1012

T
Tejun Heo 已提交
1013
	lockdep_assert_held(&cgroup_tree_mutex);
T
Tejun Heo 已提交
1014
	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
T
Tejun Heo 已提交
1015 1016
}

1017
/**
1018
 * cgroup_clear_dir - remove subsys files in a cgroup directory
1019
 * @cgrp: target cgroup
1020 1021
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
1022
static void cgroup_clear_dir(struct cgroup *cgrp, unsigned long subsys_mask)
T
Tejun Heo 已提交
1023
{
1024
	struct cgroup_subsys *ss;
1025
	int i;
T
Tejun Heo 已提交
1026

1027
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
1028
		struct cftype *cfts;
1029 1030

		if (!test_bit(i, &subsys_mask))
1031
			continue;
T
Tejun Heo 已提交
1032 1033
		list_for_each_entry(cfts, &ss->cfts, node)
			cgroup_addrm_files(cgrp, cfts, false);
1034
	}
1035 1036
}

1037
static int rebind_subsystems(struct cgroup_root *dst_root,
1038
			     unsigned long ss_mask)
1039
{
1040
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1041
	int ssid, i, ret;
1042

T
Tejun Heo 已提交
1043 1044
	lockdep_assert_held(&cgroup_tree_mutex);
	lockdep_assert_held(&cgroup_mutex);
1045

1046 1047 1048
	for_each_subsys(ss, ssid) {
		if (!(ss_mask & (1 << ssid)))
			continue;
B
Ben Blum 已提交
1049

1050
		/* if @ss is on the dummy_root, we can always move it */
1051
		if (ss->root == &cgrp_dfl_root)
1052
			continue;
1053

1054
		/* if @ss has non-root cgroups attached to it, can't move */
1055
		if (!list_empty(&ss->root->cgrp.children))
T
Tejun Heo 已提交
1056
			return -EBUSY;
1057

1058
		/* can't move between two non-dummy roots either */
1059
		if (dst_root != &cgrp_dfl_root)
1060
			return -EBUSY;
1061 1062
	}

T
Tejun Heo 已提交
1063 1064 1065
	ret = cgroup_populate_dir(&dst_root->cgrp, ss_mask);
	if (ret) {
		if (dst_root != &cgrp_dfl_root)
1066
			return ret;
1067

T
Tejun Heo 已提交
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
		/*
		 * Rebinding back to the default root is not allowed to
		 * fail.  Using both default and non-default roots should
		 * be rare.  Moving subsystems back and forth even more so.
		 * Just warn about it and continue.
		 */
		if (cgrp_dfl_root_visible) {
			pr_warning("cgroup: failed to create files (%d) while rebinding 0x%lx to default root\n",
				   ret, ss_mask);
			pr_warning("cgroup: you may retry by moving them to a different hierarchy and unbinding\n");
		}
1079
	}
1080 1081 1082 1083 1084

	/*
	 * Nothing can fail from this point on.  Remove files for the
	 * removed subsystems and rebind each subsystem.
	 */
1085
	mutex_unlock(&cgroup_mutex);
1086
	for_each_subsys(ss, ssid)
T
Tejun Heo 已提交
1087
		if (ss_mask & (1 << ssid))
1088
			cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
1089
	mutex_lock(&cgroup_mutex);
1090

1091
	for_each_subsys(ss, ssid) {
1092
		struct cgroup_root *src_root;
1093
		struct cgroup_subsys_state *css;
T
Tejun Heo 已提交
1094
		struct css_set *cset;
1095

1096 1097
		if (!(ss_mask & (1 << ssid)))
			continue;
1098

1099
		src_root = ss->root;
1100
		css = cgroup_css(&src_root->cgrp, ss);
1101

1102
		WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
1103

1104 1105
		RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
		rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
1106
		ss->root = dst_root;
1107
		css->cgroup = &dst_root->cgrp;
1108

T
Tejun Heo 已提交
1109 1110 1111 1112 1113 1114
		down_write(&css_set_rwsem);
		hash_for_each(css_set_table, i, cset, hlist)
			list_move_tail(&cset->e_cset_node[ss->id],
				       &dst_root->cgrp.e_csets[ss->id]);
		up_write(&css_set_rwsem);

1115 1116 1117 1118 1119
		src_root->subsys_mask &= ~(1 << ssid);
		src_root->cgrp.child_subsys_mask &= ~(1 << ssid);

		dst_root->subsys_mask |= 1 << ssid;
		dst_root->cgrp.child_subsys_mask |= 1 << ssid;
1120

1121 1122
		if (ss->bind)
			ss->bind(css);
1123 1124
	}

T
Tejun Heo 已提交
1125
	kernfs_activate(dst_root->cgrp.kn);
1126 1127 1128
	return 0;
}

T
Tejun Heo 已提交
1129 1130
static int cgroup_show_options(struct seq_file *seq,
			       struct kernfs_root *kf_root)
1131
{
1132
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1133
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1134
	int ssid;
1135

T
Tejun Heo 已提交
1136
	for_each_subsys(ss, ssid)
1137
		if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
1138
			seq_printf(seq, ",%s", ss->name);
1139 1140
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR)
		seq_puts(seq, ",sane_behavior");
1141
	if (root->flags & CGRP_ROOT_NOPREFIX)
1142
		seq_puts(seq, ",noprefix");
1143
	if (root->flags & CGRP_ROOT_XATTR)
A
Aristeu Rozanski 已提交
1144
		seq_puts(seq, ",xattr");
1145 1146

	spin_lock(&release_agent_path_lock);
1147 1148
	if (strlen(root->release_agent_path))
		seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1149 1150
	spin_unlock(&release_agent_path_lock);

1151
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1152
		seq_puts(seq, ",clone_children");
1153 1154
	if (strlen(root->name))
		seq_printf(seq, ",name=%s", root->name);
1155 1156 1157 1158
	return 0;
}

struct cgroup_sb_opts {
1159
	unsigned long subsys_mask;
1160
	unsigned long flags;
1161
	char *release_agent;
1162
	bool cpuset_clone_children;
1163
	char *name;
1164 1165
	/* User explicitly requested empty subsystem */
	bool none;
1166 1167
};

B
Ben Blum 已提交
1168
/*
1169 1170 1171 1172
 * Convert a hierarchy specifier into a bitmask of subsystems and
 * flags. Call with cgroup_mutex held to protect the cgroup_subsys[]
 * array. This function takes refcounts on subsystems to be used, unless it
 * returns error, in which case no refcounts are taken.
B
Ben Blum 已提交
1173
 */
B
Ben Blum 已提交
1174
static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1175
{
1176 1177
	char *token, *o = data;
	bool all_ss = false, one_ss = false;
1178
	unsigned long mask = (unsigned long)-1;
1179 1180
	struct cgroup_subsys *ss;
	int i;
1181

B
Ben Blum 已提交
1182 1183
	BUG_ON(!mutex_is_locked(&cgroup_mutex));

1184
#ifdef CONFIG_CPUSETS
1185
	mask = ~(1UL << cpuset_cgrp_id);
1186
#endif
1187

1188
	memset(opts, 0, sizeof(*opts));
1189 1190 1191 1192

	while ((token = strsep(&o, ",")) != NULL) {
		if (!*token)
			return -EINVAL;
1193
		if (!strcmp(token, "none")) {
1194 1195
			/* Explicitly have no subsystems */
			opts->none = true;
1196 1197 1198 1199 1200 1201 1202 1203 1204
			continue;
		}
		if (!strcmp(token, "all")) {
			/* Mutually exclusive option 'all' + subsystem name */
			if (one_ss)
				return -EINVAL;
			all_ss = true;
			continue;
		}
1205 1206 1207 1208
		if (!strcmp(token, "__DEVEL__sane_behavior")) {
			opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
			continue;
		}
1209
		if (!strcmp(token, "noprefix")) {
1210
			opts->flags |= CGRP_ROOT_NOPREFIX;
1211 1212 1213
			continue;
		}
		if (!strcmp(token, "clone_children")) {
1214
			opts->cpuset_clone_children = true;
1215 1216
			continue;
		}
A
Aristeu Rozanski 已提交
1217
		if (!strcmp(token, "xattr")) {
1218
			opts->flags |= CGRP_ROOT_XATTR;
A
Aristeu Rozanski 已提交
1219 1220
			continue;
		}
1221
		if (!strncmp(token, "release_agent=", 14)) {
1222 1223 1224
			/* Specifying two release agents is forbidden */
			if (opts->release_agent)
				return -EINVAL;
1225
			opts->release_agent =
1226
				kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1227 1228
			if (!opts->release_agent)
				return -ENOMEM;
1229 1230 1231
			continue;
		}
		if (!strncmp(token, "name=", 5)) {
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
			const char *name = token + 5;
			/* Can't specify an empty name */
			if (!strlen(name))
				return -EINVAL;
			/* Must match [\w.-]+ */
			for (i = 0; i < strlen(name); i++) {
				char c = name[i];
				if (isalnum(c))
					continue;
				if ((c == '.') || (c == '-') || (c == '_'))
					continue;
				return -EINVAL;
			}
			/* Specifying two names is forbidden */
			if (opts->name)
				return -EINVAL;
			opts->name = kstrndup(name,
1249
					      MAX_CGROUP_ROOT_NAMELEN - 1,
1250 1251 1252
					      GFP_KERNEL);
			if (!opts->name)
				return -ENOMEM;
1253 1254 1255 1256

			continue;
		}

1257
		for_each_subsys(ss, i) {
1258 1259 1260 1261 1262 1263 1264 1265
			if (strcmp(token, ss->name))
				continue;
			if (ss->disabled)
				continue;

			/* Mutually exclusive option 'all' + subsystem name */
			if (all_ss)
				return -EINVAL;
1266
			set_bit(i, &opts->subsys_mask);
1267 1268 1269 1270 1271 1272 1273 1274
			one_ss = true;

			break;
		}
		if (i == CGROUP_SUBSYS_COUNT)
			return -ENOENT;
	}

1275 1276
	/* Consistency checks */

1277 1278 1279
	if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
		pr_warning("cgroup: sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");

1280 1281 1282 1283
		if ((opts->flags & (CGRP_ROOT_NOPREFIX | CGRP_ROOT_XATTR)) ||
		    opts->cpuset_clone_children || opts->release_agent ||
		    opts->name) {
			pr_err("cgroup: sane_behavior: noprefix, xattr, clone_children, release_agent and name are not allowed\n");
1284 1285
			return -EINVAL;
		}
T
Tejun Heo 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
	} else {
		/*
		 * If the 'all' option was specified select all the
		 * subsystems, otherwise if 'none', 'name=' and a subsystem
		 * name options were not specified, let's default to 'all'
		 */
		if (all_ss || (!one_ss && !opts->none && !opts->name))
			for_each_subsys(ss, i)
				if (!ss->disabled)
					set_bit(i, &opts->subsys_mask);
1296

T
Tejun Heo 已提交
1297 1298 1299 1300 1301
		/*
		 * We either have to specify by name or by subsystems. (So
		 * all empty hierarchies must have a name).
		 */
		if (!opts->subsys_mask && !opts->name)
1302 1303 1304
			return -EINVAL;
	}

1305 1306 1307 1308 1309
	/*
	 * Option noprefix was introduced just for backward compatibility
	 * with the old cpuset, so we allow noprefix only if mounting just
	 * the cpuset subsystem.
	 */
1310
	if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1311 1312
		return -EINVAL;

1313 1314

	/* Can't specify "none" and some subsystems */
1315
	if (opts->subsys_mask && opts->none)
1316 1317
		return -EINVAL;

1318 1319 1320
	return 0;
}

T
Tejun Heo 已提交
1321
static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1322 1323
{
	int ret = 0;
1324
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1325
	struct cgroup_sb_opts opts;
1326
	unsigned long added_mask, removed_mask;
1327

1328 1329 1330 1331 1332
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR) {
		pr_err("cgroup: sane_behavior: remount is not allowed\n");
		return -EINVAL;
	}

T
Tejun Heo 已提交
1333
	mutex_lock(&cgroup_tree_mutex);
1334 1335 1336 1337 1338 1339 1340
	mutex_lock(&cgroup_mutex);

	/* See what subsystems are wanted */
	ret = parse_cgroupfs_options(data, &opts);
	if (ret)
		goto out_unlock;

1341
	if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1342 1343 1344
		pr_warning("cgroup: option changes via remount are deprecated (pid=%d comm=%s)\n",
			   task_tgid_nr(current), current->comm);

1345 1346
	added_mask = opts.subsys_mask & ~root->subsys_mask;
	removed_mask = root->subsys_mask & ~opts.subsys_mask;
1347

B
Ben Blum 已提交
1348
	/* Don't allow flags or name to change at remount */
1349
	if (((opts.flags ^ root->flags) & CGRP_ROOT_OPTION_MASK) ||
B
Ben Blum 已提交
1350
	    (opts.name && strcmp(opts.name, root->name))) {
1351 1352 1353
		pr_err("cgroup: option or name mismatch, new: 0x%lx \"%s\", old: 0x%lx \"%s\"\n",
		       opts.flags & CGRP_ROOT_OPTION_MASK, opts.name ?: "",
		       root->flags & CGRP_ROOT_OPTION_MASK, root->name);
1354 1355 1356 1357
		ret = -EINVAL;
		goto out_unlock;
	}

1358
	/* remounting is not allowed for populated hierarchies */
1359
	if (!list_empty(&root->cgrp.children)) {
1360
		ret = -EBUSY;
1361
		goto out_unlock;
B
Ben Blum 已提交
1362
	}
1363

1364
	ret = rebind_subsystems(root, added_mask);
1365
	if (ret)
1366
		goto out_unlock;
1367

1368
	rebind_subsystems(&cgrp_dfl_root, removed_mask);
1369

1370 1371
	if (opts.release_agent) {
		spin_lock(&release_agent_path_lock);
1372
		strcpy(root->release_agent_path, opts.release_agent);
1373 1374
		spin_unlock(&release_agent_path_lock);
	}
1375
 out_unlock:
1376
	kfree(opts.release_agent);
1377
	kfree(opts.name);
1378
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
1379
	mutex_unlock(&cgroup_tree_mutex);
1380 1381 1382
	return ret;
}

1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
/*
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
 * words after the first mount.
 */
static bool use_task_css_set_links __read_mostly;

static void cgroup_enable_task_cg_lists(void)
{
	struct task_struct *p, *g;

1395
	down_write(&css_set_rwsem);
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417

	if (use_task_css_set_links)
		goto out_unlock;

	use_task_css_set_links = true;

	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		WARN_ON_ONCE(!list_empty(&p->cg_list) ||
			     task_css_set(p) != &init_css_set);

		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
1418 1419
		 * Do it while holding siglock so that we don't end up
		 * racing against cgroup_exit().
1420
		 */
1421
		spin_lock_irq(&p->sighand->siglock);
1422 1423 1424 1425 1426 1427
		if (!(p->flags & PF_EXITING)) {
			struct css_set *cset = task_css_set(p);

			list_add(&p->cg_list, &cset->tasks);
			get_css_set(cset);
		}
1428
		spin_unlock_irq(&p->sighand->siglock);
1429 1430 1431
	} while_each_thread(g, p);
	read_unlock(&tasklist_lock);
out_unlock:
1432
	up_write(&css_set_rwsem);
1433
}
1434

1435 1436
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
T
Tejun Heo 已提交
1437 1438 1439
	struct cgroup_subsys *ss;
	int ssid;

T
Tejun Heo 已提交
1440
	atomic_set(&cgrp->refcnt, 1);
1441 1442
	INIT_LIST_HEAD(&cgrp->sibling);
	INIT_LIST_HEAD(&cgrp->children);
1443
	INIT_LIST_HEAD(&cgrp->cset_links);
1444
	INIT_LIST_HEAD(&cgrp->release_list);
1445 1446
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
T
Tejun Heo 已提交
1447
	cgrp->dummy_css.cgroup = cgrp;
T
Tejun Heo 已提交
1448 1449 1450

	for_each_subsys(ss, ssid)
		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1451
}
1452

1453
static void init_cgroup_root(struct cgroup_root *root,
1454
			     struct cgroup_sb_opts *opts)
1455
{
1456
	struct cgroup *cgrp = &root->cgrp;
1457

1458
	INIT_LIST_HEAD(&root->root_list);
1459
	atomic_set(&root->nr_cgrps, 1);
1460
	cgrp->root = root;
1461
	init_cgroup_housekeeping(cgrp);
1462
	idr_init(&root->cgroup_idr);
1463 1464 1465 1466 1467 1468

	root->flags = opts->flags;
	if (opts->release_agent)
		strcpy(root->release_agent_path, opts->release_agent);
	if (opts->name)
		strcpy(root->name, opts->name);
1469
	if (opts->cpuset_clone_children)
1470
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1471 1472
}

1473
static int cgroup_setup_root(struct cgroup_root *root, unsigned long ss_mask)
1474
{
1475
	LIST_HEAD(tmp_links);
1476
	struct cgroup *root_cgrp = &root->cgrp;
1477 1478
	struct css_set *cset;
	int i, ret;
1479

1480 1481
	lockdep_assert_held(&cgroup_tree_mutex);
	lockdep_assert_held(&cgroup_mutex);
1482

1483 1484
	ret = idr_alloc(&root->cgroup_idr, root_cgrp, 0, 1, GFP_KERNEL);
	if (ret < 0)
T
Tejun Heo 已提交
1485
		goto out;
1486
	root_cgrp->id = ret;
1487

1488
	/*
1489
	 * We're accessing css_set_count without locking css_set_rwsem here,
1490 1491 1492 1493 1494 1495
	 * but that's OK - it can only be increased by someone holding
	 * cgroup_lock, and that's us. The worst that can happen is that we
	 * have some link structures left over
	 */
	ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
	if (ret)
T
Tejun Heo 已提交
1496
		goto out;
1497

1498
	ret = cgroup_init_root_id(root);
1499
	if (ret)
T
Tejun Heo 已提交
1500
		goto out;
1501

T
Tejun Heo 已提交
1502 1503 1504 1505 1506 1507 1508 1509
	root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
					   KERNFS_ROOT_CREATE_DEACTIVATED,
					   root_cgrp);
	if (IS_ERR(root->kf_root)) {
		ret = PTR_ERR(root->kf_root);
		goto exit_root_id;
	}
	root_cgrp->kn = root->kf_root->kn;
1510

1511 1512
	ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
	if (ret)
T
Tejun Heo 已提交
1513
		goto destroy_root;
1514

1515
	ret = rebind_subsystems(root, ss_mask);
1516
	if (ret)
T
Tejun Heo 已提交
1517
		goto destroy_root;
1518

1519 1520 1521 1522 1523 1524 1525
	/*
	 * There must be no failure case after here, since rebinding takes
	 * care of subsystems' refcounts, which are explicitly dropped in
	 * the failure exit path.
	 */
	list_add(&root->root_list, &cgroup_roots);
	cgroup_root_count++;
A
Al Viro 已提交
1526

1527
	/*
1528
	 * Link the root cgroup in this hierarchy into all the css_set
1529 1530
	 * objects.
	 */
1531
	down_write(&css_set_rwsem);
1532 1533
	hash_for_each(css_set_table, i, cset, hlist)
		link_css_set(&tmp_links, cset, root_cgrp);
1534
	up_write(&css_set_rwsem);
1535

1536
	BUG_ON(!list_empty(&root_cgrp->children));
1537
	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1538

T
Tejun Heo 已提交
1539
	kernfs_activate(root_cgrp->kn);
1540
	ret = 0;
T
Tejun Heo 已提交
1541
	goto out;
1542

T
Tejun Heo 已提交
1543 1544 1545 1546
destroy_root:
	kernfs_destroy_root(root->kf_root);
	root->kf_root = NULL;
exit_root_id:
1547
	cgroup_exit_root_id(root);
T
Tejun Heo 已提交
1548
out:
1549 1550
	free_cgrp_cset_links(&tmp_links);
	return ret;
1551 1552
}

A
Al Viro 已提交
1553
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1554
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
1555
			 void *data)
1556
{
1557
	struct cgroup_root *root;
1558
	struct cgroup_sb_opts opts;
T
Tejun Heo 已提交
1559
	struct dentry *dentry;
1560
	int ret;
L
Li Zefan 已提交
1561
	bool new_sb;
1562

1563 1564 1565 1566 1567 1568
	/*
	 * The first time anyone tries to mount a cgroup, enable the list
	 * linking each css_set to its tasks and fix up all existing tasks.
	 */
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();
1569

1570
	mutex_lock(&cgroup_tree_mutex);
B
Ben Blum 已提交
1571
	mutex_lock(&cgroup_mutex);
1572 1573

	/* First find the desired set of subsystems */
1574
	ret = parse_cgroupfs_options(data, &opts);
1575
	if (ret)
1576
		goto out_unlock;
1577
retry:
T
Tejun Heo 已提交
1578
	/* look for a matching existing root */
T
Tejun Heo 已提交
1579 1580 1581 1582 1583 1584
	if (!opts.subsys_mask && !opts.none && !opts.name) {
		cgrp_dfl_root_visible = true;
		root = &cgrp_dfl_root;
		cgroup_get(&root->cgrp);
		ret = 0;
		goto out_unlock;
1585 1586
	}

1587
	for_each_root(root) {
T
Tejun Heo 已提交
1588
		bool name_match = false;
1589

1590
		if (root == &cgrp_dfl_root)
1591
			continue;
1592

B
Ben Blum 已提交
1593
		/*
T
Tejun Heo 已提交
1594 1595 1596
		 * If we asked for a name then it must match.  Also, if
		 * name matches but sybsys_mask doesn't, we should fail.
		 * Remember whether name matched.
B
Ben Blum 已提交
1597
		 */
T
Tejun Heo 已提交
1598 1599 1600 1601 1602
		if (opts.name) {
			if (strcmp(opts.name, root->name))
				continue;
			name_match = true;
		}
1603

1604
		/*
T
Tejun Heo 已提交
1605 1606
		 * If we asked for subsystems (or explicitly for no
		 * subsystems) then they must match.
1607
		 */
T
Tejun Heo 已提交
1608
		if ((opts.subsys_mask || opts.none) &&
1609
		    (opts.subsys_mask != root->subsys_mask)) {
T
Tejun Heo 已提交
1610 1611 1612 1613 1614
			if (!name_match)
				continue;
			ret = -EBUSY;
			goto out_unlock;
		}
1615

1616
		if ((root->flags ^ opts.flags) & CGRP_ROOT_OPTION_MASK) {
1617 1618 1619
			if ((root->flags | opts.flags) & CGRP_ROOT_SANE_BEHAVIOR) {
				pr_err("cgroup: sane_behavior: new mount options should match the existing superblock\n");
				ret = -EINVAL;
1620
				goto out_unlock;
1621 1622 1623
			} else {
				pr_warning("cgroup: new mount options do not match the existing superblock, will be ignored\n");
			}
1624
		}
1625

T
Tejun Heo 已提交
1626
		/*
1627
		 * A root's lifetime is governed by its root cgroup.  Zero
T
Tejun Heo 已提交
1628 1629 1630 1631 1632
		 * ref indicate that the root is being destroyed.  Wait for
		 * destruction to complete so that the subsystems are free.
		 * We can use wait_queue for the wait but this path is
		 * super cold.  Let's just sleep for a bit and retry.
		 */
1633
		if (!atomic_inc_not_zero(&root->cgrp.refcnt)) {
T
Tejun Heo 已提交
1634 1635 1636
			mutex_unlock(&cgroup_mutex);
			mutex_unlock(&cgroup_tree_mutex);
			msleep(10);
1637 1638
			mutex_lock(&cgroup_tree_mutex);
			mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1639 1640
			goto retry;
		}
1641

T
Tejun Heo 已提交
1642
		ret = 0;
T
Tejun Heo 已提交
1643
		goto out_unlock;
1644 1645
	}

1646
	/*
1647 1648 1649
	 * No such thing, create a new one.  name= matching without subsys
	 * specification is allowed for already existing hierarchies but we
	 * can't create new one without subsys specification.
1650
	 */
1651 1652 1653
	if (!opts.subsys_mask && !opts.none) {
		ret = -EINVAL;
		goto out_unlock;
1654 1655
	}

1656 1657 1658
	root = kzalloc(sizeof(*root), GFP_KERNEL);
	if (!root) {
		ret = -ENOMEM;
T
Tejun Heo 已提交
1659
		goto out_unlock;
1660
	}
1661

1662 1663
	init_cgroup_root(root, &opts);

T
Tejun Heo 已提交
1664
	ret = cgroup_setup_root(root, opts.subsys_mask);
T
Tejun Heo 已提交
1665 1666
	if (ret)
		cgroup_free_root(root);
1667

1668
out_unlock:
1669
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
1670
	mutex_unlock(&cgroup_tree_mutex);
1671

1672 1673
	kfree(opts.release_agent);
	kfree(opts.name);
A
Aristeu Rozanski 已提交
1674

T
Tejun Heo 已提交
1675
	if (ret)
1676
		return ERR_PTR(ret);
T
Tejun Heo 已提交
1677

L
Li Zefan 已提交
1678 1679
	dentry = kernfs_mount(fs_type, flags, root->kf_root, &new_sb);
	if (IS_ERR(dentry) || !new_sb)
1680
		cgroup_put(&root->cgrp);
T
Tejun Heo 已提交
1681 1682 1683 1684 1685 1686
	return dentry;
}

static void cgroup_kill_sb(struct super_block *sb)
{
	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
1687
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
T
Tejun Heo 已提交
1688

1689
	cgroup_put(&root->cgrp);
T
Tejun Heo 已提交
1690
	kernfs_kill_sb(sb);
1691 1692 1693 1694
}

static struct file_system_type cgroup_fs_type = {
	.name = "cgroup",
A
Al Viro 已提交
1695
	.mount = cgroup_mount,
1696 1697 1698
	.kill_sb = cgroup_kill_sb,
};

1699 1700
static struct kobject *cgroup_kobj;

1701
/**
1702
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1703 1704 1705 1706
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
1707 1708 1709 1710 1711
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
T
Tejun Heo 已提交
1712
 * Return value is the same as kernfs_path().
1713
 */
T
Tejun Heo 已提交
1714
char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1715
{
1716
	struct cgroup_root *root;
1717
	struct cgroup *cgrp;
T
Tejun Heo 已提交
1718 1719
	int hierarchy_id = 1;
	char *path = NULL;
1720 1721

	mutex_lock(&cgroup_mutex);
1722
	down_read(&css_set_rwsem);
1723

1724 1725
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

1726 1727
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
T
Tejun Heo 已提交
1728
		path = cgroup_path(cgrp, buf, buflen);
1729 1730
	} else {
		/* if no hierarchy exists, everyone is in "/" */
T
Tejun Heo 已提交
1731 1732
		if (strlcpy(buf, "/", buflen) < buflen)
			path = buf;
1733 1734
	}

1735
	up_read(&css_set_rwsem);
1736
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
1737
	return path;
1738
}
1739
EXPORT_SYMBOL_GPL(task_cgroup_path);
1740

1741
/* used to track tasks and other necessary states during migration */
1742
struct cgroup_taskset {
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
	/* the src and dst cset list running through cset->mg_node */
	struct list_head	src_csets;
	struct list_head	dst_csets;

	/*
	 * Fields for cgroup_taskset_*() iteration.
	 *
	 * Before migration is committed, the target migration tasks are on
	 * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
	 * the csets on ->dst_csets.  ->csets point to either ->src_csets
	 * or ->dst_csets depending on whether migration is committed.
	 *
	 * ->cur_csets and ->cur_task point to the current task position
	 * during iteration.
	 */
	struct list_head	*csets;
	struct css_set		*cur_cset;
	struct task_struct	*cur_task;
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
};

/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
 *
 * @tset iteration is initialized and the first task is returned.
 */
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
{
1771 1772 1773 1774
	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
	tset->cur_task = NULL;

	return cgroup_taskset_next(tset);
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
}

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
{
1786 1787
	struct css_set *cset = tset->cur_cset;
	struct task_struct *task = tset->cur_task;
1788

1789 1790 1791 1792 1793 1794
	while (&cset->mg_node != tset->csets) {
		if (!task)
			task = list_first_entry(&cset->mg_tasks,
						struct task_struct, cg_list);
		else
			task = list_next_entry(task, cg_list);
1795

1796 1797 1798 1799 1800
		if (&task->cg_list != &cset->mg_tasks) {
			tset->cur_cset = cset;
			tset->cur_task = task;
			return task;
		}
1801

1802 1803 1804
		cset = list_next_entry(cset, mg_node);
		task = NULL;
	}
1805

1806
	return NULL;
1807 1808
}

1809
/**
B
Ben Blum 已提交
1810
 * cgroup_task_migrate - move a task from one cgroup to another.
1811 1812 1813
 * @old_cgrp; the cgroup @tsk is being migrated from
 * @tsk: the task being migrated
 * @new_cset: the new css_set @tsk is being attached to
B
Ben Blum 已提交
1814
 *
1815
 * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
B
Ben Blum 已提交
1816
 */
1817 1818 1819
static void cgroup_task_migrate(struct cgroup *old_cgrp,
				struct task_struct *tsk,
				struct css_set *new_cset)
B
Ben Blum 已提交
1820
{
1821
	struct css_set *old_cset;
B
Ben Blum 已提交
1822

1823 1824 1825
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

B
Ben Blum 已提交
1826
	/*
1827 1828 1829
	 * We are synchronized through threadgroup_lock() against PF_EXITING
	 * setting such that we can't race against cgroup_exit() changing the
	 * css_set to init_css_set and dropping the old one.
B
Ben Blum 已提交
1830
	 */
1831
	WARN_ON_ONCE(tsk->flags & PF_EXITING);
1832
	old_cset = task_css_set(tsk);
B
Ben Blum 已提交
1833

1834
	get_css_set(new_cset);
1835
	rcu_assign_pointer(tsk->cgroups, new_cset);
B
Ben Blum 已提交
1836

1837 1838 1839 1840 1841 1842 1843
	/*
	 * Use move_tail so that cgroup_taskset_first() still returns the
	 * leader after migration.  This works because cgroup_migrate()
	 * ensures that the dst_cset of the leader is the first on the
	 * tset's dst_csets list.
	 */
	list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
B
Ben Blum 已提交
1844 1845

	/*
1846 1847 1848
	 * We just gained a reference on old_cset by taking it from the
	 * task. As trading it for new_cset is protected by cgroup_mutex,
	 * we're safe to drop it here; it will be freed under RCU.
B
Ben Blum 已提交
1849
	 */
1850
	set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
1851
	put_css_set_locked(old_cset, false);
B
Ben Blum 已提交
1852 1853
}

L
Li Zefan 已提交
1854
/**
1855 1856
 * cgroup_migrate_finish - cleanup after attach
 * @preloaded_csets: list of preloaded css_sets
B
Ben Blum 已提交
1857
 *
1858 1859
 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
 * those functions for details.
B
Ben Blum 已提交
1860
 */
1861
static void cgroup_migrate_finish(struct list_head *preloaded_csets)
B
Ben Blum 已提交
1862
{
1863
	struct css_set *cset, *tmp_cset;
B
Ben Blum 已提交
1864

1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985
	lockdep_assert_held(&cgroup_mutex);

	down_write(&css_set_rwsem);
	list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
		cset->mg_src_cgrp = NULL;
		cset->mg_dst_cset = NULL;
		list_del_init(&cset->mg_preload_node);
		put_css_set_locked(cset, false);
	}
	up_write(&css_set_rwsem);
}

/**
 * cgroup_migrate_add_src - add a migration source css_set
 * @src_cset: the source css_set to add
 * @dst_cgrp: the destination cgroup
 * @preloaded_csets: list of preloaded css_sets
 *
 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
 * @src_cset and add it to @preloaded_csets, which should later be cleaned
 * up by cgroup_migrate_finish().
 *
 * This function may be called without holding threadgroup_lock even if the
 * target is a process.  Threads may be created and destroyed but as long
 * as cgroup_mutex is not dropped, no new css_set can be put into play and
 * the preloaded css_sets are guaranteed to cover all migrations.
 */
static void cgroup_migrate_add_src(struct css_set *src_cset,
				   struct cgroup *dst_cgrp,
				   struct list_head *preloaded_csets)
{
	struct cgroup *src_cgrp;

	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);

	/* nothing to do if this cset already belongs to the cgroup */
	if (src_cgrp == dst_cgrp)
		return;

	if (!list_empty(&src_cset->mg_preload_node))
		return;

	WARN_ON(src_cset->mg_src_cgrp);
	WARN_ON(!list_empty(&src_cset->mg_tasks));
	WARN_ON(!list_empty(&src_cset->mg_node));

	src_cset->mg_src_cgrp = src_cgrp;
	get_css_set(src_cset);
	list_add(&src_cset->mg_preload_node, preloaded_csets);
}

/**
 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
 * @dst_cgrp: the destination cgroup
 * @preloaded_csets: list of preloaded source css_sets
 *
 * Tasks are about to be moved to @dst_cgrp and all the source css_sets
 * have been preloaded to @preloaded_csets.  This function looks up and
 * pins all destination css_sets, links each to its source, and put them on
 * @preloaded_csets.
 *
 * This function must be called after cgroup_migrate_add_src() has been
 * called on each migration source css_set.  After migration is performed
 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
 * @preloaded_csets.
 */
static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
				      struct list_head *preloaded_csets)
{
	LIST_HEAD(csets);
	struct css_set *src_cset;

	lockdep_assert_held(&cgroup_mutex);

	/* look up the dst cset for each src cset and link it to src */
	list_for_each_entry(src_cset, preloaded_csets, mg_preload_node) {
		struct css_set *dst_cset;

		dst_cset = find_css_set(src_cset, dst_cgrp);
		if (!dst_cset)
			goto err;

		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
		src_cset->mg_dst_cset = dst_cset;

		if (list_empty(&dst_cset->mg_preload_node))
			list_add(&dst_cset->mg_preload_node, &csets);
		else
			put_css_set(dst_cset, false);
	}

	list_splice(&csets, preloaded_csets);
	return 0;
err:
	cgroup_migrate_finish(&csets);
	return -ENOMEM;
}

/**
 * cgroup_migrate - migrate a process or task to a cgroup
 * @cgrp: the destination cgroup
 * @leader: the leader of the process or the task to migrate
 * @threadgroup: whether @leader points to the whole process or a single task
 *
 * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
 * process, the caller must be holding threadgroup_lock of @leader.  The
 * caller is also responsible for invoking cgroup_migrate_add_src() and
 * cgroup_migrate_prepare_dst() on the targets before invoking this
 * function and following up with cgroup_migrate_finish().
 *
 * As long as a controller's ->can_attach() doesn't fail, this function is
 * guaranteed to succeed.  This means that, excluding ->can_attach()
 * failure, when migrating multiple targets, the success or failure can be
 * decided for all targets by invoking group_migrate_prepare_dst() before
 * actually starting migrating.
 */
static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
			  bool threadgroup)
B
Ben Blum 已提交
1986
{
1987 1988 1989 1990 1991
	struct cgroup_taskset tset = {
		.src_csets	= LIST_HEAD_INIT(tset.src_csets),
		.dst_csets	= LIST_HEAD_INIT(tset.dst_csets),
		.csets		= &tset.src_csets,
	};
T
Tejun Heo 已提交
1992
	struct cgroup_subsys_state *css, *failed_css = NULL;
1993 1994 1995
	struct css_set *cset, *tmp_cset;
	struct task_struct *task, *tmp_task;
	int i, ret;
B
Ben Blum 已提交
1996

1997 1998 1999 2000 2001
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
2002
	down_write(&css_set_rwsem);
2003
	rcu_read_lock();
2004
	task = leader;
B
Ben Blum 已提交
2005
	do {
2006 2007
		/* @task either already exited or can't exit until the end */
		if (task->flags & PF_EXITING)
2008
			goto next;
2009

2010 2011
		/* leave @task alone if post_fork() hasn't linked it yet */
		if (list_empty(&task->cg_list))
2012
			goto next;
2013

2014
		cset = task_css_set(task);
2015
		if (!cset->mg_src_cgrp)
2016
			goto next;
2017

2018
		/*
2019 2020
		 * cgroup_taskset_first() must always return the leader.
		 * Take care to avoid disturbing the ordering.
2021
		 */
2022 2023 2024 2025 2026 2027
		list_move_tail(&task->cg_list, &cset->mg_tasks);
		if (list_empty(&cset->mg_node))
			list_add_tail(&cset->mg_node, &tset.src_csets);
		if (list_empty(&cset->mg_dst_cset->mg_node))
			list_move_tail(&cset->mg_dst_cset->mg_node,
				       &tset.dst_csets);
2028
	next:
2029 2030
		if (!threadgroup)
			break;
2031
	} while_each_thread(leader, task);
2032
	rcu_read_unlock();
2033
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2034

2035
	/* methods shouldn't be called if no task is actually migrating */
2036 2037
	if (list_empty(&tset.src_csets))
		return 0;
2038

2039
	/* check that we can legitimately attach to the cgroup */
2040
	for_each_e_css(css, i, cgrp) {
T
Tejun Heo 已提交
2041
		if (css->ss->can_attach) {
2042 2043
			ret = css->ss->can_attach(css, &tset);
			if (ret) {
T
Tejun Heo 已提交
2044
				failed_css = css;
B
Ben Blum 已提交
2045 2046 2047 2048 2049 2050
				goto out_cancel_attach;
			}
		}
	}

	/*
2051 2052 2053
	 * Now that we're guaranteed success, proceed to move all tasks to
	 * the new cgroup.  There are no failure cases after here, so this
	 * is the commit point.
B
Ben Blum 已提交
2054
	 */
2055
	down_write(&css_set_rwsem);
2056 2057 2058 2059
	list_for_each_entry(cset, &tset.src_csets, mg_node) {
		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
			cgroup_task_migrate(cset->mg_src_cgrp, task,
					    cset->mg_dst_cset);
B
Ben Blum 已提交
2060
	}
2061
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2062 2063

	/*
2064 2065 2066
	 * Migration is committed, all target tasks are now on dst_csets.
	 * Nothing is sensitive to fork() after this point.  Notify
	 * controllers that migration is complete.
B
Ben Blum 已提交
2067
	 */
2068
	tset.csets = &tset.dst_csets;
B
Ben Blum 已提交
2069

2070
	for_each_e_css(css, i, cgrp)
T
Tejun Heo 已提交
2071 2072
		if (css->ss->attach)
			css->ss->attach(css, &tset);
B
Ben Blum 已提交
2073

2074
	ret = 0;
2075 2076
	goto out_release_tset;

B
Ben Blum 已提交
2077
out_cancel_attach:
2078
	for_each_e_css(css, i, cgrp) {
2079 2080 2081 2082
		if (css == failed_css)
			break;
		if (css->ss->cancel_attach)
			css->ss->cancel_attach(css, &tset);
B
Ben Blum 已提交
2083
	}
2084 2085 2086 2087
out_release_tset:
	down_write(&css_set_rwsem);
	list_splice_init(&tset.dst_csets, &tset.src_csets);
	list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
2088
		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2089 2090 2091
		list_del_init(&cset->mg_node);
	}
	up_write(&css_set_rwsem);
2092
	return ret;
B
Ben Blum 已提交
2093 2094
}

2095 2096 2097 2098 2099 2100
/**
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
 * @dst_cgrp: the cgroup to attach to
 * @leader: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
 *
2101
 * Call holding cgroup_mutex and threadgroup_lock of @leader.
2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129
 */
static int cgroup_attach_task(struct cgroup *dst_cgrp,
			      struct task_struct *leader, bool threadgroup)
{
	LIST_HEAD(preloaded_csets);
	struct task_struct *task;
	int ret;

	/* look up all src csets */
	down_read(&css_set_rwsem);
	rcu_read_lock();
	task = leader;
	do {
		cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
				       &preloaded_csets);
		if (!threadgroup)
			break;
	} while_each_thread(leader, task);
	rcu_read_unlock();
	up_read(&css_set_rwsem);

	/* prepare dst csets and commit */
	ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
	if (!ret)
		ret = cgroup_migrate(dst_cgrp, leader, threadgroup);

	cgroup_migrate_finish(&preloaded_csets);
	return ret;
B
Ben Blum 已提交
2130 2131 2132 2133
}

/*
 * Find the task_struct of the task to attach by vpid and pass it along to the
2134
 * function to attach either it or all tasks in its threadgroup. Will lock
2135
 * cgroup_mutex and threadgroup.
2136
 */
B
Ben Blum 已提交
2137
static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup)
2138 2139
{
	struct task_struct *tsk;
2140
	const struct cred *cred = current_cred(), *tcred;
2141 2142
	int ret;

B
Ben Blum 已提交
2143 2144 2145
	if (!cgroup_lock_live_group(cgrp))
		return -ENODEV;

2146 2147
retry_find_task:
	rcu_read_lock();
2148
	if (pid) {
2149
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2150 2151
		if (!tsk) {
			rcu_read_unlock();
S
SeongJae Park 已提交
2152
			ret = -ESRCH;
2153
			goto out_unlock_cgroup;
2154
		}
B
Ben Blum 已提交
2155 2156 2157 2158
		/*
		 * even if we're attaching all tasks in the thread group, we
		 * only need to check permissions on one of them.
		 */
2159
		tcred = __task_cred(tsk);
2160 2161 2162
		if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
		    !uid_eq(cred->euid, tcred->uid) &&
		    !uid_eq(cred->euid, tcred->suid)) {
2163
			rcu_read_unlock();
2164 2165
			ret = -EACCES;
			goto out_unlock_cgroup;
2166
		}
2167 2168
	} else
		tsk = current;
2169 2170

	if (threadgroup)
2171
		tsk = tsk->group_leader;
2172 2173

	/*
2174
	 * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2175 2176 2177
	 * trapped in a cpuset, or RT worker may be born in a cgroup
	 * with no rt_runtime allocated.  Just say no.
	 */
2178
	if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2179 2180 2181 2182 2183
		ret = -EINVAL;
		rcu_read_unlock();
		goto out_unlock_cgroup;
	}

2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200
	get_task_struct(tsk);
	rcu_read_unlock();

	threadgroup_lock(tsk);
	if (threadgroup) {
		if (!thread_group_leader(tsk)) {
			/*
			 * a race with de_thread from another thread's exec()
			 * may strip us of our leadership, if this happens,
			 * there is no choice but to throw this task away and
			 * try again; this is
			 * "double-double-toil-and-trouble-check locking".
			 */
			threadgroup_unlock(tsk);
			put_task_struct(tsk);
			goto retry_find_task;
		}
2201 2202 2203 2204
	}

	ret = cgroup_attach_task(cgrp, tsk, threadgroup);

2205 2206
	threadgroup_unlock(tsk);

2207
	put_task_struct(tsk);
2208
out_unlock_cgroup:
T
Tejun Heo 已提交
2209
	mutex_unlock(&cgroup_mutex);
2210 2211 2212
	return ret;
}

2213 2214 2215 2216 2217 2218 2219
/**
 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
 * @from: attach to all cgroups of a given task
 * @tsk: the task to be attached
 */
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
{
2220
	struct cgroup_root *root;
2221 2222
	int retval = 0;

T
Tejun Heo 已提交
2223
	mutex_lock(&cgroup_mutex);
2224
	for_each_root(root) {
2225 2226
		struct cgroup *from_cgrp;

2227
		if (root == &cgrp_dfl_root)
2228 2229
			continue;

2230 2231 2232
		down_read(&css_set_rwsem);
		from_cgrp = task_cgroup_from_root(from, root);
		up_read(&css_set_rwsem);
2233

L
Li Zefan 已提交
2234
		retval = cgroup_attach_task(from_cgrp, tsk, false);
2235 2236 2237
		if (retval)
			break;
	}
T
Tejun Heo 已提交
2238
	mutex_unlock(&cgroup_mutex);
2239 2240 2241 2242 2243

	return retval;
}
EXPORT_SYMBOL_GPL(cgroup_attach_task_all);

2244 2245
static int cgroup_tasks_write(struct cgroup_subsys_state *css,
			      struct cftype *cft, u64 pid)
B
Ben Blum 已提交
2246
{
2247
	return attach_task_by_pid(css->cgroup, pid, false);
B
Ben Blum 已提交
2248 2249
}

2250 2251
static int cgroup_procs_write(struct cgroup_subsys_state *css,
			      struct cftype *cft, u64 tgid)
2252
{
2253
	return attach_task_by_pid(css->cgroup, tgid, true);
2254 2255
}

2256
static int cgroup_release_agent_write(struct cgroup_subsys_state *css,
2257
				      struct cftype *cft, char *buffer)
2258
{
2259
	struct cgroup_root *root = css->cgroup->root;
2260 2261

	BUILD_BUG_ON(sizeof(root->release_agent_path) < PATH_MAX);
2262
	if (!cgroup_lock_live_group(css->cgroup))
2263
		return -ENODEV;
2264
	spin_lock(&release_agent_path_lock);
2265 2266
	strlcpy(root->release_agent_path, buffer,
		sizeof(root->release_agent_path));
2267
	spin_unlock(&release_agent_path_lock);
T
Tejun Heo 已提交
2268
	mutex_unlock(&cgroup_mutex);
2269 2270 2271
	return 0;
}

2272
static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2273
{
2274
	struct cgroup *cgrp = seq_css(seq)->cgroup;
2275

2276 2277 2278 2279
	if (!cgroup_lock_live_group(cgrp))
		return -ENODEV;
	seq_puts(seq, cgrp->root->release_agent_path);
	seq_putc(seq, '\n');
T
Tejun Heo 已提交
2280
	mutex_unlock(&cgroup_mutex);
2281 2282 2283
	return 0;
}

2284
static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2285
{
2286 2287 2288
	struct cgroup *cgrp = seq_css(seq)->cgroup;

	seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
2289 2290 2291
	return 0;
}

T
Tejun Heo 已提交
2292 2293
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
2294
{
T
Tejun Heo 已提交
2295 2296 2297
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = of->kn->priv;
	struct cgroup_subsys_state *css;
2298
	int ret;
2299

T
Tejun Heo 已提交
2300 2301 2302 2303 2304 2305 2306 2307 2308
	/*
	 * kernfs guarantees that a file isn't deleted with operations in
	 * flight, which means that the matching css is and stays alive and
	 * doesn't need to be pinned.  The RCU locking is not necessary
	 * either.  It's just for the convenience of using cgroup_css().
	 */
	rcu_read_lock();
	css = cgroup_css(cgrp, cft->ss);
	rcu_read_unlock();
2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323

	if (cft->write_string) {
		ret = cft->write_string(css, cft, strstrip(buf));
	} else if (cft->write_u64) {
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
	} else if (cft->trigger) {
		ret = cft->trigger(css, (unsigned int)cft->private);
2324
	} else {
2325
		ret = -EINVAL;
2326
	}
T
Tejun Heo 已提交
2327

2328
	return ret ?: nbytes;
2329 2330
}

2331
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
2332
{
T
Tejun Heo 已提交
2333
	return seq_cft(seq)->seq_start(seq, ppos);
2334 2335
}

2336
static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
2337
{
T
Tejun Heo 已提交
2338
	return seq_cft(seq)->seq_next(seq, v, ppos);
2339 2340
}

2341
static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
2342
{
T
Tejun Heo 已提交
2343
	seq_cft(seq)->seq_stop(seq, v);
2344 2345
}

2346
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2347
{
2348 2349
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
2350

2351 2352
	if (cft->seq_show)
		return cft->seq_show(m, arg);
2353

2354
	if (cft->read_u64)
2355 2356 2357 2358 2359 2360
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
2361 2362
}

T
Tejun Heo 已提交
2363 2364 2365 2366
static struct kernfs_ops cgroup_kf_single_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_show		= cgroup_seqfile_show,
2367 2368
};

T
Tejun Heo 已提交
2369 2370 2371 2372 2373 2374 2375 2376
static struct kernfs_ops cgroup_kf_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_start		= cgroup_seqfile_start,
	.seq_next		= cgroup_seqfile_next,
	.seq_stop		= cgroup_seqfile_stop,
	.seq_show		= cgroup_seqfile_show,
};
2377 2378 2379 2380

/*
 * cgroup_rename - Only allow simple rename of directories in place.
 */
T
Tejun Heo 已提交
2381 2382
static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
			 const char *new_name_str)
2383
{
T
Tejun Heo 已提交
2384
	struct cgroup *cgrp = kn->priv;
2385 2386
	int ret;

T
Tejun Heo 已提交
2387
	if (kernfs_type(kn) != KERNFS_DIR)
2388
		return -ENOTDIR;
T
Tejun Heo 已提交
2389
	if (kn->parent != new_parent)
2390
		return -EIO;
2391

2392 2393 2394 2395 2396 2397
	/*
	 * This isn't a proper migration and its usefulness is very
	 * limited.  Disallow if sane_behavior.
	 */
	if (cgroup_sane_behavior(cgrp))
		return -EPERM;
L
Li Zefan 已提交
2398

2399 2400 2401 2402 2403 2404 2405
	/*
	 * We're gonna grab cgroup_tree_mutex which nests outside kernfs
	 * active_ref.  kernfs_rename() doesn't require active_ref
	 * protection.  Break them before grabbing cgroup_tree_mutex.
	 */
	kernfs_break_active_protection(new_parent);
	kernfs_break_active_protection(kn);
L
Li Zefan 已提交
2406

T
Tejun Heo 已提交
2407 2408
	mutex_lock(&cgroup_tree_mutex);
	mutex_lock(&cgroup_mutex);
L
Li Zefan 已提交
2409

T
Tejun Heo 已提交
2410
	ret = kernfs_rename(kn, new_parent, new_name_str);
L
Li Zefan 已提交
2411

T
Tejun Heo 已提交
2412 2413
	mutex_unlock(&cgroup_mutex);
	mutex_unlock(&cgroup_tree_mutex);
2414 2415 2416

	kernfs_unbreak_active_protection(kn);
	kernfs_unbreak_active_protection(new_parent);
T
Tejun Heo 已提交
2417
	return ret;
L
Li Zefan 已提交
2418 2419
}

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
/* set uid and gid of cgroup dirs and files to that of the creator */
static int cgroup_kn_set_ugid(struct kernfs_node *kn)
{
	struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
			       .ia_uid = current_fsuid(),
			       .ia_gid = current_fsgid(), };

	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
	    gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
		return 0;

	return kernfs_setattr(kn, &iattr);
}

2434
static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2435
{
T
Tejun Heo 已提交
2436
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
2437 2438
	struct kernfs_node *kn;
	struct lock_class_key *key = NULL;
2439
	int ret;
T
Tejun Heo 已提交
2440

T
Tejun Heo 已提交
2441 2442 2443 2444 2445 2446
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	key = &cft->lockdep_key;
#endif
	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
				  cgroup_file_mode(cft), 0, cft->kf_ops, cft,
				  NULL, false, key);
2447 2448 2449 2450 2451 2452 2453
	if (IS_ERR(kn))
		return PTR_ERR(kn);

	ret = cgroup_kn_set_ugid(kn);
	if (ret)
		kernfs_remove(kn);
	return ret;
2454 2455
}

2456 2457 2458 2459 2460 2461 2462
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
 * @cgrp: the target cgroup
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
2463 2464 2465
 * For removals, this function never fails.  If addition fails, this
 * function doesn't remove files already added.  The caller is responsible
 * for cleaning up.
2466
 */
2467 2468
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add)
2469
{
A
Aristeu Rozanski 已提交
2470
	struct cftype *cft;
2471 2472
	int ret;

T
Tejun Heo 已提交
2473
	lockdep_assert_held(&cgroup_tree_mutex);
T
Tejun Heo 已提交
2474 2475

	for (cft = cfts; cft->name[0] != '\0'; cft++) {
2476
		/* does cft->flags tell us to skip this file on @cgrp? */
T
Tejun Heo 已提交
2477 2478
		if ((cft->flags & CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
			continue;
2479 2480
		if ((cft->flags & CFTYPE_INSANE) && cgroup_sane_behavior(cgrp))
			continue;
2481 2482 2483 2484 2485
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
			continue;
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
			continue;

2486
		if (is_add) {
2487
			ret = cgroup_add_file(cgrp, cft);
2488
			if (ret) {
2489
				pr_warn("cgroup_addrm_files: failed to add %s, err=%d\n",
2490 2491 2492
					cft->name, ret);
				return ret;
			}
2493 2494
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
2495
		}
2496
	}
2497
	return 0;
2498 2499
}

2500
static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
2501 2502
{
	LIST_HEAD(pending);
2503
	struct cgroup_subsys *ss = cfts[0].ss;
2504
	struct cgroup *root = &ss->root->cgrp;
2505
	struct cgroup_subsys_state *css;
2506
	int ret = 0;
2507

2508
	lockdep_assert_held(&cgroup_tree_mutex);
2509

2510
	/* add/rm files for all cgroups created before */
2511
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
2512 2513
		struct cgroup *cgrp = css->cgroup;

2514 2515 2516
		if (cgroup_is_dead(cgrp))
			continue;

2517
		ret = cgroup_addrm_files(cgrp, cfts, is_add);
2518 2519
		if (ret)
			break;
2520
	}
2521 2522 2523

	if (is_add && !ret)
		kernfs_activate(root->kn);
2524
	return ret;
2525 2526
}

2527
static void cgroup_exit_cftypes(struct cftype *cfts)
2528
{
2529
	struct cftype *cft;
2530

T
Tejun Heo 已提交
2531 2532 2533 2534 2535
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		/* free copy for custom atomic_write_len, see init_cftypes() */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
			kfree(cft->kf_ops);
		cft->kf_ops = NULL;
2536
		cft->ss = NULL;
T
Tejun Heo 已提交
2537
	}
2538 2539
}

T
Tejun Heo 已提交
2540
static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
2541 2542 2543
{
	struct cftype *cft;

T
Tejun Heo 已提交
2544 2545 2546
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		struct kernfs_ops *kf_ops;

T
Tejun Heo 已提交
2547 2548
		WARN_ON(cft->ss || cft->kf_ops);

T
Tejun Heo 已提交
2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565
		if (cft->seq_start)
			kf_ops = &cgroup_kf_ops;
		else
			kf_ops = &cgroup_kf_single_ops;

		/*
		 * Ugh... if @cft wants a custom max_write_len, we need to
		 * make a copy of kf_ops to set its atomic_write_len.
		 */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
			kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
			if (!kf_ops) {
				cgroup_exit_cftypes(cfts);
				return -ENOMEM;
			}
			kf_ops->atomic_write_len = cft->max_write_len;
		}
2566

T
Tejun Heo 已提交
2567
		cft->kf_ops = kf_ops;
2568
		cft->ss = ss;
T
Tejun Heo 已提交
2569
	}
2570

T
Tejun Heo 已提交
2571
	return 0;
2572 2573
}

2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584
static int cgroup_rm_cftypes_locked(struct cftype *cfts)
{
	lockdep_assert_held(&cgroup_tree_mutex);

	if (!cfts || !cfts[0].ss)
		return -ENOENT;

	list_del(&cfts->node);
	cgroup_apply_cftypes(cfts, false);
	cgroup_exit_cftypes(cfts);
	return 0;
2585 2586
}

2587 2588 2589 2590
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
2591 2592 2593
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
2594 2595
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
2596
 * registered.
2597
 */
2598
int cgroup_rm_cftypes(struct cftype *cfts)
2599
{
2600
	int ret;
2601

2602 2603 2604 2605
	mutex_lock(&cgroup_tree_mutex);
	ret = cgroup_rm_cftypes_locked(cfts);
	mutex_unlock(&cgroup_tree_mutex);
	return ret;
T
Tejun Heo 已提交
2606 2607
}

2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621
/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
A
Aristeu Rozanski 已提交
2622
int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
2623
{
2624
	int ret;
2625

2626 2627
	if (!cfts || cfts[0].name[0] == '\0')
		return 0;
2628

T
Tejun Heo 已提交
2629 2630 2631
	ret = cgroup_init_cftypes(ss, cfts);
	if (ret)
		return ret;
2632

2633 2634
	mutex_lock(&cgroup_tree_mutex);

T
Tejun Heo 已提交
2635
	list_add_tail(&cfts->node, &ss->cfts);
2636
	ret = cgroup_apply_cftypes(cfts, true);
2637
	if (ret)
2638
		cgroup_rm_cftypes_locked(cfts);
2639

2640
	mutex_unlock(&cgroup_tree_mutex);
2641
	return ret;
2642 2643
}

L
Li Zefan 已提交
2644 2645 2646 2647 2648 2649
/**
 * cgroup_task_count - count the number of tasks in a cgroup.
 * @cgrp: the cgroup in question
 *
 * Return the number of tasks in the cgroup.
 */
2650
static int cgroup_task_count(const struct cgroup *cgrp)
2651 2652
{
	int count = 0;
2653
	struct cgrp_cset_link *link;
2654

2655
	down_read(&css_set_rwsem);
2656 2657
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		count += atomic_read(&link->cset->refcount);
2658
	up_read(&css_set_rwsem);
2659 2660 2661
	return count;
}

2662
/**
2663 2664 2665
 * css_next_child - find the next child of a given css
 * @pos_css: the current position (%NULL to initiate traversal)
 * @parent_css: css whose children to walk
2666
 *
2667
 * This function returns the next child of @parent_css and should be called
2668 2669 2670
 * under either cgroup_mutex or RCU read lock.  The only requirement is
 * that @parent_css and @pos_css are accessible.  The next sibling is
 * guaranteed to be returned regardless of their states.
2671
 */
2672 2673 2674
struct cgroup_subsys_state *
css_next_child(struct cgroup_subsys_state *pos_css,
	       struct cgroup_subsys_state *parent_css)
2675
{
2676 2677
	struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;
	struct cgroup *cgrp = parent_css->cgroup;
2678 2679
	struct cgroup *next;

T
Tejun Heo 已提交
2680
	cgroup_assert_mutexes_or_rcu_locked();
2681 2682 2683 2684

	/*
	 * @pos could already have been removed.  Once a cgroup is removed,
	 * its ->sibling.next is no longer updated when its next sibling
2685 2686 2687 2688 2689 2690 2691
	 * changes.  As CGRP_DEAD assertion is serialized and happens
	 * before the cgroup is taken off the ->sibling list, if we see it
	 * unasserted, it's guaranteed that the next sibling hasn't
	 * finished its grace period even if it's already removed, and thus
	 * safe to dereference from this RCU critical section.  If
	 * ->sibling.next is inaccessible, cgroup_is_dead() is guaranteed
	 * to be visible as %true here.
2692 2693 2694 2695 2696 2697 2698 2699
	 *
	 * If @pos is dead, its next pointer can't be dereferenced;
	 * however, as each cgroup is given a monotonically increasing
	 * unique serial number and always appended to the sibling list,
	 * the next one can be found by walking the parent's children until
	 * we see a cgroup with higher serial number than @pos's.  While
	 * this path can be slower, it's taken only when either the current
	 * cgroup is removed or iteration and removal race.
2700
	 */
2701 2702 2703
	if (!pos) {
		next = list_entry_rcu(cgrp->children.next, struct cgroup, sibling);
	} else if (likely(!cgroup_is_dead(pos))) {
2704
		next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
2705 2706 2707 2708
	} else {
		list_for_each_entry_rcu(next, &cgrp->children, sibling)
			if (next->serial_nr > pos->serial_nr)
				break;
2709 2710
	}

2711 2712 2713 2714 2715 2716 2717
	/*
	 * @next, if not pointing to the head, can be dereferenced and is
	 * the next sibling; however, it might have @ss disabled.  If so,
	 * fast-forward to the next enabled one.
	 */
	while (&next->sibling != &cgrp->children) {
		struct cgroup_subsys_state *next_css = cgroup_css(next, parent_css->ss);
2718

2719 2720 2721 2722 2723
		if (next_css)
			return next_css;
		next = list_entry_rcu(next->sibling.next, struct cgroup, sibling);
	}
	return NULL;
2724 2725
}

2726
/**
2727
 * css_next_descendant_pre - find the next descendant for pre-order walk
2728
 * @pos: the current position (%NULL to initiate traversal)
2729
 * @root: css whose descendants to walk
2730
 *
2731
 * To be used by css_for_each_descendant_pre().  Find the next descendant
2732 2733
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
2734
 *
2735 2736 2737 2738
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
2739
 */
2740 2741 2742
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
2743
{
2744
	struct cgroup_subsys_state *next;
2745

T
Tejun Heo 已提交
2746
	cgroup_assert_mutexes_or_rcu_locked();
2747

2748
	/* if first iteration, visit @root */
2749
	if (!pos)
2750
		return root;
2751 2752

	/* visit the first child if exists */
2753
	next = css_next_child(NULL, pos);
2754 2755 2756 2757
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
2758 2759
	while (pos != root) {
		next = css_next_child(pos, css_parent(pos));
2760
		if (next)
2761
			return next;
2762
		pos = css_parent(pos);
2763
	}
2764 2765 2766 2767

	return NULL;
}

2768
/**
2769 2770
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
2771
 *
2772 2773
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
2774
 * subtree of @pos.
2775
 *
2776 2777 2778 2779
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
2780
 */
2781 2782
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
2783
{
2784
	struct cgroup_subsys_state *last, *tmp;
2785

T
Tejun Heo 已提交
2786
	cgroup_assert_mutexes_or_rcu_locked();
2787 2788 2789 2790 2791

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
2792
		css_for_each_child(tmp, last)
2793 2794 2795 2796 2797 2798
			pos = tmp;
	} while (pos);

	return last;
}

2799 2800
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
2801
{
2802
	struct cgroup_subsys_state *last;
2803 2804 2805

	do {
		last = pos;
2806
		pos = css_next_child(NULL, pos);
2807 2808 2809 2810 2811 2812
	} while (pos);

	return last;
}

/**
2813
 * css_next_descendant_post - find the next descendant for post-order walk
2814
 * @pos: the current position (%NULL to initiate traversal)
2815
 * @root: css whose descendants to walk
2816
 *
2817
 * To be used by css_for_each_descendant_post().  Find the next descendant
2818 2819
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
2820
 *
2821 2822 2823 2824 2825
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
2826
 */
2827 2828 2829
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
2830
{
2831
	struct cgroup_subsys_state *next;
2832

T
Tejun Heo 已提交
2833
	cgroup_assert_mutexes_or_rcu_locked();
2834

2835 2836 2837
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
2838

2839 2840 2841 2842
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

2843
	/* if there's an unvisited sibling, visit its leftmost descendant */
2844
	next = css_next_child(pos, css_parent(pos));
2845
	if (next)
2846
		return css_leftmost_descendant(next);
2847 2848

	/* no sibling left, visit parent */
2849
	return css_parent(pos);
2850 2851
}

2852
/**
2853
 * css_advance_task_iter - advance a task itererator to the next css_set
2854 2855 2856
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
2857
 */
2858
static void css_advance_task_iter(struct css_task_iter *it)
2859 2860 2861 2862 2863 2864 2865 2866
{
	struct list_head *l = it->cset_link;
	struct cgrp_cset_link *link;
	struct css_set *cset;

	/* Advance to the next non-empty css_set */
	do {
		l = l->next;
2867
		if (l == &it->origin_css->cgroup->cset_links) {
2868 2869 2870 2871 2872
			it->cset_link = NULL;
			return;
		}
		link = list_entry(l, struct cgrp_cset_link, cset_link);
		cset = link->cset;
T
Tejun Heo 已提交
2873 2874
	} while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));

2875
	it->cset_link = l;
T
Tejun Heo 已提交
2876 2877 2878 2879 2880

	if (!list_empty(&cset->tasks))
		it->task = cset->tasks.next;
	else
		it->task = cset->mg_tasks.next;
2881 2882
}

2883
/**
2884 2885
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
2886 2887
 * @it: the task iterator to use
 *
2888 2889 2890 2891
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
2892 2893 2894 2895 2896
 *
 * Note that this function acquires a lock which is released when the
 * iteration finishes.  The caller can't sleep while iteration is in
 * progress.
 */
2897 2898
void css_task_iter_start(struct cgroup_subsys_state *css,
			 struct css_task_iter *it)
2899
	__acquires(css_set_rwsem)
2900
{
2901 2902
	/* no one should try to iterate before mounting cgroups */
	WARN_ON_ONCE(!use_task_css_set_links);
2903

2904
	down_read(&css_set_rwsem);
2905

2906 2907
	it->origin_css = css;
	it->cset_link = &css->cgroup->cset_links;
2908

2909
	css_advance_task_iter(it);
2910 2911
}

2912
/**
2913
 * css_task_iter_next - return the next task for the iterator
2914 2915 2916
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
2917 2918
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
2919
 */
2920
struct task_struct *css_task_iter_next(struct css_task_iter *it)
2921 2922 2923
{
	struct task_struct *res;
	struct list_head *l = it->task;
T
Tejun Heo 已提交
2924 2925
	struct cgrp_cset_link *link = list_entry(it->cset_link,
					struct cgrp_cset_link, cset_link);
2926 2927

	/* If the iterator cg is NULL, we have no tasks */
2928
	if (!it->cset_link)
2929 2930
		return NULL;
	res = list_entry(l, struct task_struct, cg_list);
T
Tejun Heo 已提交
2931 2932 2933 2934 2935 2936

	/*
	 * Advance iterator to find next entry.  cset->tasks is consumed
	 * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
	 * next cset.
	 */
2937
	l = l->next;
T
Tejun Heo 已提交
2938 2939 2940 2941 2942

	if (l == &link->cset->tasks)
		l = link->cset->mg_tasks.next;

	if (l == &link->cset->mg_tasks)
2943
		css_advance_task_iter(it);
T
Tejun Heo 已提交
2944
	else
2945
		it->task = l;
T
Tejun Heo 已提交
2946

2947 2948 2949
	return res;
}

2950
/**
2951
 * css_task_iter_end - finish task iteration
2952 2953
 * @it: the task iterator to finish
 *
2954
 * Finish task iteration started by css_task_iter_start().
2955
 */
2956
void css_task_iter_end(struct css_task_iter *it)
2957
	__releases(css_set_rwsem)
2958
{
2959
	up_read(&css_set_rwsem);
2960 2961 2962
}

/**
2963 2964 2965
 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
 * @to: cgroup to which the tasks will be moved
 * @from: cgroup in which the tasks currently reside
2966
 *
2967 2968 2969 2970 2971
 * Locking rules between cgroup_post_fork() and the migration path
 * guarantee that, if a task is forking while being migrated, the new child
 * is guaranteed to be either visible in the source cgroup after the
 * parent's migration is complete or put into the target cgroup.  No task
 * can slip out of migration through forking.
2972
 */
2973
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
2974
{
2975 2976
	LIST_HEAD(preloaded_csets);
	struct cgrp_cset_link *link;
2977
	struct css_task_iter it;
2978
	struct task_struct *task;
2979
	int ret;
2980

2981
	mutex_lock(&cgroup_mutex);
2982

2983 2984 2985 2986 2987
	/* all tasks in @from are being moved, all csets are source */
	down_read(&css_set_rwsem);
	list_for_each_entry(link, &from->cset_links, cset_link)
		cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
	up_read(&css_set_rwsem);
2988

2989 2990 2991
	ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
	if (ret)
		goto out_err;
2992

2993 2994 2995 2996
	/*
	 * Migrate tasks one-by-one until @form is empty.  This fails iff
	 * ->can_attach() fails.
	 */
2997 2998 2999 3000 3001 3002 3003 3004
	do {
		css_task_iter_start(&from->dummy_css, &it);
		task = css_task_iter_next(&it);
		if (task)
			get_task_struct(task);
		css_task_iter_end(&it);

		if (task) {
3005
			ret = cgroup_migrate(to, task, false);
3006 3007 3008
			put_task_struct(task);
		}
	} while (task && !ret);
3009 3010
out_err:
	cgroup_migrate_finish(&preloaded_csets);
T
Tejun Heo 已提交
3011
	mutex_unlock(&cgroup_mutex);
3012
	return ret;
3013 3014
}

3015
/*
3016
 * Stuff for reading the 'tasks'/'procs' files.
3017 3018 3019 3020 3021 3022 3023 3024
 *
 * Reading this file can return large amounts of data if a cgroup has
 * *lots* of attached tasks. So it may need several calls to read(),
 * but we cannot guarantee that the information we produce is correct
 * unless we produce it entirely atomically.
 *
 */

3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
/* which pidlist file are we talking about? */
enum cgroup_filetype {
	CGROUP_FILE_PROCS,
	CGROUP_FILE_TASKS,
};

/*
 * A pidlist is a list of pids that virtually represents the contents of one
 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
 * a pair (one each for procs, tasks) for each pid namespace that's relevant
 * to the cgroup.
 */
struct cgroup_pidlist {
	/*
	 * used to find which pidlist is wanted. doesn't change as long as
	 * this particular list stays in the list.
	*/
	struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
	/* array of xids */
	pid_t *list;
	/* how many elements the above list has */
	int length;
	/* each of these stored in a list by its cgroup */
	struct list_head links;
	/* pointer to the cgroup we belong to, for list removal purposes */
	struct cgroup *owner;
3051 3052
	/* for delayed destruction */
	struct delayed_work destroy_dwork;
3053 3054
};

3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067
/*
 * The following two functions "fix" the issue where there are more pids
 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
 * TODO: replace with a kernel-wide solution to this problem
 */
#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
static void *pidlist_allocate(int count)
{
	if (PIDLIST_TOO_LARGE(count))
		return vmalloc(count * sizeof(pid_t));
	else
		return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
}
3068

3069 3070 3071 3072 3073 3074 3075 3076
static void pidlist_free(void *p)
{
	if (is_vmalloc_addr(p))
		vfree(p);
	else
		kfree(p);
}

3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
/*
 * Used to destroy all pidlists lingering waiting for destroy timer.  None
 * should be left afterwards.
 */
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
{
	struct cgroup_pidlist *l, *tmp_l;

	mutex_lock(&cgrp->pidlist_mutex);
	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
	mutex_unlock(&cgrp->pidlist_mutex);

	flush_workqueue(cgroup_pidlist_destroy_wq);
	BUG_ON(!list_empty(&cgrp->pidlists));
}

static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
						destroy_dwork);
	struct cgroup_pidlist *tofree = NULL;

	mutex_lock(&l->owner->pidlist_mutex);

	/*
3104 3105
	 * Destroy iff we didn't get queued again.  The state won't change
	 * as destroy_dwork can only be queued while locked.
3106
	 */
3107
	if (!delayed_work_pending(dwork)) {
3108 3109 3110 3111 3112 3113 3114 3115 3116 3117
		list_del(&l->links);
		pidlist_free(l->list);
		put_pid_ns(l->key.ns);
		tofree = l;
	}

	mutex_unlock(&l->owner->pidlist_mutex);
	kfree(tofree);
}

3118
/*
3119
 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3120
 * Returns the number of unique elements.
3121
 */
3122
static int pidlist_uniq(pid_t *list, int length)
3123
{
3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147
	int src, dest = 1;

	/*
	 * we presume the 0th element is unique, so i starts at 1. trivial
	 * edge cases first; no work needs to be done for either
	 */
	if (length == 0 || length == 1)
		return length;
	/* src and dest walk down the list; dest counts unique elements */
	for (src = 1; src < length; src++) {
		/* find next unique element */
		while (list[src] == list[src-1]) {
			src++;
			if (src == length)
				goto after;
		}
		/* dest always points to where the next unique element goes */
		list[dest] = list[src];
		dest++;
	}
after:
	return dest;
}

3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180
/*
 * The two pid files - task and cgroup.procs - guaranteed that the result
 * is sorted, which forced this whole pidlist fiasco.  As pid order is
 * different per namespace, each namespace needs differently sorted list,
 * making it impossible to use, for example, single rbtree of member tasks
 * sorted by task pointer.  As pidlists can be fairly large, allocating one
 * per open file is dangerous, so cgroup had to implement shared pool of
 * pidlists keyed by cgroup and namespace.
 *
 * All this extra complexity was caused by the original implementation
 * committing to an entirely unnecessary property.  In the long term, we
 * want to do away with it.  Explicitly scramble sort order if
 * sane_behavior so that no such expectation exists in the new interface.
 *
 * Scrambling is done by swapping every two consecutive bits, which is
 * non-identity one-to-one mapping which disturbs sort order sufficiently.
 */
static pid_t pid_fry(pid_t pid)
{
	unsigned a = pid & 0x55555555;
	unsigned b = pid & 0xAAAAAAAA;

	return (a << 1) | (b >> 1);
}

static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
{
	if (cgroup_sane_behavior(cgrp))
		return pid_fry(pid);
	else
		return pid;
}

3181 3182 3183 3184 3185
static int cmppid(const void *a, const void *b)
{
	return *(pid_t *)a - *(pid_t *)b;
}

3186 3187 3188 3189 3190
static int fried_cmppid(const void *a, const void *b)
{
	return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
}

T
Tejun Heo 已提交
3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205
static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
						  enum cgroup_filetype type)
{
	struct cgroup_pidlist *l;
	/* don't need task_nsproxy() if we're looking at ourself */
	struct pid_namespace *ns = task_active_pid_ns(current);

	lockdep_assert_held(&cgrp->pidlist_mutex);

	list_for_each_entry(l, &cgrp->pidlists, links)
		if (l->key.type == type && l->key.ns == ns)
			return l;
	return NULL;
}

3206 3207 3208 3209 3210 3211
/*
 * find the appropriate pidlist for our purpose (given procs vs tasks)
 * returns with the lock on that pidlist already held, and takes care
 * of the use count, or returns NULL with no locks held if we're out of
 * memory.
 */
T
Tejun Heo 已提交
3212 3213
static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
						enum cgroup_filetype type)
3214 3215
{
	struct cgroup_pidlist *l;
3216

T
Tejun Heo 已提交
3217 3218 3219 3220 3221 3222
	lockdep_assert_held(&cgrp->pidlist_mutex);

	l = cgroup_pidlist_find(cgrp, type);
	if (l)
		return l;

3223
	/* entry not found; create a new one */
3224
	l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
T
Tejun Heo 已提交
3225
	if (!l)
3226
		return l;
T
Tejun Heo 已提交
3227

3228
	INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3229
	l->key.type = type;
T
Tejun Heo 已提交
3230 3231
	/* don't need task_nsproxy() if we're looking at ourself */
	l->key.ns = get_pid_ns(task_active_pid_ns(current));
3232 3233 3234 3235 3236
	l->owner = cgrp;
	list_add(&l->links, &cgrp->pidlists);
	return l;
}

3237 3238 3239
/*
 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
 */
3240 3241
static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
			      struct cgroup_pidlist **lp)
3242 3243 3244 3245
{
	pid_t *array;
	int length;
	int pid, n = 0; /* used for populating the array */
3246
	struct css_task_iter it;
3247
	struct task_struct *tsk;
3248 3249
	struct cgroup_pidlist *l;

3250 3251
	lockdep_assert_held(&cgrp->pidlist_mutex);

3252 3253 3254 3255 3256 3257 3258
	/*
	 * If cgroup gets more users after we read count, we won't have
	 * enough space - tough.  This race is indistinguishable to the
	 * caller from the case that the additional cgroup users didn't
	 * show up until sometime later on.
	 */
	length = cgroup_task_count(cgrp);
3259
	array = pidlist_allocate(length);
3260 3261 3262
	if (!array)
		return -ENOMEM;
	/* now, populate the array */
3263 3264
	css_task_iter_start(&cgrp->dummy_css, &it);
	while ((tsk = css_task_iter_next(&it))) {
3265
		if (unlikely(n == length))
3266
			break;
3267
		/* get tgid or pid for procs or tasks file respectively */
3268 3269 3270 3271
		if (type == CGROUP_FILE_PROCS)
			pid = task_tgid_vnr(tsk);
		else
			pid = task_pid_vnr(tsk);
3272 3273
		if (pid > 0) /* make sure to only use valid results */
			array[n++] = pid;
3274
	}
3275
	css_task_iter_end(&it);
3276 3277
	length = n;
	/* now sort & (if procs) strip out duplicates */
3278 3279 3280 3281
	if (cgroup_sane_behavior(cgrp))
		sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
	else
		sort(array, length, sizeof(pid_t), cmppid, NULL);
3282
	if (type == CGROUP_FILE_PROCS)
3283
		length = pidlist_uniq(array, length);
T
Tejun Heo 已提交
3284 3285

	l = cgroup_pidlist_find_create(cgrp, type);
3286
	if (!l) {
T
Tejun Heo 已提交
3287
		mutex_unlock(&cgrp->pidlist_mutex);
3288
		pidlist_free(array);
3289
		return -ENOMEM;
3290
	}
T
Tejun Heo 已提交
3291 3292

	/* store array, freeing old if necessary */
3293
	pidlist_free(l->list);
3294 3295
	l->list = array;
	l->length = length;
3296
	*lp = l;
3297
	return 0;
3298 3299
}

B
Balbir Singh 已提交
3300
/**
L
Li Zefan 已提交
3301
 * cgroupstats_build - build and fill cgroupstats
B
Balbir Singh 已提交
3302 3303 3304
 * @stats: cgroupstats to fill information into
 * @dentry: A dentry entry belonging to the cgroup for which stats have
 * been requested.
L
Li Zefan 已提交
3305 3306 3307
 *
 * Build and fill cgroupstats so that taskstats can export it to user
 * space.
B
Balbir Singh 已提交
3308 3309 3310
 */
int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
{
T
Tejun Heo 已提交
3311
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
3312
	struct cgroup *cgrp;
3313
	struct css_task_iter it;
B
Balbir Singh 已提交
3314
	struct task_struct *tsk;
3315

T
Tejun Heo 已提交
3316 3317 3318 3319 3320
	/* it should be kernfs_node belonging to cgroupfs and is a directory */
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
		return -EINVAL;

3321 3322
	mutex_lock(&cgroup_mutex);

B
Balbir Singh 已提交
3323
	/*
T
Tejun Heo 已提交
3324 3325 3326
	 * We aren't being called from kernfs and there's no guarantee on
	 * @kn->priv's validity.  For this and css_tryget_from_dir(),
	 * @kn->priv is RCU safe.  Let's do the RCU dancing.
B
Balbir Singh 已提交
3327
	 */
T
Tejun Heo 已提交
3328 3329
	rcu_read_lock();
	cgrp = rcu_dereference(kn->priv);
3330
	if (!cgrp || cgroup_is_dead(cgrp)) {
T
Tejun Heo 已提交
3331
		rcu_read_unlock();
3332
		mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3333 3334
		return -ENOENT;
	}
3335
	rcu_read_unlock();
B
Balbir Singh 已提交
3336

3337 3338
	css_task_iter_start(&cgrp->dummy_css, &it);
	while ((tsk = css_task_iter_next(&it))) {
B
Balbir Singh 已提交
3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357
		switch (tsk->state) {
		case TASK_RUNNING:
			stats->nr_running++;
			break;
		case TASK_INTERRUPTIBLE:
			stats->nr_sleeping++;
			break;
		case TASK_UNINTERRUPTIBLE:
			stats->nr_uninterruptible++;
			break;
		case TASK_STOPPED:
			stats->nr_stopped++;
			break;
		default:
			if (delayacct_is_task_waiting_on_io(tsk))
				stats->nr_io_wait++;
			break;
		}
	}
3358
	css_task_iter_end(&it);
B
Balbir Singh 已提交
3359

3360
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3361
	return 0;
B
Balbir Singh 已提交
3362 3363
}

3364

3365
/*
3366
 * seq_file methods for the tasks/procs files. The seq_file position is the
3367
 * next pid to display; the seq_file iterator is a pointer to the pid
3368
 * in the cgroup->l->list array.
3369
 */
3370

3371
static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
3372
{
3373 3374 3375 3376 3377 3378
	/*
	 * Initially we receive a position value that corresponds to
	 * one more than the last pid shown (or 0 on the first call or
	 * after a seek to the start). Use a binary-search to find the
	 * next pid to display, if any
	 */
T
Tejun Heo 已提交
3379
	struct kernfs_open_file *of = s->private;
3380
	struct cgroup *cgrp = seq_css(s)->cgroup;
3381
	struct cgroup_pidlist *l;
3382
	enum cgroup_filetype type = seq_cft(s)->private;
3383
	int index = 0, pid = *pos;
3384 3385 3386 3387 3388
	int *iter, ret;

	mutex_lock(&cgrp->pidlist_mutex);

	/*
3389
	 * !NULL @of->priv indicates that this isn't the first start()
3390
	 * after open.  If the matching pidlist is around, we can use that.
3391
	 * Look for it.  Note that @of->priv can't be used directly.  It
3392 3393
	 * could already have been destroyed.
	 */
3394 3395
	if (of->priv)
		of->priv = cgroup_pidlist_find(cgrp, type);
3396 3397 3398 3399 3400

	/*
	 * Either this is the first start() after open or the matching
	 * pidlist has been destroyed inbetween.  Create a new one.
	 */
3401 3402 3403
	if (!of->priv) {
		ret = pidlist_array_load(cgrp, type,
					 (struct cgroup_pidlist **)&of->priv);
3404 3405 3406
		if (ret)
			return ERR_PTR(ret);
	}
3407
	l = of->priv;
3408 3409

	if (pid) {
3410
		int end = l->length;
S
Stephen Rothwell 已提交
3411

3412 3413
		while (index < end) {
			int mid = (index + end) / 2;
3414
			if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
3415 3416
				index = mid;
				break;
3417
			} else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
3418 3419 3420 3421 3422 3423
				index = mid + 1;
			else
				end = mid;
		}
	}
	/* If we're off the end of the array, we're done */
3424
	if (index >= l->length)
3425 3426
		return NULL;
	/* Update the abstract position to be the actual pid that we found */
3427
	iter = l->list + index;
3428
	*pos = cgroup_pid_fry(cgrp, *iter);
3429 3430 3431
	return iter;
}

3432
static void cgroup_pidlist_stop(struct seq_file *s, void *v)
3433
{
T
Tejun Heo 已提交
3434
	struct kernfs_open_file *of = s->private;
3435
	struct cgroup_pidlist *l = of->priv;
3436

3437 3438
	if (l)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
3439
				 CGROUP_PIDLIST_DESTROY_DELAY);
3440
	mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
3441 3442
}

3443
static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
3444
{
T
Tejun Heo 已提交
3445
	struct kernfs_open_file *of = s->private;
3446
	struct cgroup_pidlist *l = of->priv;
3447 3448
	pid_t *p = v;
	pid_t *end = l->list + l->length;
3449 3450 3451 3452 3453 3454 3455 3456
	/*
	 * Advance to the next pid in the array. If this goes off the
	 * end, we're done
	 */
	p++;
	if (p >= end) {
		return NULL;
	} else {
3457
		*pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
3458 3459 3460 3461
		return p;
	}
}

3462
static int cgroup_pidlist_show(struct seq_file *s, void *v)
3463 3464 3465
{
	return seq_printf(s, "%d\n", *(int *)v);
}
3466

3467 3468 3469 3470 3471 3472 3473 3474 3475
/*
 * seq_operations functions for iterating on pidlists through seq_file -
 * independent of whether it's tasks or procs
 */
static const struct seq_operations cgroup_pidlist_seq_operations = {
	.start = cgroup_pidlist_start,
	.stop = cgroup_pidlist_stop,
	.next = cgroup_pidlist_next,
	.show = cgroup_pidlist_show,
3476 3477
};

3478 3479
static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
					 struct cftype *cft)
3480
{
3481
	return notify_on_release(css->cgroup);
3482 3483
}

3484 3485
static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
					  struct cftype *cft, u64 val)
3486
{
3487
	clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
3488
	if (val)
3489
		set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3490
	else
3491
		clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3492 3493 3494
	return 0;
}

3495 3496
static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
3497
{
3498
	return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3499 3500
}

3501 3502
static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
3503 3504
{
	if (val)
3505
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3506
	else
3507
		clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3508 3509 3510
	return 0;
}

3511
static struct cftype cgroup_base_files[] = {
3512
	{
3513
		.name = "cgroup.procs",
3514 3515 3516 3517
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
3518
		.private = CGROUP_FILE_PROCS,
B
Ben Blum 已提交
3519 3520
		.write_u64 = cgroup_procs_write,
		.mode = S_IRUGO | S_IWUSR,
3521
	},
3522 3523
	{
		.name = "cgroup.clone_children",
3524
		.flags = CFTYPE_INSANE,
3525 3526 3527
		.read_u64 = cgroup_clone_children_read,
		.write_u64 = cgroup_clone_children_write,
	},
3528 3529 3530
	{
		.name = "cgroup.sane_behavior",
		.flags = CFTYPE_ONLY_ON_ROOT,
3531
		.seq_show = cgroup_sane_behavior_show,
3532
	},
3533 3534 3535 3536 3537 3538 3539 3540 3541

	/*
	 * Historical crazy stuff.  These don't have "cgroup."  prefix and
	 * don't exist if sane_behavior.  If you're depending on these, be
	 * prepared to be burned.
	 */
	{
		.name = "tasks",
		.flags = CFTYPE_INSANE,		/* use "procs" instead */
3542 3543 3544 3545
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
3546
		.private = CGROUP_FILE_TASKS,
3547 3548 3549 3550 3551 3552 3553 3554 3555
		.write_u64 = cgroup_tasks_write,
		.mode = S_IRUGO | S_IWUSR,
	},
	{
		.name = "notify_on_release",
		.flags = CFTYPE_INSANE,
		.read_u64 = cgroup_read_notify_on_release,
		.write_u64 = cgroup_write_notify_on_release,
	},
3556 3557
	{
		.name = "release_agent",
3558
		.flags = CFTYPE_INSANE | CFTYPE_ONLY_ON_ROOT,
3559
		.seq_show = cgroup_release_agent_show,
3560
		.write_string = cgroup_release_agent_write,
3561
		.max_write_len = PATH_MAX - 1,
3562
	},
T
Tejun Heo 已提交
3563
	{ }	/* terminate */
3564 3565
};

3566
/**
3567
 * cgroup_populate_dir - create subsys files in a cgroup directory
3568 3569
 * @cgrp: target cgroup
 * @subsys_mask: mask of the subsystem ids whose files should be added
3570 3571
 *
 * On failure, no file is added.
3572
 */
3573
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask)
3574 3575
{
	struct cgroup_subsys *ss;
3576
	int i, ret = 0;
3577

3578
	/* process cftsets of each subsystem */
3579
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
3580
		struct cftype *cfts;
3581 3582

		if (!test_bit(i, &subsys_mask))
3583
			continue;
3584

T
Tejun Heo 已提交
3585 3586
		list_for_each_entry(cfts, &ss->cfts, node) {
			ret = cgroup_addrm_files(cgrp, cfts, true);
3587 3588 3589
			if (ret < 0)
				goto err;
		}
3590 3591
	}
	return 0;
3592 3593 3594
err:
	cgroup_clear_dir(cgrp, subsys_mask);
	return ret;
3595 3596
}

3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
 *    and thus css_tryget() is guaranteed to fail, the css can be offlined
 *    by invoking offline_css().  After offlining, the base ref is put.
 *    Implemented in css_killed_work_fn().
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
3619
static void css_free_work_fn(struct work_struct *work)
3620 3621
{
	struct cgroup_subsys_state *css =
3622
		container_of(work, struct cgroup_subsys_state, destroy_work);
3623
	struct cgroup *cgrp = css->cgroup;
3624

3625 3626 3627
	if (css->parent)
		css_put(css->parent);

3628
	css->ss->css_free(css);
T
Tejun Heo 已提交
3629
	cgroup_put(cgrp);
3630 3631
}

3632
static void css_free_rcu_fn(struct rcu_head *rcu_head)
3633 3634
{
	struct cgroup_subsys_state *css =
3635
		container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
3636

3637
	INIT_WORK(&css->destroy_work, css_free_work_fn);
3638
	queue_work(cgroup_destroy_wq, &css->destroy_work);
3639 3640
}

3641 3642 3643 3644 3645
static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

3646
	RCU_INIT_POINTER(css->cgroup->subsys[css->ss->id], NULL);
3647
	call_rcu(&css->rcu_head, css_free_rcu_fn);
3648 3649
}

3650 3651
static void init_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss,
		     struct cgroup *cgrp)
3652
{
3653
	css->cgroup = cgrp;
3654
	css->ss = ss;
3655
	css->flags = 0;
3656 3657

	if (cgrp->parent)
3658
		css->parent = cgroup_css(cgrp->parent, ss);
3659
	else
3660
		css->flags |= CSS_ROOT;
3661

3662
	BUG_ON(cgroup_css(cgrp, ss));
3663 3664
}

3665
/* invoke ->css_online() on a new CSS and mark it online if successful */
3666
static int online_css(struct cgroup_subsys_state *css)
3667
{
3668
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
3669 3670
	int ret = 0;

T
Tejun Heo 已提交
3671
	lockdep_assert_held(&cgroup_tree_mutex);
3672 3673
	lockdep_assert_held(&cgroup_mutex);

3674
	if (ss->css_online)
3675
		ret = ss->css_online(css);
3676
	if (!ret) {
3677
		css->flags |= CSS_ONLINE;
3678
		css->cgroup->nr_css++;
3679
		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
3680
	}
T
Tejun Heo 已提交
3681
	return ret;
3682 3683
}

3684
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
3685
static void offline_css(struct cgroup_subsys_state *css)
3686
{
3687
	struct cgroup_subsys *ss = css->ss;
3688

T
Tejun Heo 已提交
3689
	lockdep_assert_held(&cgroup_tree_mutex);
3690 3691 3692 3693 3694
	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

3695
	if (ss->css_offline)
3696
		ss->css_offline(css);
3697

3698
	css->flags &= ~CSS_ONLINE;
3699
	css->cgroup->nr_css--;
3700
	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], css);
3701 3702
}

3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725
/**
 * create_css - create a cgroup_subsys_state
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
 * css is online and installed in @cgrp with all interface files created.
 * Returns 0 on success, -errno on failure.
 */
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss)
{
	struct cgroup *parent = cgrp->parent;
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgroup_mutex);

	css = ss->css_alloc(cgroup_css(parent, ss));
	if (IS_ERR(css))
		return PTR_ERR(css);

	err = percpu_ref_init(&css->refcnt, css_release);
	if (err)
3726
		goto err_free_css;
3727 3728 3729

	init_css(css, ss, cgrp);

3730
	err = cgroup_populate_dir(cgrp, 1 << ss->id);
3731
	if (err)
3732
		goto err_free_percpu_ref;
3733 3734 3735

	err = online_css(css);
	if (err)
3736
		goto err_clear_dir;
3737

3738
	cgroup_get(cgrp);
3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
	css_get(css->parent);

	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
	    parent->parent) {
		pr_warning("cgroup: %s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
			   current->comm, current->pid, ss->name);
		if (!strcmp(ss->name, "memory"))
			pr_warning("cgroup: \"memory\" requires setting use_hierarchy to 1 on the root.\n");
		ss->warned_broken_hierarchy = true;
	}

	return 0;

3752
err_clear_dir:
3753
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
3754
err_free_percpu_ref:
3755
	percpu_ref_cancel_init(&css->refcnt);
3756
err_free_css:
3757 3758 3759 3760
	ss->css_free(css);
	return err;
}

T
Tejun Heo 已提交
3761
/**
L
Li Zefan 已提交
3762 3763
 * cgroup_create - create a cgroup
 * @parent: cgroup that will be parent of the new cgroup
T
Tejun Heo 已提交
3764
 * @name: name of the new cgroup
T
Tejun Heo 已提交
3765
 * @mode: mode to set on new cgroup
3766
 */
T
Tejun Heo 已提交
3767
static long cgroup_create(struct cgroup *parent, const char *name,
T
Tejun Heo 已提交
3768
			  umode_t mode)
3769
{
3770
	struct cgroup *cgrp;
3771
	struct cgroup_root *root = parent->root;
3772
	int ssid, err;
3773
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
3774
	struct kernfs_node *kn;
3775

T
Tejun Heo 已提交
3776 3777 3778 3779 3780 3781
	/*
	 * XXX: The default hierarchy isn't fully implemented yet.  Block
	 * !root cgroup creation on it for now.
	 */
	if (root == &cgrp_dfl_root)
		return -EINVAL;
3782

T
Tejun Heo 已提交
3783
	/* allocate the cgroup and its ID, 0 is reserved for the root */
3784 3785
	cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
	if (!cgrp)
3786 3787
		return -ENOMEM;

T
Tejun Heo 已提交
3788
	mutex_lock(&cgroup_tree_mutex);
3789

3790 3791 3792 3793 3794 3795 3796 3797 3798
	/*
	 * Only live parents can have children.  Note that the liveliness
	 * check isn't strictly necessary because cgroup_mkdir() and
	 * cgroup_rmdir() are fully synchronized by i_mutex; however, do it
	 * anyway so that locking is contained inside cgroup proper and we
	 * don't get nasty surprises if we ever grow another caller.
	 */
	if (!cgroup_lock_live_group(parent)) {
		err = -ENODEV;
T
Tejun Heo 已提交
3799
		goto err_unlock_tree;
3800 3801 3802 3803 3804 3805 3806 3807 3808 3809
	}

	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
	cgrp->id = idr_alloc(&root->cgroup_idr, NULL, 1, 0, GFP_KERNEL);
	if (cgrp->id < 0) {
		err = -ENOMEM;
		goto err_unlock;
3810 3811
	}

3812
	init_cgroup_housekeeping(cgrp);
3813

3814
	cgrp->parent = parent;
3815
	cgrp->dummy_css.parent = &parent->dummy_css;
3816
	cgrp->root = parent->root;
3817

3818 3819 3820
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

3821 3822
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
3823

T
Tejun Heo 已提交
3824
	/* create the directory */
T
Tejun Heo 已提交
3825
	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
T
Tejun Heo 已提交
3826 3827
	if (IS_ERR(kn)) {
		err = PTR_ERR(kn);
3828
		goto err_free_id;
T
Tejun Heo 已提交
3829 3830
	}
	cgrp->kn = kn;
3831

3832
	/*
3833 3834
	 * This extra ref will be put in cgroup_free_fn() and guarantees
	 * that @cgrp->kn is always accessible.
3835
	 */
3836
	kernfs_get(kn);
3837

3838
	cgrp->serial_nr = cgroup_serial_nr_next++;
3839

3840 3841
	/* allocation complete, commit to creation */
	list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
3842
	atomic_inc(&root->nr_cgrps);
3843
	cgroup_get(parent);
3844

3845 3846 3847 3848
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
3849 3850
	idr_replace(&root->cgroup_idr, cgrp, cgrp->id);

3851 3852 3853 3854
	err = cgroup_kn_set_ugid(kn);
	if (err)
		goto err_destroy;

3855
	err = cgroup_addrm_files(cgrp, cgroup_base_files, true);
3856 3857 3858
	if (err)
		goto err_destroy;

3859
	/* let's create and online css's */
T
Tejun Heo 已提交
3860
	for_each_subsys(ss, ssid) {
3861
		if (parent->child_subsys_mask & (1 << ssid)) {
T
Tejun Heo 已提交
3862 3863 3864 3865
			err = create_css(cgrp, ss);
			if (err)
				goto err_destroy;
		}
3866
	}
3867

3868 3869
	cgrp->child_subsys_mask = parent->child_subsys_mask;

T
Tejun Heo 已提交
3870 3871
	kernfs_activate(kn);

3872
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3873
	mutex_unlock(&cgroup_tree_mutex);
3874 3875 3876

	return 0;

T
Tejun Heo 已提交
3877
err_free_id:
3878
	idr_remove(&root->cgroup_idr, cgrp->id);
3879 3880
err_unlock:
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3881 3882
err_unlock_tree:
	mutex_unlock(&cgroup_tree_mutex);
3883
	kfree(cgrp);
3884
	return err;
3885 3886 3887 3888

err_destroy:
	cgroup_destroy_locked(cgrp);
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3889
	mutex_unlock(&cgroup_tree_mutex);
3890
	return err;
3891 3892
}

T
Tejun Heo 已提交
3893 3894
static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
			umode_t mode)
3895
{
T
Tejun Heo 已提交
3896
	struct cgroup *parent = parent_kn->priv;
3897
	int ret;
3898

3899 3900 3901 3902 3903 3904 3905 3906
	/*
	 * cgroup_create() grabs cgroup_tree_mutex which nests outside
	 * kernfs active_ref and cgroup_create() already synchronizes
	 * properly against removal through cgroup_lock_live_group().
	 * Break it before calling cgroup_create().
	 */
	cgroup_get(parent);
	kernfs_break_active_protection(parent_kn);
3907

3908 3909 3910 3911 3912
	ret = cgroup_create(parent, name, mode);

	kernfs_unbreak_active_protection(parent_kn);
	cgroup_put(parent);
	return ret;
3913 3914
}

3915 3916 3917 3918 3919
/*
 * This is called when the refcnt of a css is confirmed to be killed.
 * css_tryget() is now guaranteed to fail.
 */
static void css_killed_work_fn(struct work_struct *work)
3920
{
3921 3922 3923
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
	struct cgroup *cgrp = css->cgroup;
3924

T
Tejun Heo 已提交
3925
	mutex_lock(&cgroup_tree_mutex);
3926 3927
	mutex_lock(&cgroup_mutex);

3928 3929 3930 3931 3932 3933
	/*
	 * css_tryget() is guaranteed to fail now.  Tell subsystems to
	 * initate destruction.
	 */
	offline_css(css);

3934 3935 3936 3937 3938
	/*
	 * If @cgrp is marked dead, it's waiting for refs of all css's to
	 * be disabled before proceeding to the second phase of cgroup
	 * destruction.  If we are the last one, kick it off.
	 */
3939
	if (!cgrp->nr_css && cgroup_is_dead(cgrp))
3940 3941 3942
		cgroup_destroy_css_killed(cgrp);

	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3943
	mutex_unlock(&cgroup_tree_mutex);
3944 3945 3946 3947 3948 3949 3950 3951 3952

	/*
	 * Put the css refs from kill_css().  Each css holds an extra
	 * reference to the cgroup's dentry and cgroup removal proceeds
	 * regardless of css refs.  On the last put of each css, whenever
	 * that may be, the extra dentry ref is put so that dentry
	 * destruction happens only after all css's are released.
	 */
	css_put(css);
3953 3954
}

3955 3956
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
3957 3958 3959 3960
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

3961
	INIT_WORK(&css->destroy_work, css_killed_work_fn);
3962
	queue_work(cgroup_destroy_wq, &css->destroy_work);
3963 3964
}

3965 3966 3967 3968 3969 3970 3971 3972 3973 3974
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
 * asynchronously once css_tryget() is guaranteed to fail and when the
 * reference count reaches zero, @css will be released.
 */
static void kill_css(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
3975
{
3976 3977
	lockdep_assert_held(&cgroup_tree_mutex);

T
Tejun Heo 已提交
3978 3979 3980 3981
	/*
	 * This must happen before css is disassociated with its cgroup.
	 * See seq_css() for details.
	 */
3982
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
3983

T
Tejun Heo 已提交
3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
	 * css_tryget().  We can't simply call percpu_ref_kill() and
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
 * guarantee that css_tryget() won't succeed by the time ->css_offline() is
 * invoked.  To satisfy all the requirements, destruction is implemented in
 * the following two steps.
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
4027 4028
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4029
{
4030
	struct cgroup *child;
T
Tejun Heo 已提交
4031
	struct cgroup_subsys_state *css;
4032
	bool empty;
T
Tejun Heo 已提交
4033
	int ssid;
4034

T
Tejun Heo 已提交
4035
	lockdep_assert_held(&cgroup_tree_mutex);
4036 4037
	lockdep_assert_held(&cgroup_mutex);

4038
	/*
4039
	 * css_set_rwsem synchronizes access to ->cset_links and prevents
4040
	 * @cgrp from being removed while put_css_set() is in progress.
4041
	 */
4042
	down_read(&css_set_rwsem);
4043
	empty = list_empty(&cgrp->cset_links);
4044
	up_read(&css_set_rwsem);
4045
	if (!empty)
4046
		return -EBUSY;
L
Li Zefan 已提交
4047

4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
	/*
	 * Make sure there's no live children.  We can't test ->children
	 * emptiness as dead children linger on it while being destroyed;
	 * otherwise, "rmdir parent/child parent" may fail with -EBUSY.
	 */
	empty = true;
	rcu_read_lock();
	list_for_each_entry_rcu(child, &cgrp->children, sibling) {
		empty = cgroup_is_dead(child);
		if (!empty)
			break;
	}
	rcu_read_unlock();
	if (!empty)
		return -EBUSY;

4064 4065 4066
	/*
	 * Mark @cgrp dead.  This prevents further task migration and child
	 * creation by disabling cgroup_lock_live_group().  Note that
4067
	 * CGRP_DEAD assertion is depended upon by css_next_child() to
4068
	 * resume iteration after dropping RCU read lock.  See
4069
	 * css_next_child() for details.
4070
	 */
4071
	set_bit(CGRP_DEAD, &cgrp->flags);
4072

4073
	/*
T
Tejun Heo 已提交
4074 4075
	 * Initiate massacre of all css's.  cgroup_destroy_css_killed()
	 * will be invoked to perform the rest of destruction once the
4076 4077
	 * percpu refs of all css's are confirmed to be killed.  This
	 * involves removing the subsystem's files, drop cgroup_mutex.
4078
	 */
4079
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
4080 4081
	for_each_css(css, ssid, cgrp)
		kill_css(css);
4082
	mutex_lock(&cgroup_mutex);
4083 4084 4085 4086 4087 4088 4089 4090

	/* CGRP_DEAD is set, remove from ->release_list for the last time */
	raw_spin_lock(&release_list_lock);
	if (!list_empty(&cgrp->release_list))
		list_del_init(&cgrp->release_list);
	raw_spin_unlock(&release_list_lock);

	/*
4091 4092 4093 4094 4095 4096 4097 4098
	 * If @cgrp has css's attached, the second stage of cgroup
	 * destruction is kicked off from css_killed_work_fn() after the
	 * refs of all attached css's are killed.  If @cgrp doesn't have
	 * any css, we kick it off here.
	 */
	if (!cgrp->nr_css)
		cgroup_destroy_css_killed(cgrp);

T
Tejun Heo 已提交
4099 4100 4101
	/* remove @cgrp directory along with the base files */
	mutex_unlock(&cgroup_mutex);

4102
	/*
T
Tejun Heo 已提交
4103 4104 4105 4106 4107
	 * There are two control paths which try to determine cgroup from
	 * dentry without going through kernfs - cgroupstats_build() and
	 * css_tryget_from_dir().  Those are supported by RCU protecting
	 * clearing of cgrp->kn->priv backpointer, which should happen
	 * after all files under it have been removed.
4108
	 */
4109
	kernfs_remove(cgrp->kn);	/* @cgrp has an extra ref on its kn */
T
Tejun Heo 已提交
4110 4111
	RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL);

4112
	mutex_lock(&cgroup_mutex);
4113

4114 4115 4116
	return 0;
};

4117
/**
4118
 * cgroup_destroy_css_killed - the second step of cgroup destruction
4119 4120 4121
 * @work: cgroup->destroy_free_work
 *
 * This function is invoked from a work item for a cgroup which is being
4122 4123 4124
 * destroyed after all css's are offlined and performs the rest of
 * destruction.  This is the second step of destruction described in the
 * comment above cgroup_destroy_locked().
4125
 */
4126
static void cgroup_destroy_css_killed(struct cgroup *cgrp)
4127 4128 4129
{
	struct cgroup *parent = cgrp->parent;

T
Tejun Heo 已提交
4130
	lockdep_assert_held(&cgroup_tree_mutex);
4131
	lockdep_assert_held(&cgroup_mutex);
4132

4133
	/* delete this cgroup from parent->children */
4134
	list_del_rcu(&cgrp->sibling);
4135

4136
	cgroup_put(cgrp);
4137

4138
	set_bit(CGRP_RELEASABLE, &parent->flags);
4139
	check_for_release(parent);
4140 4141
}

T
Tejun Heo 已提交
4142
static int cgroup_rmdir(struct kernfs_node *kn)
4143
{
T
Tejun Heo 已提交
4144 4145
	struct cgroup *cgrp = kn->priv;
	int ret = 0;
4146

T
Tejun Heo 已提交
4147 4148 4149 4150 4151 4152 4153 4154
	/*
	 * This is self-destruction but @kn can't be removed while this
	 * callback is in progress.  Let's break active protection.  Once
	 * the protection is broken, @cgrp can be destroyed at any point.
	 * Pin it so that it stays accessible.
	 */
	cgroup_get(cgrp);
	kernfs_break_active_protection(kn);
4155

T
Tejun Heo 已提交
4156
	mutex_lock(&cgroup_tree_mutex);
4157
	mutex_lock(&cgroup_mutex);
4158 4159

	/*
T
Tejun Heo 已提交
4160 4161
	 * @cgrp might already have been destroyed while we're trying to
	 * grab the mutexes.
4162
	 */
T
Tejun Heo 已提交
4163 4164
	if (!cgroup_is_dead(cgrp))
		ret = cgroup_destroy_locked(cgrp);
4165

4166
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
4167
	mutex_unlock(&cgroup_tree_mutex);
4168

T
Tejun Heo 已提交
4169 4170
	kernfs_unbreak_active_protection(kn);
	cgroup_put(cgrp);
4171
	return ret;
4172 4173
}

T
Tejun Heo 已提交
4174 4175 4176 4177 4178 4179 4180 4181
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
	.remount_fs		= cgroup_remount,
	.show_options		= cgroup_show_options,
	.mkdir			= cgroup_mkdir,
	.rmdir			= cgroup_rmdir,
	.rename			= cgroup_rename,
};

4182
static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
4183 4184
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
4185 4186

	printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4187

T
Tejun Heo 已提交
4188
	mutex_lock(&cgroup_tree_mutex);
4189 4190
	mutex_lock(&cgroup_mutex);

T
Tejun Heo 已提交
4191
	INIT_LIST_HEAD(&ss->cfts);
4192

4193 4194 4195
	/* Create the root cgroup state for this subsystem */
	ss->root = &cgrp_dfl_root;
	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
4196 4197
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
4198
	init_css(css, ss, &cgrp_dfl_root.cgrp);
4199

L
Li Zefan 已提交
4200
	/* Update the init_css_set to contain a subsys
4201
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
4202
	 * newly registered, all tasks and hence the
4203
	 * init_css_set is in the subsystem's root cgroup. */
4204
	init_css_set.subsys[ss->id] = css;
4205 4206 4207

	need_forkexit_callback |= ss->fork || ss->exit;

L
Li Zefan 已提交
4208 4209 4210 4211 4212
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

4213
	BUG_ON(online_css(css));
4214

4215
	cgrp_dfl_root.subsys_mask |= 1 << ss->id;
B
Ben Blum 已提交
4216 4217

	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
4218
	mutex_unlock(&cgroup_tree_mutex);
B
Ben Blum 已提交
4219 4220
}

4221
/**
L
Li Zefan 已提交
4222 4223 4224 4225
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
4226 4227 4228
 */
int __init cgroup_init_early(void)
{
T
Tejun Heo 已提交
4229 4230
	static struct cgroup_sb_opts __initdata opts =
		{ .flags = CGRP_ROOT_SANE_BEHAVIOR };
4231
	struct cgroup_subsys *ss;
4232
	int i;
4233

4234
	init_cgroup_root(&cgrp_dfl_root, &opts);
4235
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4236

T
Tejun Heo 已提交
4237
	for_each_subsys(ss, i) {
4238
		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
4239 4240
		     "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
4241
		     ss->id, ss->name);
4242 4243 4244
		WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
		     "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);

4245
		ss->id = i;
4246
		ss->name = cgroup_subsys_name[i];
4247 4248 4249 4250 4251 4252 4253 4254

		if (ss->early_init)
			cgroup_init_subsys(ss);
	}
	return 0;
}

/**
L
Li Zefan 已提交
4255 4256 4257 4258
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
4259 4260 4261
 */
int __init cgroup_init(void)
{
4262
	struct cgroup_subsys *ss;
4263
	unsigned long key;
4264
	int ssid, err;
4265

T
Tejun Heo 已提交
4266
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
4267

4268
	mutex_lock(&cgroup_tree_mutex);
T
Tejun Heo 已提交
4269 4270
	mutex_lock(&cgroup_mutex);

4271 4272 4273 4274
	/* Add init_css_set to the hash table */
	key = css_set_hash(init_css_set.subsys);
	hash_add(css_set_table, &init_css_set.hlist, key);

4275
	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4276

T
Tejun Heo 已提交
4277
	mutex_unlock(&cgroup_mutex);
4278
	mutex_unlock(&cgroup_tree_mutex);
T
Tejun Heo 已提交
4279

4280 4281 4282 4283
	for_each_subsys(ss, ssid) {
		if (!ss->early_init)
			cgroup_init_subsys(ss);

T
Tejun Heo 已提交
4284 4285 4286
		list_add_tail(&init_css_set.e_cset_node[ssid],
			      &cgrp_dfl_root.cgrp.e_csets[ssid]);

4287 4288 4289 4290 4291 4292
		/*
		 * cftype registration needs kmalloc and can't be done
		 * during early_init.  Register base cftypes separately.
		 */
		if (ss->base_cftypes)
			WARN_ON(cgroup_add_cftypes(ss, ss->base_cftypes));
4293 4294 4295
	}

	cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
T
Tejun Heo 已提交
4296 4297
	if (!cgroup_kobj)
		return -ENOMEM;
4298

4299
	err = register_filesystem(&cgroup_fs_type);
4300 4301
	if (err < 0) {
		kobject_put(cgroup_kobj);
T
Tejun Heo 已提交
4302
		return err;
4303
	}
4304

L
Li Zefan 已提交
4305
	proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
T
Tejun Heo 已提交
4306
	return 0;
4307
}
4308

4309 4310 4311 4312 4313
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
4314
	 * Use 1 for @max_active.
4315 4316 4317 4318
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
4319
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
4320
	BUG_ON(!cgroup_destroy_wq);
4321 4322 4323 4324 4325 4326 4327 4328 4329

	/*
	 * Used to destroy pidlists and separate to serve as flush domain.
	 * Cap @max_active to 1 too.
	 */
	cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
						    0, 1);
	BUG_ON(!cgroup_pidlist_destroy_wq);

4330 4331 4332 4333
	return 0;
}
core_initcall(cgroup_wq_init);

4334 4335 4336 4337 4338 4339 4340
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 */

/* TODO: Use a proper seq_file iterator */
4341
int proc_cgroup_show(struct seq_file *m, void *v)
4342 4343 4344
{
	struct pid *pid;
	struct task_struct *tsk;
T
Tejun Heo 已提交
4345
	char *buf, *path;
4346
	int retval;
4347
	struct cgroup_root *root;
4348 4349

	retval = -ENOMEM;
T
Tejun Heo 已提交
4350
	buf = kmalloc(PATH_MAX, GFP_KERNEL);
4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362
	if (!buf)
		goto out;

	retval = -ESRCH;
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
	if (!tsk)
		goto out_free;

	retval = 0;

	mutex_lock(&cgroup_mutex);
4363
	down_read(&css_set_rwsem);
4364

4365
	for_each_root(root) {
4366
		struct cgroup_subsys *ss;
4367
		struct cgroup *cgrp;
T
Tejun Heo 已提交
4368
		int ssid, count = 0;
4369

T
Tejun Heo 已提交
4370
		if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
4371 4372
			continue;

4373
		seq_printf(m, "%d:", root->hierarchy_id);
T
Tejun Heo 已提交
4374
		for_each_subsys(ss, ssid)
4375
			if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
4376
				seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
4377 4378 4379
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
4380
		seq_putc(m, ':');
4381
		cgrp = task_cgroup_from_root(tsk, root);
T
Tejun Heo 已提交
4382 4383 4384
		path = cgroup_path(cgrp, buf, PATH_MAX);
		if (!path) {
			retval = -ENAMETOOLONG;
4385
			goto out_unlock;
T
Tejun Heo 已提交
4386 4387
		}
		seq_puts(m, path);
4388 4389 4390 4391
		seq_putc(m, '\n');
	}

out_unlock:
4392
	up_read(&css_set_rwsem);
4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403
	mutex_unlock(&cgroup_mutex);
	put_task_struct(tsk);
out_free:
	kfree(buf);
out:
	return retval;
}

/* Display information about each subsystem and each hierarchy */
static int proc_cgroupstats_show(struct seq_file *m, void *v)
{
4404
	struct cgroup_subsys *ss;
4405 4406
	int i;

4407
	seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
B
Ben Blum 已提交
4408 4409 4410 4411 4412
	/*
	 * ideally we don't want subsystems moving around while we do this.
	 * cgroup_mutex is also necessary to guarantee an atomic snapshot of
	 * subsys/hierarchy state.
	 */
4413
	mutex_lock(&cgroup_mutex);
4414 4415

	for_each_subsys(ss, i)
4416 4417
		seq_printf(m, "%s\t%d\t%d\t%d\n",
			   ss->name, ss->root->hierarchy_id,
4418
			   atomic_read(&ss->root->nr_cgrps), !ss->disabled);
4419

4420 4421 4422 4423 4424 4425
	mutex_unlock(&cgroup_mutex);
	return 0;
}

static int cgroupstats_open(struct inode *inode, struct file *file)
{
A
Al Viro 已提交
4426
	return single_open(file, proc_cgroupstats_show, NULL);
4427 4428
}

4429
static const struct file_operations proc_cgroupstats_operations = {
4430 4431 4432 4433 4434 4435
	.open = cgroupstats_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

4436
/**
4437
 * cgroup_fork - initialize cgroup related fields during copy_process()
L
Li Zefan 已提交
4438
 * @child: pointer to task_struct of forking parent process.
4439
 *
4440 4441 4442
 * A task is associated with the init_css_set until cgroup_post_fork()
 * attaches it to the parent's css_set.  Empty cg_list indicates that
 * @child isn't holding reference to its css_set.
4443 4444 4445
 */
void cgroup_fork(struct task_struct *child)
{
4446
	RCU_INIT_POINTER(child->cgroups, &init_css_set);
4447
	INIT_LIST_HEAD(&child->cg_list);
4448 4449
}

4450
/**
L
Li Zefan 已提交
4451 4452 4453
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
4454 4455 4456
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
4457
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
4458
 * list.
L
Li Zefan 已提交
4459
 */
4460 4461
void cgroup_post_fork(struct task_struct *child)
{
4462
	struct cgroup_subsys *ss;
4463 4464
	int i;

4465
	/*
4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484
	 * This may race against cgroup_enable_task_cg_links().  As that
	 * function sets use_task_css_set_links before grabbing
	 * tasklist_lock and we just went through tasklist_lock to add
	 * @child, it's guaranteed that either we see the set
	 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
	 * @child during its iteration.
	 *
	 * If we won the race, @child is associated with %current's
	 * css_set.  Grabbing css_set_rwsem guarantees both that the
	 * association is stable, and, on completion of the parent's
	 * migration, @child is visible in the source of migration or
	 * already in the destination cgroup.  This guarantee is necessary
	 * when implementing operations which need to migrate all tasks of
	 * a cgroup to another.
	 *
	 * Note that if we lose to cgroup_enable_task_cg_links(), @child
	 * will remain in init_css_set.  This is safe because all tasks are
	 * in the init_css_set before cg_links is enabled and there's no
	 * operation which transfers all tasks out of init_css_set.
4485
	 */
4486
	if (use_task_css_set_links) {
4487 4488
		struct css_set *cset;

4489
		down_write(&css_set_rwsem);
4490
		cset = task_css_set(current);
4491 4492 4493 4494 4495
		if (list_empty(&child->cg_list)) {
			rcu_assign_pointer(child->cgroups, cset);
			list_add(&child->cg_list, &cset->tasks);
			get_css_set(cset);
		}
4496
		up_write(&css_set_rwsem);
4497
	}
4498 4499 4500 4501 4502 4503 4504

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
4505
		for_each_subsys(ss, i)
4506 4507 4508
			if (ss->fork)
				ss->fork(child);
	}
4509
}
4510

4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
4523 4524 4525 4526 4527
 * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
 * call cgroup_exit() while the task is still competent to handle
 * notify_on_release(), then leave the task attached to the root cgroup in
 * each hierarchy for the remainder of its exit.  No need to bother with
 * init_css_set refcnting.  init_css_set never goes away and we can't race
4528
 * with migration path - PF_EXITING is visible to migration path.
4529
 */
4530
void cgroup_exit(struct task_struct *tsk)
4531
{
4532
	struct cgroup_subsys *ss;
4533
	struct css_set *cset;
4534
	bool put_cset = false;
4535
	int i;
4536 4537

	/*
4538 4539
	 * Unlink from @tsk from its css_set.  As migration path can't race
	 * with us, we can check cg_list without grabbing css_set_rwsem.
4540 4541
	 */
	if (!list_empty(&tsk->cg_list)) {
4542
		down_write(&css_set_rwsem);
4543
		list_del_init(&tsk->cg_list);
4544
		up_write(&css_set_rwsem);
4545
		put_cset = true;
4546 4547
	}

4548
	/* Reassign the task to the init_css_set. */
4549 4550
	cset = task_css_set(tsk);
	RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
4551

4552
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
4553 4554
		/* see cgroup_post_fork() for details */
		for_each_subsys(ss, i) {
4555
			if (ss->exit) {
4556 4557
				struct cgroup_subsys_state *old_css = cset->subsys[i];
				struct cgroup_subsys_state *css = task_css(tsk, i);
4558

4559
				ss->exit(css, old_css, tsk);
4560 4561 4562 4563
			}
		}
	}

4564 4565
	if (put_cset)
		put_css_set(cset, true);
4566
}
4567

4568
static void check_for_release(struct cgroup *cgrp)
4569
{
4570
	if (cgroup_is_releasable(cgrp) &&
T
Tejun Heo 已提交
4571
	    list_empty(&cgrp->cset_links) && list_empty(&cgrp->children)) {
4572 4573
		/*
		 * Control Group is currently removeable. If it's not
4574
		 * already queued for a userspace notification, queue
4575 4576
		 * it now
		 */
4577
		int need_schedule_work = 0;
4578

4579
		raw_spin_lock(&release_list_lock);
4580
		if (!cgroup_is_dead(cgrp) &&
4581 4582
		    list_empty(&cgrp->release_list)) {
			list_add(&cgrp->release_list, &release_list);
4583 4584
			need_schedule_work = 1;
		}
4585
		raw_spin_unlock(&release_list_lock);
4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617
		if (need_schedule_work)
			schedule_work(&release_agent_work);
	}
}

/*
 * Notify userspace when a cgroup is released, by running the
 * configured release agent with the name of the cgroup (path
 * relative to the root of cgroup file system) as the argument.
 *
 * Most likely, this user command will try to rmdir this cgroup.
 *
 * This races with the possibility that some other task will be
 * attached to this cgroup before it is removed, or that some other
 * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
 * unused, and this cgroup will be reprieved from its death sentence,
 * to continue to serve a useful existence.  Next time it's released,
 * we will get notified again, if it still has 'notify_on_release' set.
 *
 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
 * means only wait until the task is successfully execve()'d.  The
 * separate release agent task is forked by call_usermodehelper(),
 * then control in this thread returns here, without waiting for the
 * release agent task.  We don't bother to wait because the caller of
 * this routine has no use for the exit status of the release agent
 * task, so no sense holding our caller up for that.
 */
static void cgroup_release_agent(struct work_struct *work)
{
	BUG_ON(work != &release_agent_work);
	mutex_lock(&cgroup_mutex);
4618
	raw_spin_lock(&release_list_lock);
4619 4620 4621
	while (!list_empty(&release_list)) {
		char *argv[3], *envp[3];
		int i;
T
Tejun Heo 已提交
4622
		char *pathbuf = NULL, *agentbuf = NULL, *path;
4623
		struct cgroup *cgrp = list_entry(release_list.next,
4624 4625
						    struct cgroup,
						    release_list);
4626
		list_del_init(&cgrp->release_list);
4627
		raw_spin_unlock(&release_list_lock);
T
Tejun Heo 已提交
4628
		pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
4629 4630
		if (!pathbuf)
			goto continue_free;
T
Tejun Heo 已提交
4631 4632
		path = cgroup_path(cgrp, pathbuf, PATH_MAX);
		if (!path)
4633 4634 4635 4636
			goto continue_free;
		agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
		if (!agentbuf)
			goto continue_free;
4637 4638

		i = 0;
4639
		argv[i++] = agentbuf;
T
Tejun Heo 已提交
4640
		argv[i++] = path;
4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654
		argv[i] = NULL;

		i = 0;
		/* minimal command environment */
		envp[i++] = "HOME=/";
		envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
		envp[i] = NULL;

		/* Drop the lock while we invoke the usermode helper,
		 * since the exec could involve hitting disk and hence
		 * be a slow process */
		mutex_unlock(&cgroup_mutex);
		call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
		mutex_lock(&cgroup_mutex);
4655 4656 4657
 continue_free:
		kfree(pathbuf);
		kfree(agentbuf);
4658
		raw_spin_lock(&release_list_lock);
4659
	}
4660
	raw_spin_unlock(&release_list_lock);
4661 4662
	mutex_unlock(&cgroup_mutex);
}
4663 4664 4665

static int __init cgroup_disable(char *str)
{
4666
	struct cgroup_subsys *ss;
4667
	char *token;
4668
	int i;
4669 4670 4671 4672

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
4673

T
Tejun Heo 已提交
4674
		for_each_subsys(ss, i) {
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685
			if (!strcmp(token, ss->name)) {
				ss->disabled = 1;
				printk(KERN_INFO "Disabling %s control group"
					" subsystem\n", ss->name);
				break;
			}
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
4686

4687
/**
4688
 * css_tryget_from_dir - get corresponding css from the dentry of a cgroup dir
4689 4690
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
4691
 *
4692 4693 4694
 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
 * to get the corresponding css and return it.  If such css doesn't exist
 * or can't be pinned, an ERR_PTR value is returned.
S
Stephane Eranian 已提交
4695
 */
4696 4697
struct cgroup_subsys_state *css_tryget_from_dir(struct dentry *dentry,
						struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
4698
{
T
Tejun Heo 已提交
4699 4700
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
	struct cgroup_subsys_state *css = NULL;
S
Stephane Eranian 已提交
4701 4702
	struct cgroup *cgrp;

4703
	/* is @dentry a cgroup dir? */
T
Tejun Heo 已提交
4704 4705
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
S
Stephane Eranian 已提交
4706 4707
		return ERR_PTR(-EBADF);

4708 4709
	rcu_read_lock();

T
Tejun Heo 已提交
4710 4711 4712 4713 4714 4715 4716 4717
	/*
	 * This path doesn't originate from kernfs and @kn could already
	 * have been or be removed at any point.  @kn->priv is RCU
	 * protected for this access.  See destroy_locked() for details.
	 */
	cgrp = rcu_dereference(kn->priv);
	if (cgrp)
		css = cgroup_css(cgrp, ss);
4718 4719 4720 4721 4722 4723

	if (!css || !css_tryget(css))
		css = ERR_PTR(-ENOENT);

	rcu_read_unlock();
	return css;
S
Stephane Eranian 已提交
4724 4725
}

4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
	struct cgroup *cgrp;

T
Tejun Heo 已提交
4738
	cgroup_assert_mutexes_or_rcu_locked();
4739 4740 4741

	cgrp = idr_find(&ss->root->cgroup_idr, id);
	if (cgrp)
4742
		return cgroup_css(cgrp, ss);
4743
	return NULL;
S
Stephane Eranian 已提交
4744 4745
}

4746
#ifdef CONFIG_CGROUP_DEBUG
4747 4748
static struct cgroup_subsys_state *
debug_css_alloc(struct cgroup_subsys_state *parent_css)
4749 4750 4751 4752 4753 4754 4755 4756 4757
{
	struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);

	if (!css)
		return ERR_PTR(-ENOMEM);

	return css;
}

4758
static void debug_css_free(struct cgroup_subsys_state *css)
4759
{
4760
	kfree(css);
4761 4762
}

4763 4764
static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
4765
{
4766
	return cgroup_task_count(css->cgroup);
4767 4768
}

4769 4770
static u64 current_css_set_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
4771 4772 4773 4774
{
	return (u64)(unsigned long)current->cgroups;
}

4775
static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
L
Li Zefan 已提交
4776
					 struct cftype *cft)
4777 4778 4779 4780
{
	u64 count;

	rcu_read_lock();
4781
	count = atomic_read(&task_css_set(current)->refcount);
4782 4783 4784 4785
	rcu_read_unlock();
	return count;
}

4786
static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
4787
{
4788
	struct cgrp_cset_link *link;
4789
	struct css_set *cset;
T
Tejun Heo 已提交
4790 4791 4792 4793 4794
	char *name_buf;

	name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
	if (!name_buf)
		return -ENOMEM;
4795

4796
	down_read(&css_set_rwsem);
4797
	rcu_read_lock();
4798
	cset = rcu_dereference(current->cgroups);
4799
	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
4800 4801
		struct cgroup *c = link->cgrp;

T
Tejun Heo 已提交
4802
		cgroup_name(c, name_buf, NAME_MAX + 1);
4803
		seq_printf(seq, "Root %d group %s\n",
T
Tejun Heo 已提交
4804
			   c->root->hierarchy_id, name_buf);
4805 4806
	}
	rcu_read_unlock();
4807
	up_read(&css_set_rwsem);
T
Tejun Heo 已提交
4808
	kfree(name_buf);
4809 4810 4811 4812
	return 0;
}

#define MAX_TASKS_SHOWN_PER_CSS 25
4813
static int cgroup_css_links_read(struct seq_file *seq, void *v)
4814
{
4815
	struct cgroup_subsys_state *css = seq_css(seq);
4816
	struct cgrp_cset_link *link;
4817

4818
	down_read(&css_set_rwsem);
4819
	list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
4820
		struct css_set *cset = link->cset;
4821 4822
		struct task_struct *task;
		int count = 0;
T
Tejun Heo 已提交
4823

4824
		seq_printf(seq, "css_set %p\n", cset);
T
Tejun Heo 已提交
4825

4826
		list_for_each_entry(task, &cset->tasks, cg_list) {
T
Tejun Heo 已提交
4827 4828 4829 4830 4831 4832 4833 4834 4835
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
		}

		list_for_each_entry(task, &cset->mg_tasks, cg_list) {
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
4836
		}
T
Tejun Heo 已提交
4837 4838 4839
		continue;
	overflow:
		seq_puts(seq, "  ...\n");
4840
	}
4841
	up_read(&css_set_rwsem);
4842 4843 4844
	return 0;
}

4845
static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
4846
{
4847
	return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865
}

static struct cftype debug_files[] =  {
	{
		.name = "taskcount",
		.read_u64 = debug_taskcount_read,
	},

	{
		.name = "current_css_set",
		.read_u64 = current_css_set_read,
	},

	{
		.name = "current_css_set_refcount",
		.read_u64 = current_css_set_refcount_read,
	},

4866 4867
	{
		.name = "current_css_set_cg_links",
4868
		.seq_show = current_css_set_cg_links_read,
4869 4870 4871 4872
	},

	{
		.name = "cgroup_css_links",
4873
		.seq_show = cgroup_css_links_read,
4874 4875
	},

4876 4877 4878 4879 4880
	{
		.name = "releasable",
		.read_u64 = releasable_read,
	},

4881 4882
	{ }	/* terminate */
};
4883

4884
struct cgroup_subsys debug_cgrp_subsys = {
4885 4886
	.css_alloc = debug_css_alloc,
	.css_free = debug_css_free,
4887
	.base_cftypes = debug_files,
4888 4889
};
#endif /* CONFIG_CGROUP_DEBUG */