cgroup.c 147.3 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/cgroup.h>
32
#include <linux/cred.h>
33
#include <linux/ctype.h>
34
#include <linux/errno.h>
35
#include <linux/init_task.h>
36 37
#include <linux/kernel.h>
#include <linux/list.h>
38
#include <linux/magic.h>
39 40 41 42
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
43
#include <linux/proc_fs.h>
44 45 46 47
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
48
#include <linux/rwsem.h>
49
#include <linux/string.h>
50
#include <linux/sort.h>
51
#include <linux/kmod.h>
B
Balbir Singh 已提交
52 53
#include <linux/delayacct.h>
#include <linux/cgroupstats.h>
54
#include <linux/hashtable.h>
L
Li Zefan 已提交
55
#include <linux/pid_namespace.h>
56
#include <linux/idr.h>
57
#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
58
#include <linux/kthread.h>
T
Tejun Heo 已提交
59
#include <linux/delay.h>
B
Balbir Singh 已提交
60

A
Arun Sharma 已提交
61
#include <linux/atomic.h>
62

63 64 65 66 67 68 69 70
/*
 * pidlists linger the following amount before being destroyed.  The goal
 * is avoiding frequent destruction in the middle of consecutive read calls
 * Expiring in the middle is a performance problem not a correctness one.
 * 1 sec should be enough.
 */
#define CGROUP_PIDLIST_DESTROY_DELAY	HZ

T
Tejun Heo 已提交
71 72 73
#define CGROUP_FILE_NAME_MAX		(MAX_CGROUP_TYPE_NAMELEN +	\
					 MAX_CFTYPE_NAME + 2)

T
Tejun Heo 已提交
74 75 76 77
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
78 79
 * css_set_rwsem protects task->cgroups pointer, the list of css_set
 * objects, and the chain of tasks off each css_set.
T
Tejun Heo 已提交
80
 *
81 82
 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 * cgroup.h can use them for lockdep annotations.
T
Tejun Heo 已提交
83
 */
T
Tejun Heo 已提交
84 85
#ifdef CONFIG_PROVE_RCU
DEFINE_MUTEX(cgroup_mutex);
86 87 88
DECLARE_RWSEM(css_set_rwsem);
EXPORT_SYMBOL_GPL(cgroup_mutex);
EXPORT_SYMBOL_GPL(css_set_rwsem);
T
Tejun Heo 已提交
89
#else
90
static DEFINE_MUTEX(cgroup_mutex);
91
static DECLARE_RWSEM(css_set_rwsem);
T
Tejun Heo 已提交
92 93
#endif

94
/*
95 96
 * Protects cgroup_idr and css_idr so that IDs can be released without
 * grabbing cgroup_mutex.
97 98 99
 */
static DEFINE_SPINLOCK(cgroup_idr_lock);

100 101 102 103 104
/*
 * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
 * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
 */
static DEFINE_SPINLOCK(release_agent_path_lock);
105

T
Tejun Heo 已提交
106
#define cgroup_assert_mutex_or_rcu_locked()				\
107 108
	rcu_lockdep_assert(rcu_read_lock_held() ||			\
			   lockdep_is_held(&cgroup_mutex),		\
T
Tejun Heo 已提交
109
			   "cgroup_mutex or RCU read lock required");
110

111 112 113 114 115 116 117 118
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

119 120 121 122 123 124
/*
 * pidlist destructions need to be flushed on cgroup destruction.  Use a
 * separate workqueue as flush domain.
 */
static struct workqueue_struct *cgroup_pidlist_destroy_wq;

T
Tejun Heo 已提交
125
/* generate an array of cgroup subsystem pointers */
126
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
T
Tejun Heo 已提交
127
static struct cgroup_subsys *cgroup_subsys[] = {
128 129
#include <linux/cgroup_subsys.h>
};
130 131 132 133 134
#undef SUBSYS

/* array of cgroup subsystem names */
#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
static const char *cgroup_subsys_name[] = {
135 136
#include <linux/cgroup_subsys.h>
};
137
#undef SUBSYS
138 139

/*
140
 * The default hierarchy, reserved for the subsystems that are otherwise
141 142
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
143
 */
T
Tejun Heo 已提交
144
struct cgroup_root cgrp_dfl_root;
145

T
Tejun Heo 已提交
146 147 148 149 150
/*
 * The default hierarchy always exists but is hidden until mounted for the
 * first time.  This is for backward compatibility.
 */
static bool cgrp_dfl_root_visible;
151

152 153 154 155 156 157 158
/* some controllers are not supported in the default hierarchy */
static const unsigned int cgrp_dfl_root_inhibit_ss_mask = 0
#ifdef CONFIG_CGROUP_DEBUG
	| (1 << debug_cgrp_id)
#endif
	;

159 160
/* The list of hierarchy roots */

161 162
static LIST_HEAD(cgroup_roots);
static int cgroup_root_count;
163

T
Tejun Heo 已提交
164
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
165
static DEFINE_IDR(cgroup_hierarchy_idr);
166

167
/*
168 169 170 171 172
 * Assign a monotonically increasing serial number to csses.  It guarantees
 * cgroups with bigger numbers are newer than those with smaller numbers.
 * Also, as csses are always appended to the parent's ->children list, it
 * guarantees that sibling csses are always sorted in the ascending serial
 * number order on the list.  Protected by cgroup_mutex.
173
 */
174
static u64 css_serial_nr_next = 1;
175

176
/* This flag indicates whether tasks in the fork and exit paths should
L
Li Zefan 已提交
177 178 179
 * check for fork/exit handlers to call. This avoids us having to do
 * extra work in the fork/exit path if none of the subsystems need to
 * be called.
180
 */
181
static int need_forkexit_callback __read_mostly;
182

183 184
static struct cftype cgroup_base_files[];

185
static void cgroup_put(struct cgroup *cgrp);
186
static int rebind_subsystems(struct cgroup_root *dst_root,
187
			     unsigned int ss_mask);
188
static int cgroup_destroy_locked(struct cgroup *cgrp);
189 190
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
		      bool visible);
191
static void css_release(struct percpu_ref *ref);
192
static void kill_css(struct cgroup_subsys_state *css);
193 194
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add);
195
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
196

197 198 199 200 201 202 203
/* IDR wrappers which synchronize using cgroup_idr_lock */
static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
			    gfp_t gfp_mask)
{
	int ret;

	idr_preload(gfp_mask);
T
Tejun Heo 已提交
204
	spin_lock_bh(&cgroup_idr_lock);
205
	ret = idr_alloc(idr, ptr, start, end, gfp_mask);
T
Tejun Heo 已提交
206
	spin_unlock_bh(&cgroup_idr_lock);
207 208 209 210 211 212 213 214
	idr_preload_end();
	return ret;
}

static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
{
	void *ret;

T
Tejun Heo 已提交
215
	spin_lock_bh(&cgroup_idr_lock);
216
	ret = idr_replace(idr, ptr, id);
T
Tejun Heo 已提交
217
	spin_unlock_bh(&cgroup_idr_lock);
218 219 220 221 222
	return ret;
}

static void cgroup_idr_remove(struct idr *idr, int id)
{
T
Tejun Heo 已提交
223
	spin_lock_bh(&cgroup_idr_lock);
224
	idr_remove(idr, id);
T
Tejun Heo 已提交
225
	spin_unlock_bh(&cgroup_idr_lock);
226 227
}

T
Tejun Heo 已提交
228 229 230 231 232 233 234 235 236
static struct cgroup *cgroup_parent(struct cgroup *cgrp)
{
	struct cgroup_subsys_state *parent_css = cgrp->self.parent;

	if (parent_css)
		return container_of(parent_css, struct cgroup, self);
	return NULL;
}

T
Tejun Heo 已提交
237 238 239
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
240
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
T
Tejun Heo 已提交
241
 *
242 243 244 245 246
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
247 248
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
249
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
250
{
251
	if (ss)
252
		return rcu_dereference_check(cgrp->subsys[ss->id],
T
Tejun Heo 已提交
253
					lockdep_is_held(&cgroup_mutex));
254
	else
255
		return &cgrp->self;
T
Tejun Heo 已提交
256
}
257

258 259 260
/**
 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
261
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
262 263 264 265 266 267 268 269 270 271 272 273
 *
 * Similar to cgroup_css() but returns the effctive css, which is defined
 * as the matching css of the nearest ancestor including self which has @ss
 * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
 * function is guaranteed to return non-NULL css.
 */
static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
						struct cgroup_subsys *ss)
{
	lockdep_assert_held(&cgroup_mutex);

	if (!ss)
274
		return &cgrp->self;
275 276 277 278

	if (!(cgrp->root->subsys_mask & (1 << ss->id)))
		return NULL;

T
Tejun Heo 已提交
279 280 281
	while (cgroup_parent(cgrp) &&
	       !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
		cgrp = cgroup_parent(cgrp);
282 283

	return cgroup_css(cgrp, ss);
T
Tejun Heo 已提交
284
}
285

286
/* convenient tests for these bits */
287
static inline bool cgroup_is_dead(const struct cgroup *cgrp)
288
{
289
	return !(cgrp->self.flags & CSS_ONLINE);
290 291
}

T
Tejun Heo 已提交
292
struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
293
{
T
Tejun Heo 已提交
294
	struct cgroup *cgrp = of->kn->parent->priv;
T
Tejun Heo 已提交
295
	struct cftype *cft = of_cft(of);
T
Tejun Heo 已提交
296 297 298 299 300 301 302 303 304 305 306 307

	/*
	 * This is open and unprotected implementation of cgroup_css().
	 * seq_css() is only called from a kernfs file operation which has
	 * an active reference on the file.  Because all the subsystem
	 * files are drained before a css is disassociated with a cgroup,
	 * the matching css from the cgroup's subsys table is guaranteed to
	 * be and stay valid until the enclosing operation is complete.
	 */
	if (cft->ss)
		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
	else
308
		return &cgrp->self;
309
}
T
Tejun Heo 已提交
310
EXPORT_SYMBOL_GPL(of_css);
311

312 313 314 315 316 317 318 319 320 321 322 323 324 325
/**
 * cgroup_is_descendant - test ancestry
 * @cgrp: the cgroup to be tested
 * @ancestor: possible ancestor of @cgrp
 *
 * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
 * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
 * and @ancestor are accessible.
 */
bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
{
	while (cgrp) {
		if (cgrp == ancestor)
			return true;
T
Tejun Heo 已提交
326
		cgrp = cgroup_parent(cgrp);
327 328 329
	}
	return false;
}
330

331
static int cgroup_is_releasable(const struct cgroup *cgrp)
332 333
{
	const int bits =
334 335 336
		(1 << CGRP_RELEASABLE) |
		(1 << CGRP_NOTIFY_ON_RELEASE);
	return (cgrp->flags & bits) == bits;
337 338
}

339
static int notify_on_release(const struct cgroup *cgrp)
340
{
341
	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
342 343
}

T
Tejun Heo 已提交
344 345 346 347 348 349
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
350
 * Should be called under cgroup_[tree_]mutex.
T
Tejun Heo 已提交
351 352 353 354 355 356 357 358
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

359 360 361 362 363 364 365 366 367 368 369 370 371 372
/**
 * for_each_e_css - iterate all effective css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_[tree_]mutex.
 */
#define for_each_e_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
			;						\
		else

373
/**
T
Tejun Heo 已提交
374
 * for_each_subsys - iterate all enabled cgroup subsystems
375
 * @ss: the iteration cursor
376
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
377
 */
378
#define for_each_subsys(ss, ssid)					\
T
Tejun Heo 已提交
379 380
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&		\
	     (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
381

382 383
/* iterate across the hierarchies */
#define for_each_root(root)						\
384
	list_for_each_entry((root), &cgroup_roots, root_list)
385

386 387
/* iterate over child cgrps, lock should be held throughout iteration */
#define cgroup_for_each_live_child(child, cgrp)				\
388
	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
T
Tejun Heo 已提交
389
		if (({ lockdep_assert_held(&cgroup_mutex);		\
390 391 392
		       cgroup_is_dead(child); }))			\
			;						\
		else
393

394 395 396
/* the list of cgroups eligible for automatic release. Protected by
 * release_list_lock */
static LIST_HEAD(release_list);
397
static DEFINE_RAW_SPINLOCK(release_list_lock);
398 399
static void cgroup_release_agent(struct work_struct *work);
static DECLARE_WORK(release_agent_work, cgroup_release_agent);
400
static void check_for_release(struct cgroup *cgrp);
401

402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
/*
 * A cgroup can be associated with multiple css_sets as different tasks may
 * belong to different cgroups on different hierarchies.  In the other
 * direction, a css_set is naturally associated with multiple cgroups.
 * This M:N relationship is represented by the following link structure
 * which exists for each association and allows traversing the associations
 * from both sides.
 */
struct cgrp_cset_link {
	/* the cgroup and css_set this link associates */
	struct cgroup		*cgrp;
	struct css_set		*cset;

	/* list of cgrp_cset_links anchored at cgrp->cset_links */
	struct list_head	cset_link;

	/* list of cgrp_cset_links anchored at css_set->cgrp_links */
	struct list_head	cgrp_link;
420 421
};

422 423
/*
 * The default css_set - used by init and its children prior to any
424 425 426 427 428
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */
T
Tejun Heo 已提交
429
struct css_set init_css_set = {
430 431 432 433 434 435 436
	.refcount		= ATOMIC_INIT(1),
	.cgrp_links		= LIST_HEAD_INIT(init_css_set.cgrp_links),
	.tasks			= LIST_HEAD_INIT(init_css_set.tasks),
	.mg_tasks		= LIST_HEAD_INIT(init_css_set.mg_tasks),
	.mg_preload_node	= LIST_HEAD_INIT(init_css_set.mg_preload_node),
	.mg_node		= LIST_HEAD_INIT(init_css_set.mg_node),
};
437

438
static int css_set_count	= 1;	/* 1 for init_css_set */
439

440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
/**
 * cgroup_update_populated - updated populated count of a cgroup
 * @cgrp: the target cgroup
 * @populated: inc or dec populated count
 *
 * @cgrp is either getting the first task (css_set) or losing the last.
 * Update @cgrp->populated_cnt accordingly.  The count is propagated
 * towards root so that a given cgroup's populated_cnt is zero iff the
 * cgroup and all its descendants are empty.
 *
 * @cgrp's interface file "cgroup.populated" is zero if
 * @cgrp->populated_cnt is zero and 1 otherwise.  When @cgrp->populated_cnt
 * changes from or to zero, userland is notified that the content of the
 * interface file has changed.  This can be used to detect when @cgrp and
 * its descendants become populated or empty.
 */
static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
{
	lockdep_assert_held(&css_set_rwsem);

	do {
		bool trigger;

		if (populated)
			trigger = !cgrp->populated_cnt++;
		else
			trigger = !--cgrp->populated_cnt;

		if (!trigger)
			break;

		if (cgrp->populated_kn)
			kernfs_notify(cgrp->populated_kn);
T
Tejun Heo 已提交
473
		cgrp = cgroup_parent(cgrp);
474 475 476
	} while (cgrp);
}

477 478 479 480 481
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
482
#define CSS_SET_HASH_BITS	7
483
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
484

485
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
486
{
487
	unsigned long key = 0UL;
488 489
	struct cgroup_subsys *ss;
	int i;
490

491
	for_each_subsys(ss, i)
492 493
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
494

495
	return key;
496 497
}

498
static void put_css_set_locked(struct css_set *cset, bool taskexit)
499
{
500
	struct cgrp_cset_link *link, *tmp_link;
T
Tejun Heo 已提交
501 502
	struct cgroup_subsys *ss;
	int ssid;
503

504 505 506
	lockdep_assert_held(&css_set_rwsem);

	if (!atomic_dec_and_test(&cset->refcount))
507
		return;
508

509
	/* This css_set is dead. unlink it and release cgroup refcounts */
T
Tejun Heo 已提交
510 511
	for_each_subsys(ss, ssid)
		list_del(&cset->e_cset_node[ssid]);
512
	hash_del(&cset->hlist);
513 514
	css_set_count--;

515
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
516
		struct cgroup *cgrp = link->cgrp;
517

518 519
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
L
Li Zefan 已提交
520

521
		/* @cgrp can't go away while we're holding css_set_rwsem */
522 523 524 525 526 527 528
		if (list_empty(&cgrp->cset_links)) {
			cgroup_update_populated(cgrp, false);
			if (notify_on_release(cgrp)) {
				if (taskexit)
					set_bit(CGRP_RELEASABLE, &cgrp->flags);
				check_for_release(cgrp);
			}
529
		}
530 531

		kfree(link);
532
	}
533

534
	kfree_rcu(cset, rcu_head);
535 536
}

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
static void put_css_set(struct css_set *cset, bool taskexit)
{
	/*
	 * Ensure that the refcount doesn't hit zero while any readers
	 * can see it. Similar to atomic_dec_and_lock(), but for an
	 * rwlock
	 */
	if (atomic_add_unless(&cset->refcount, -1, 1))
		return;

	down_write(&css_set_rwsem);
	put_css_set_locked(cset, taskexit);
	up_write(&css_set_rwsem);
}

552 553 554
/*
 * refcounted get/put for css_set objects
 */
555
static inline void get_css_set(struct css_set *cset)
556
{
557
	atomic_inc(&cset->refcount);
558 559
}

560
/**
561
 * compare_css_sets - helper function for find_existing_css_set().
562 563
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
564 565 566
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
567
 * Returns true if "cset" matches "old_cset" except for the hierarchy
568 569
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
570 571
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
572 573 574 575 576
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
	struct list_head *l1, *l2;

577 578 579 580 581 582
	/*
	 * On the default hierarchy, there can be csets which are
	 * associated with the same set of cgroups but different csses.
	 * Let's first ensure that csses match.
	 */
	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
583 584 585 586
		return false;

	/*
	 * Compare cgroup pointers in order to distinguish between
587 588 589
	 * different cgroups in hierarchies.  As different cgroups may
	 * share the same effective css, this comparison is always
	 * necessary.
590
	 */
591 592
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
593
	while (1) {
594
		struct cgrp_cset_link *link1, *link2;
595
		struct cgroup *cgrp1, *cgrp2;
596 597 598 599

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
600 601
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
602 603
			break;
		} else {
604
			BUG_ON(l2 == &old_cset->cgrp_links);
605 606
		}
		/* Locate the cgroups associated with these links. */
607 608 609 610
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
611
		/* Hierarchies should be linked in the same order. */
612
		BUG_ON(cgrp1->root != cgrp2->root);
613 614 615 616 617 618 619 620

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
621 622
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
623 624
				return false;
		} else {
625
			if (cgrp1 != cgrp2)
626 627 628 629 630 631
				return false;
		}
	}
	return true;
}

632 633 634 635 636
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
637
 */
638 639 640
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
641
{
642
	struct cgroup_root *root = cgrp->root;
643
	struct cgroup_subsys *ss;
644
	struct css_set *cset;
645
	unsigned long key;
646
	int i;
647

B
Ben Blum 已提交
648 649 650 651 652
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
653
	for_each_subsys(ss, i) {
654
		if (root->subsys_mask & (1UL << i)) {
655 656 657 658 659
			/*
			 * @ss is in this hierarchy, so we want the
			 * effective css from @cgrp.
			 */
			template[i] = cgroup_e_css(cgrp, ss);
660
		} else {
661 662 663 664
			/*
			 * @ss is not in this hierarchy, so we don't want
			 * to change the css.
			 */
665
			template[i] = old_cset->subsys[i];
666 667 668
		}
	}

669
	key = css_set_hash(template);
670 671
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
672 673 674
			continue;

		/* This css_set matches what we need */
675
		return cset;
676
	}
677 678 679 680 681

	/* No existing cgroup group matched */
	return NULL;
}

682
static void free_cgrp_cset_links(struct list_head *links_to_free)
683
{
684
	struct cgrp_cset_link *link, *tmp_link;
685

686 687
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
688 689 690 691
		kfree(link);
	}
}

692 693 694 695 696 697 698
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
699
 */
700
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
701
{
702
	struct cgrp_cset_link *link;
703
	int i;
704 705 706

	INIT_LIST_HEAD(tmp_links);

707
	for (i = 0; i < count; i++) {
708
		link = kzalloc(sizeof(*link), GFP_KERNEL);
709
		if (!link) {
710
			free_cgrp_cset_links(tmp_links);
711 712
			return -ENOMEM;
		}
713
		list_add(&link->cset_link, tmp_links);
714 715 716 717
	}
	return 0;
}

718 719
/**
 * link_css_set - a helper function to link a css_set to a cgroup
720
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
721
 * @cset: the css_set to be linked
722 723
 * @cgrp: the destination cgroup
 */
724 725
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
726
{
727
	struct cgrp_cset_link *link;
728

729
	BUG_ON(list_empty(tmp_links));
T
Tejun Heo 已提交
730 731 732 733

	if (cgroup_on_dfl(cgrp))
		cset->dfl_cgrp = cgrp;

734 735
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
736
	link->cgrp = cgrp;
737 738 739

	if (list_empty(&cgrp->cset_links))
		cgroup_update_populated(cgrp, true);
740
	list_move(&link->cset_link, &cgrp->cset_links);
741

742 743 744 745
	/*
	 * Always add links to the tail of the list so that the list
	 * is sorted by order of hierarchy creation
	 */
746
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
747 748
}

749 750 751 752 753 754 755
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
756
 */
757 758
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
759
{
760
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
761
	struct css_set *cset;
762 763
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
764
	struct cgroup_subsys *ss;
765
	unsigned long key;
T
Tejun Heo 已提交
766
	int ssid;
767

768 769
	lockdep_assert_held(&cgroup_mutex);

770 771
	/* First see if we already have a cgroup group that matches
	 * the desired set */
772
	down_read(&css_set_rwsem);
773 774 775
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
776
	up_read(&css_set_rwsem);
777

778 779
	if (cset)
		return cset;
780

781
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
782
	if (!cset)
783 784
		return NULL;

785
	/* Allocate all the cgrp_cset_link objects that we'll need */
786
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
787
		kfree(cset);
788 789 790
		return NULL;
	}

791
	atomic_set(&cset->refcount, 1);
792
	INIT_LIST_HEAD(&cset->cgrp_links);
793
	INIT_LIST_HEAD(&cset->tasks);
T
Tejun Heo 已提交
794
	INIT_LIST_HEAD(&cset->mg_tasks);
795
	INIT_LIST_HEAD(&cset->mg_preload_node);
796
	INIT_LIST_HEAD(&cset->mg_node);
797
	INIT_HLIST_NODE(&cset->hlist);
798 799 800

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
801
	memcpy(cset->subsys, template, sizeof(cset->subsys));
802

803
	down_write(&css_set_rwsem);
804
	/* Add reference counts and links from the new css_set. */
805
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
806
		struct cgroup *c = link->cgrp;
807

808 809
		if (c->root == cgrp->root)
			c = cgrp;
810
		link_css_set(&tmp_links, cset, c);
811
	}
812

813
	BUG_ON(!list_empty(&tmp_links));
814 815

	css_set_count++;
816

T
Tejun Heo 已提交
817
	/* Add @cset to the hash table */
818 819
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
820

T
Tejun Heo 已提交
821 822 823 824
	for_each_subsys(ss, ssid)
		list_add_tail(&cset->e_cset_node[ssid],
			      &cset->subsys[ssid]->cgroup->e_csets[ssid]);

825
	up_write(&css_set_rwsem);
826

827
	return cset;
828 829
}

830
static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
831
{
832
	struct cgroup *root_cgrp = kf_root->kn->priv;
T
Tejun Heo 已提交
833

834
	return root_cgrp->root;
T
Tejun Heo 已提交
835 836
}

837
static int cgroup_init_root_id(struct cgroup_root *root)
838 839 840 841 842
{
	int id;

	lockdep_assert_held(&cgroup_mutex);

843
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
844 845 846 847 848 849 850
	if (id < 0)
		return id;

	root->hierarchy_id = id;
	return 0;
}

851
static void cgroup_exit_root_id(struct cgroup_root *root)
852 853 854 855 856 857 858 859 860
{
	lockdep_assert_held(&cgroup_mutex);

	if (root->hierarchy_id) {
		idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
		root->hierarchy_id = 0;
	}
}

861
static void cgroup_free_root(struct cgroup_root *root)
862 863 864 865 866 867 868 869 870 871
{
	if (root) {
		/* hierarhcy ID shoulid already have been released */
		WARN_ON_ONCE(root->hierarchy_id);

		idr_destroy(&root->cgroup_idr);
		kfree(root);
	}
}

872
static void cgroup_destroy_root(struct cgroup_root *root)
873
{
874
	struct cgroup *cgrp = &root->cgrp;
875 876
	struct cgrp_cset_link *link, *tmp_link;

T
Tejun Heo 已提交
877
	mutex_lock(&cgroup_mutex);
878

T
Tejun Heo 已提交
879
	BUG_ON(atomic_read(&root->nr_cgrps));
880
	BUG_ON(!list_empty(&cgrp->self.children));
881 882

	/* Rebind all subsystems back to the default hierarchy */
883
	rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
884 885

	/*
886 887
	 * Release all the links from cset_links to this hierarchy's
	 * root cgroup
888
	 */
889
	down_write(&css_set_rwsem);
890 891 892 893 894 895

	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
		kfree(link);
	}
896
	up_write(&css_set_rwsem);
897 898 899 900 901 902 903 904 905 906

	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
		cgroup_root_count--;
	}

	cgroup_exit_root_id(root);

	mutex_unlock(&cgroup_mutex);

T
Tejun Heo 已提交
907
	kernfs_destroy_root(root->kf_root);
908 909 910
	cgroup_free_root(root);
}

911 912
/* look up cgroup associated with given css_set on the specified hierarchy */
static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
913
					    struct cgroup_root *root)
914 915 916
{
	struct cgroup *res = NULL;

917 918 919
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

920
	if (cset == &init_css_set) {
921
		res = &root->cgrp;
922
	} else {
923 924 925
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
926
			struct cgroup *c = link->cgrp;
927

928 929 930 931 932 933
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
934

935 936 937 938
	BUG_ON(!res);
	return res;
}

939
/*
940 941 942 943
 * Return the cgroup for "task" from the given hierarchy. Must be
 * called with cgroup_mutex and css_set_rwsem held.
 */
static struct cgroup *task_cgroup_from_root(struct task_struct *task,
944
					    struct cgroup_root *root)
945 946 947 948 949 950 951 952 953
{
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
	return cset_cgroup_from_root(task_css_set(task), root);
}

954 955 956 957 958 959
/*
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
960
 * cgroup_attach_task() can increment it again.  Because a count of zero
961 962 963 964 965 966 967 968 969 970 971 972 973
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
 * (usually) take cgroup_mutex.  These are the two most performance
 * critical pieces of code here.  The exception occurs on cgroup_exit(),
 * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
 * is taken, and if the cgroup count is zero, a usermode call made
L
Li Zefan 已提交
974 975
 * to the release agent with the name of the cgroup (path relative to
 * the root of cgroup file system) as the argument.
976 977 978 979
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
980
 * least one task in the system (init, pid == 1), therefore, root cgroup
981
 * always has either children cgroups and/or using tasks.  So we don't
982
 * need a special hack to ensure that root cgroup cannot be deleted.
983 984
 *
 * P.S.  One more locking exception.  RCU is used to guard the
985
 * update of a tasks cgroup pointer by cgroup_attach_task()
986 987
 */

988
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask);
T
Tejun Heo 已提交
989
static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
990
static const struct file_operations proc_cgroupstats_operations;
991

T
Tejun Heo 已提交
992 993
static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
			      char *buf)
994
{
T
Tejun Heo 已提交
995 996 997 998 999 1000 1001
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
		snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
			 cft->ss->name, cft->name);
	else
		strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
	return buf;
1002 1003
}

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
 * returns cft->mode if ->mode is not 0
 * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
 * returns S_IRUGO if it has only a read handler
 * returns S_IWUSR if it has only a write hander
 */
static umode_t cgroup_file_mode(const struct cftype *cft)
1014
{
1015
	umode_t mode = 0;
1016

1017 1018 1019 1020 1021 1022
	if (cft->mode)
		return cft->mode;

	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
		mode |= S_IRUGO;

1023
	if (cft->write_u64 || cft->write_s64 || cft->write)
1024 1025 1026
		mode |= S_IWUSR;

	return mode;
1027 1028
}

1029
static void cgroup_get(struct cgroup *cgrp)
1030
{
T
Tejun Heo 已提交
1031
	WARN_ON_ONCE(cgroup_is_dead(cgrp));
1032
	css_get(&cgrp->self);
1033 1034
}

1035
static void cgroup_put(struct cgroup *cgrp)
1036
{
1037
	css_put(&cgrp->self);
1038 1039
}

1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
/**
 * cgroup_refresh_child_subsys_mask - update child_subsys_mask
 * @cgrp: the target cgroup
 *
 * On the default hierarchy, a subsystem may request other subsystems to be
 * enabled together through its ->depends_on mask.  In such cases, more
 * subsystems than specified in "cgroup.subtree_control" may be enabled.
 *
 * This function determines which subsystems need to be enabled given the
 * current @cgrp->subtree_control and records it in
 * @cgrp->child_subsys_mask.  The resulting mask is always a superset of
 * @cgrp->subtree_control and follows the usual hierarchy rules.
 */
1053 1054
static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp)
{
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
	struct cgroup *parent = cgroup_parent(cgrp);
	unsigned int cur_ss_mask = cgrp->subtree_control;
	struct cgroup_subsys *ss;
	int ssid;

	lockdep_assert_held(&cgroup_mutex);

	if (!cgroup_on_dfl(cgrp)) {
		cgrp->child_subsys_mask = cur_ss_mask;
		return;
	}

	while (true) {
		unsigned int new_ss_mask = cur_ss_mask;

		for_each_subsys(ss, ssid)
			if (cur_ss_mask & (1 << ssid))
				new_ss_mask |= ss->depends_on;

		/*
		 * Mask out subsystems which aren't available.  This can
		 * happen only if some depended-upon subsystems were bound
		 * to non-default hierarchies.
		 */
		if (parent)
			new_ss_mask &= parent->child_subsys_mask;
		else
			new_ss_mask &= cgrp->root->subsys_mask;

		if (new_ss_mask == cur_ss_mask)
			break;
		cur_ss_mask = new_ss_mask;
	}

	cgrp->child_subsys_mask = cur_ss_mask;
1090 1091
}

1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
/**
 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper undoes cgroup_kn_lock_live() and should be invoked before
 * the method finishes if locking succeeded.  Note that once this function
 * returns the cgroup returned by cgroup_kn_lock_live() may become
 * inaccessible any time.  If the caller intends to continue to access the
 * cgroup, it should pin it before invoking this function.
 */
static void cgroup_kn_unlock(struct kernfs_node *kn)
1103
{
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;

	mutex_unlock(&cgroup_mutex);

	kernfs_unbreak_active_protection(kn);
	cgroup_put(cgrp);
1115 1116
}

1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
/**
 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper is to be used by a cgroup kernfs method currently servicing
 * @kn.  It breaks the active protection, performs cgroup locking and
 * verifies that the associated cgroup is alive.  Returns the cgroup if
 * alive; otherwise, %NULL.  A successful return should be undone by a
 * matching cgroup_kn_unlock() invocation.
 *
 * Any cgroup kernfs method implementation which requires locking the
 * associated cgroup should use this helper.  It avoids nesting cgroup
 * locking under kernfs active protection and allows all kernfs operations
 * including self-removal.
 */
static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
T
Tejun Heo 已提交
1133
{
1134 1135 1136 1137 1138 1139
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;
T
Tejun Heo 已提交
1140

1141
	/*
1142
	 * We're gonna grab cgroup_mutex which nests outside kernfs
1143 1144 1145
	 * active_ref.  cgroup liveliness check alone provides enough
	 * protection against removal.  Ensure @cgrp stays accessible and
	 * break the active_ref protection.
1146
	 */
1147 1148 1149
	cgroup_get(cgrp);
	kernfs_break_active_protection(kn);

T
Tejun Heo 已提交
1150
	mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1151

1152 1153 1154 1155 1156
	if (!cgroup_is_dead(cgrp))
		return cgrp;

	cgroup_kn_unlock(kn);
	return NULL;
1157
}
T
Tejun Heo 已提交
1158

1159
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
1160
{
T
Tejun Heo 已提交
1161
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
1162

1163
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
1164
	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
T
Tejun Heo 已提交
1165 1166
}

1167
/**
1168
 * cgroup_clear_dir - remove subsys files in a cgroup directory
1169
 * @cgrp: target cgroup
1170 1171
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
1172
static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask)
T
Tejun Heo 已提交
1173
{
1174
	struct cgroup_subsys *ss;
1175
	int i;
T
Tejun Heo 已提交
1176

1177
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
1178
		struct cftype *cfts;
1179

1180
		if (!(subsys_mask & (1 << i)))
1181
			continue;
T
Tejun Heo 已提交
1182 1183
		list_for_each_entry(cfts, &ss->cfts, node)
			cgroup_addrm_files(cgrp, cfts, false);
1184
	}
1185 1186
}

1187
static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask)
1188
{
1189
	struct cgroup_subsys *ss;
1190
	unsigned int tmp_ss_mask;
T
Tejun Heo 已提交
1191
	int ssid, i, ret;
1192

T
Tejun Heo 已提交
1193
	lockdep_assert_held(&cgroup_mutex);
1194

1195 1196 1197
	for_each_subsys(ss, ssid) {
		if (!(ss_mask & (1 << ssid)))
			continue;
B
Ben Blum 已提交
1198

1199 1200
		/* if @ss has non-root csses attached to it, can't move */
		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
T
Tejun Heo 已提交
1201
			return -EBUSY;
1202

1203
		/* can't move between two non-dummy roots either */
1204
		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1205
			return -EBUSY;
1206 1207
	}

1208 1209 1210 1211 1212 1213
	/* skip creating root files on dfl_root for inhibited subsystems */
	tmp_ss_mask = ss_mask;
	if (dst_root == &cgrp_dfl_root)
		tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;

	ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask);
T
Tejun Heo 已提交
1214 1215
	if (ret) {
		if (dst_root != &cgrp_dfl_root)
1216
			return ret;
1217

T
Tejun Heo 已提交
1218 1219 1220 1221 1222 1223 1224
		/*
		 * Rebinding back to the default root is not allowed to
		 * fail.  Using both default and non-default roots should
		 * be rare.  Moving subsystems back and forth even more so.
		 * Just warn about it and continue.
		 */
		if (cgrp_dfl_root_visible) {
1225
			pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
1226
				ret, ss_mask);
1227
			pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
T
Tejun Heo 已提交
1228
		}
1229
	}
1230 1231 1232 1233 1234

	/*
	 * Nothing can fail from this point on.  Remove files for the
	 * removed subsystems and rebind each subsystem.
	 */
1235
	for_each_subsys(ss, ssid)
T
Tejun Heo 已提交
1236
		if (ss_mask & (1 << ssid))
1237
			cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
1238

1239
	for_each_subsys(ss, ssid) {
1240
		struct cgroup_root *src_root;
1241
		struct cgroup_subsys_state *css;
T
Tejun Heo 已提交
1242
		struct css_set *cset;
1243

1244 1245
		if (!(ss_mask & (1 << ssid)))
			continue;
1246

1247
		src_root = ss->root;
1248
		css = cgroup_css(&src_root->cgrp, ss);
1249

1250
		WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
1251

1252 1253
		RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
		rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
1254
		ss->root = dst_root;
1255
		css->cgroup = &dst_root->cgrp;
1256

T
Tejun Heo 已提交
1257 1258 1259 1260 1261 1262
		down_write(&css_set_rwsem);
		hash_for_each(css_set_table, i, cset, hlist)
			list_move_tail(&cset->e_cset_node[ss->id],
				       &dst_root->cgrp.e_csets[ss->id]);
		up_write(&css_set_rwsem);

1263
		src_root->subsys_mask &= ~(1 << ssid);
1264 1265
		src_root->cgrp.subtree_control &= ~(1 << ssid);
		cgroup_refresh_child_subsys_mask(&src_root->cgrp);
1266

1267
		/* default hierarchy doesn't enable controllers by default */
1268
		dst_root->subsys_mask |= 1 << ssid;
1269 1270 1271 1272
		if (dst_root != &cgrp_dfl_root) {
			dst_root->cgrp.subtree_control |= 1 << ssid;
			cgroup_refresh_child_subsys_mask(&dst_root->cgrp);
		}
1273

1274 1275
		if (ss->bind)
			ss->bind(css);
1276 1277
	}

T
Tejun Heo 已提交
1278
	kernfs_activate(dst_root->cgrp.kn);
1279 1280 1281
	return 0;
}

T
Tejun Heo 已提交
1282 1283
static int cgroup_show_options(struct seq_file *seq,
			       struct kernfs_root *kf_root)
1284
{
1285
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1286
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1287
	int ssid;
1288

T
Tejun Heo 已提交
1289
	for_each_subsys(ss, ssid)
1290
		if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
1291
			seq_printf(seq, ",%s", ss->name);
1292 1293
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR)
		seq_puts(seq, ",sane_behavior");
1294
	if (root->flags & CGRP_ROOT_NOPREFIX)
1295
		seq_puts(seq, ",noprefix");
1296
	if (root->flags & CGRP_ROOT_XATTR)
A
Aristeu Rozanski 已提交
1297
		seq_puts(seq, ",xattr");
1298 1299

	spin_lock(&release_agent_path_lock);
1300 1301
	if (strlen(root->release_agent_path))
		seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1302 1303
	spin_unlock(&release_agent_path_lock);

1304
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1305
		seq_puts(seq, ",clone_children");
1306 1307
	if (strlen(root->name))
		seq_printf(seq, ",name=%s", root->name);
1308 1309 1310 1311
	return 0;
}

struct cgroup_sb_opts {
1312 1313
	unsigned int subsys_mask;
	unsigned int flags;
1314
	char *release_agent;
1315
	bool cpuset_clone_children;
1316
	char *name;
1317 1318
	/* User explicitly requested empty subsystem */
	bool none;
1319 1320
};

B
Ben Blum 已提交
1321
static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1322
{
1323 1324
	char *token, *o = data;
	bool all_ss = false, one_ss = false;
1325
	unsigned int mask = -1U;
1326 1327
	struct cgroup_subsys *ss;
	int i;
1328 1329

#ifdef CONFIG_CPUSETS
1330
	mask = ~(1U << cpuset_cgrp_id);
1331
#endif
1332

1333
	memset(opts, 0, sizeof(*opts));
1334 1335 1336 1337

	while ((token = strsep(&o, ",")) != NULL) {
		if (!*token)
			return -EINVAL;
1338
		if (!strcmp(token, "none")) {
1339 1340
			/* Explicitly have no subsystems */
			opts->none = true;
1341 1342 1343 1344 1345 1346 1347 1348 1349
			continue;
		}
		if (!strcmp(token, "all")) {
			/* Mutually exclusive option 'all' + subsystem name */
			if (one_ss)
				return -EINVAL;
			all_ss = true;
			continue;
		}
1350 1351 1352 1353
		if (!strcmp(token, "__DEVEL__sane_behavior")) {
			opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
			continue;
		}
1354
		if (!strcmp(token, "noprefix")) {
1355
			opts->flags |= CGRP_ROOT_NOPREFIX;
1356 1357 1358
			continue;
		}
		if (!strcmp(token, "clone_children")) {
1359
			opts->cpuset_clone_children = true;
1360 1361
			continue;
		}
A
Aristeu Rozanski 已提交
1362
		if (!strcmp(token, "xattr")) {
1363
			opts->flags |= CGRP_ROOT_XATTR;
A
Aristeu Rozanski 已提交
1364 1365
			continue;
		}
1366
		if (!strncmp(token, "release_agent=", 14)) {
1367 1368 1369
			/* Specifying two release agents is forbidden */
			if (opts->release_agent)
				return -EINVAL;
1370
			opts->release_agent =
1371
				kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1372 1373
			if (!opts->release_agent)
				return -ENOMEM;
1374 1375 1376
			continue;
		}
		if (!strncmp(token, "name=", 5)) {
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
			const char *name = token + 5;
			/* Can't specify an empty name */
			if (!strlen(name))
				return -EINVAL;
			/* Must match [\w.-]+ */
			for (i = 0; i < strlen(name); i++) {
				char c = name[i];
				if (isalnum(c))
					continue;
				if ((c == '.') || (c == '-') || (c == '_'))
					continue;
				return -EINVAL;
			}
			/* Specifying two names is forbidden */
			if (opts->name)
				return -EINVAL;
			opts->name = kstrndup(name,
1394
					      MAX_CGROUP_ROOT_NAMELEN - 1,
1395 1396 1397
					      GFP_KERNEL);
			if (!opts->name)
				return -ENOMEM;
1398 1399 1400 1401

			continue;
		}

1402
		for_each_subsys(ss, i) {
1403 1404 1405 1406 1407 1408 1409 1410
			if (strcmp(token, ss->name))
				continue;
			if (ss->disabled)
				continue;

			/* Mutually exclusive option 'all' + subsystem name */
			if (all_ss)
				return -EINVAL;
1411
			opts->subsys_mask |= (1 << i);
1412 1413 1414 1415 1416 1417 1418 1419
			one_ss = true;

			break;
		}
		if (i == CGROUP_SUBSYS_COUNT)
			return -ENOENT;
	}

1420 1421
	/* Consistency checks */

1422
	if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1423
		pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
1424

1425 1426 1427
		if ((opts->flags & (CGRP_ROOT_NOPREFIX | CGRP_ROOT_XATTR)) ||
		    opts->cpuset_clone_children || opts->release_agent ||
		    opts->name) {
1428
			pr_err("sane_behavior: noprefix, xattr, clone_children, release_agent and name are not allowed\n");
1429 1430
			return -EINVAL;
		}
T
Tejun Heo 已提交
1431 1432 1433 1434 1435 1436 1437 1438 1439
	} else {
		/*
		 * If the 'all' option was specified select all the
		 * subsystems, otherwise if 'none', 'name=' and a subsystem
		 * name options were not specified, let's default to 'all'
		 */
		if (all_ss || (!one_ss && !opts->none && !opts->name))
			for_each_subsys(ss, i)
				if (!ss->disabled)
1440
					opts->subsys_mask |= (1 << i);
1441

T
Tejun Heo 已提交
1442 1443 1444 1445 1446
		/*
		 * We either have to specify by name or by subsystems. (So
		 * all empty hierarchies must have a name).
		 */
		if (!opts->subsys_mask && !opts->name)
1447 1448 1449
			return -EINVAL;
	}

1450 1451 1452 1453 1454
	/*
	 * Option noprefix was introduced just for backward compatibility
	 * with the old cpuset, so we allow noprefix only if mounting just
	 * the cpuset subsystem.
	 */
1455
	if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1456 1457
		return -EINVAL;

1458 1459

	/* Can't specify "none" and some subsystems */
1460
	if (opts->subsys_mask && opts->none)
1461 1462
		return -EINVAL;

1463 1464 1465
	return 0;
}

T
Tejun Heo 已提交
1466
static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1467 1468
{
	int ret = 0;
1469
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1470
	struct cgroup_sb_opts opts;
1471
	unsigned int added_mask, removed_mask;
1472

1473
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1474
		pr_err("sane_behavior: remount is not allowed\n");
1475 1476 1477
		return -EINVAL;
	}

1478 1479 1480 1481 1482 1483 1484
	mutex_lock(&cgroup_mutex);

	/* See what subsystems are wanted */
	ret = parse_cgroupfs_options(data, &opts);
	if (ret)
		goto out_unlock;

1485
	if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1486
		pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1487
			task_tgid_nr(current), current->comm);
1488

1489 1490
	added_mask = opts.subsys_mask & ~root->subsys_mask;
	removed_mask = root->subsys_mask & ~opts.subsys_mask;
1491

B
Ben Blum 已提交
1492
	/* Don't allow flags or name to change at remount */
1493
	if (((opts.flags ^ root->flags) & CGRP_ROOT_OPTION_MASK) ||
B
Ben Blum 已提交
1494
	    (opts.name && strcmp(opts.name, root->name))) {
1495
		pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
1496 1497
		       opts.flags & CGRP_ROOT_OPTION_MASK, opts.name ?: "",
		       root->flags & CGRP_ROOT_OPTION_MASK, root->name);
1498 1499 1500 1501
		ret = -EINVAL;
		goto out_unlock;
	}

1502
	/* remounting is not allowed for populated hierarchies */
1503
	if (!list_empty(&root->cgrp.self.children)) {
1504
		ret = -EBUSY;
1505
		goto out_unlock;
B
Ben Blum 已提交
1506
	}
1507

1508
	ret = rebind_subsystems(root, added_mask);
1509
	if (ret)
1510
		goto out_unlock;
1511

1512
	rebind_subsystems(&cgrp_dfl_root, removed_mask);
1513

1514 1515
	if (opts.release_agent) {
		spin_lock(&release_agent_path_lock);
1516
		strcpy(root->release_agent_path, opts.release_agent);
1517 1518
		spin_unlock(&release_agent_path_lock);
	}
1519
 out_unlock:
1520
	kfree(opts.release_agent);
1521
	kfree(opts.name);
1522 1523 1524 1525
	mutex_unlock(&cgroup_mutex);
	return ret;
}

1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
/*
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
 * words after the first mount.
 */
static bool use_task_css_set_links __read_mostly;

static void cgroup_enable_task_cg_lists(void)
{
	struct task_struct *p, *g;

1538
	down_write(&css_set_rwsem);
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560

	if (use_task_css_set_links)
		goto out_unlock;

	use_task_css_set_links = true;

	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		WARN_ON_ONCE(!list_empty(&p->cg_list) ||
			     task_css_set(p) != &init_css_set);

		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
1561 1562
		 * Do it while holding siglock so that we don't end up
		 * racing against cgroup_exit().
1563
		 */
1564
		spin_lock_irq(&p->sighand->siglock);
1565 1566 1567 1568 1569 1570
		if (!(p->flags & PF_EXITING)) {
			struct css_set *cset = task_css_set(p);

			list_add(&p->cg_list, &cset->tasks);
			get_css_set(cset);
		}
1571
		spin_unlock_irq(&p->sighand->siglock);
1572 1573 1574
	} while_each_thread(g, p);
	read_unlock(&tasklist_lock);
out_unlock:
1575
	up_write(&css_set_rwsem);
1576
}
1577

1578 1579
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
T
Tejun Heo 已提交
1580 1581 1582
	struct cgroup_subsys *ss;
	int ssid;

1583 1584
	INIT_LIST_HEAD(&cgrp->self.sibling);
	INIT_LIST_HEAD(&cgrp->self.children);
1585
	INIT_LIST_HEAD(&cgrp->cset_links);
1586
	INIT_LIST_HEAD(&cgrp->release_list);
1587 1588
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
1589
	cgrp->self.cgroup = cgrp;
1590
	cgrp->self.flags |= CSS_ONLINE;
T
Tejun Heo 已提交
1591 1592 1593

	for_each_subsys(ss, ssid)
		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1594 1595

	init_waitqueue_head(&cgrp->offline_waitq);
1596
}
1597

1598
static void init_cgroup_root(struct cgroup_root *root,
1599
			     struct cgroup_sb_opts *opts)
1600
{
1601
	struct cgroup *cgrp = &root->cgrp;
1602

1603
	INIT_LIST_HEAD(&root->root_list);
1604
	atomic_set(&root->nr_cgrps, 1);
1605
	cgrp->root = root;
1606
	init_cgroup_housekeeping(cgrp);
1607
	idr_init(&root->cgroup_idr);
1608 1609 1610 1611 1612 1613

	root->flags = opts->flags;
	if (opts->release_agent)
		strcpy(root->release_agent_path, opts->release_agent);
	if (opts->name)
		strcpy(root->name, opts->name);
1614
	if (opts->cpuset_clone_children)
1615
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1616 1617
}

1618
static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
1619
{
1620
	LIST_HEAD(tmp_links);
1621
	struct cgroup *root_cgrp = &root->cgrp;
1622 1623
	struct css_set *cset;
	int i, ret;
1624

1625
	lockdep_assert_held(&cgroup_mutex);
1626

1627
	ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT);
1628
	if (ret < 0)
T
Tejun Heo 已提交
1629
		goto out;
1630
	root_cgrp->id = ret;
1631

1632 1633 1634 1635
	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release);
	if (ret)
		goto out;

1636
	/*
1637
	 * We're accessing css_set_count without locking css_set_rwsem here,
1638 1639 1640 1641 1642 1643
	 * but that's OK - it can only be increased by someone holding
	 * cgroup_lock, and that's us. The worst that can happen is that we
	 * have some link structures left over
	 */
	ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
	if (ret)
1644
		goto cancel_ref;
1645

1646
	ret = cgroup_init_root_id(root);
1647
	if (ret)
1648
		goto cancel_ref;
1649

T
Tejun Heo 已提交
1650 1651 1652 1653 1654 1655 1656 1657
	root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
					   KERNFS_ROOT_CREATE_DEACTIVATED,
					   root_cgrp);
	if (IS_ERR(root->kf_root)) {
		ret = PTR_ERR(root->kf_root);
		goto exit_root_id;
	}
	root_cgrp->kn = root->kf_root->kn;
1658

1659 1660
	ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
	if (ret)
T
Tejun Heo 已提交
1661
		goto destroy_root;
1662

1663
	ret = rebind_subsystems(root, ss_mask);
1664
	if (ret)
T
Tejun Heo 已提交
1665
		goto destroy_root;
1666

1667 1668 1669 1670 1671 1672 1673
	/*
	 * There must be no failure case after here, since rebinding takes
	 * care of subsystems' refcounts, which are explicitly dropped in
	 * the failure exit path.
	 */
	list_add(&root->root_list, &cgroup_roots);
	cgroup_root_count++;
A
Al Viro 已提交
1674

1675
	/*
1676
	 * Link the root cgroup in this hierarchy into all the css_set
1677 1678
	 * objects.
	 */
1679
	down_write(&css_set_rwsem);
1680 1681
	hash_for_each(css_set_table, i, cset, hlist)
		link_css_set(&tmp_links, cset, root_cgrp);
1682
	up_write(&css_set_rwsem);
1683

1684
	BUG_ON(!list_empty(&root_cgrp->self.children));
1685
	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1686

T
Tejun Heo 已提交
1687
	kernfs_activate(root_cgrp->kn);
1688
	ret = 0;
T
Tejun Heo 已提交
1689
	goto out;
1690

T
Tejun Heo 已提交
1691 1692 1693 1694
destroy_root:
	kernfs_destroy_root(root->kf_root);
	root->kf_root = NULL;
exit_root_id:
1695
	cgroup_exit_root_id(root);
1696 1697
cancel_ref:
	percpu_ref_cancel_init(&root_cgrp->self.refcnt);
T
Tejun Heo 已提交
1698
out:
1699 1700
	free_cgrp_cset_links(&tmp_links);
	return ret;
1701 1702
}

A
Al Viro 已提交
1703
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1704
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
1705
			 void *data)
1706
{
1707
	struct cgroup_root *root;
1708
	struct cgroup_sb_opts opts;
T
Tejun Heo 已提交
1709
	struct dentry *dentry;
1710
	int ret;
L
Li Zefan 已提交
1711
	bool new_sb;
1712

1713 1714 1715 1716 1717 1718
	/*
	 * The first time anyone tries to mount a cgroup, enable the list
	 * linking each css_set to its tasks and fix up all existing tasks.
	 */
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();
1719

B
Ben Blum 已提交
1720
	mutex_lock(&cgroup_mutex);
1721 1722

	/* First find the desired set of subsystems */
1723
	ret = parse_cgroupfs_options(data, &opts);
1724
	if (ret)
1725
		goto out_unlock;
1726

T
Tejun Heo 已提交
1727
	/* look for a matching existing root */
T
Tejun Heo 已提交
1728 1729 1730 1731 1732 1733
	if (!opts.subsys_mask && !opts.none && !opts.name) {
		cgrp_dfl_root_visible = true;
		root = &cgrp_dfl_root;
		cgroup_get(&root->cgrp);
		ret = 0;
		goto out_unlock;
1734 1735
	}

1736
	for_each_root(root) {
T
Tejun Heo 已提交
1737
		bool name_match = false;
1738

1739
		if (root == &cgrp_dfl_root)
1740
			continue;
1741

B
Ben Blum 已提交
1742
		/*
T
Tejun Heo 已提交
1743 1744 1745
		 * If we asked for a name then it must match.  Also, if
		 * name matches but sybsys_mask doesn't, we should fail.
		 * Remember whether name matched.
B
Ben Blum 已提交
1746
		 */
T
Tejun Heo 已提交
1747 1748 1749 1750 1751
		if (opts.name) {
			if (strcmp(opts.name, root->name))
				continue;
			name_match = true;
		}
1752

1753
		/*
T
Tejun Heo 已提交
1754 1755
		 * If we asked for subsystems (or explicitly for no
		 * subsystems) then they must match.
1756
		 */
T
Tejun Heo 已提交
1757
		if ((opts.subsys_mask || opts.none) &&
1758
		    (opts.subsys_mask != root->subsys_mask)) {
T
Tejun Heo 已提交
1759 1760 1761 1762 1763
			if (!name_match)
				continue;
			ret = -EBUSY;
			goto out_unlock;
		}
1764

1765
		if ((root->flags ^ opts.flags) & CGRP_ROOT_OPTION_MASK) {
1766
			if ((root->flags | opts.flags) & CGRP_ROOT_SANE_BEHAVIOR) {
1767
				pr_err("sane_behavior: new mount options should match the existing superblock\n");
1768
				ret = -EINVAL;
1769
				goto out_unlock;
1770
			} else {
1771
				pr_warn("new mount options do not match the existing superblock, will be ignored\n");
1772
			}
1773
		}
1774

T
Tejun Heo 已提交
1775
		/*
1776 1777 1778 1779 1780 1781
		 * A root's lifetime is governed by its root cgroup.
		 * tryget_live failure indicate that the root is being
		 * destroyed.  Wait for destruction to complete so that the
		 * subsystems are free.  We can use wait_queue for the wait
		 * but this path is super cold.  Let's just sleep for a bit
		 * and retry.
T
Tejun Heo 已提交
1782
		 */
1783
		if (!percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
T
Tejun Heo 已提交
1784 1785
			mutex_unlock(&cgroup_mutex);
			msleep(10);
1786 1787
			ret = restart_syscall();
			goto out_free;
T
Tejun Heo 已提交
1788
		}
1789

T
Tejun Heo 已提交
1790
		ret = 0;
T
Tejun Heo 已提交
1791
		goto out_unlock;
1792 1793
	}

1794
	/*
1795 1796 1797
	 * No such thing, create a new one.  name= matching without subsys
	 * specification is allowed for already existing hierarchies but we
	 * can't create new one without subsys specification.
1798
	 */
1799 1800 1801
	if (!opts.subsys_mask && !opts.none) {
		ret = -EINVAL;
		goto out_unlock;
1802 1803
	}

1804 1805 1806
	root = kzalloc(sizeof(*root), GFP_KERNEL);
	if (!root) {
		ret = -ENOMEM;
T
Tejun Heo 已提交
1807
		goto out_unlock;
1808
	}
1809

1810 1811
	init_cgroup_root(root, &opts);

T
Tejun Heo 已提交
1812
	ret = cgroup_setup_root(root, opts.subsys_mask);
T
Tejun Heo 已提交
1813 1814
	if (ret)
		cgroup_free_root(root);
1815

1816
out_unlock:
1817
	mutex_unlock(&cgroup_mutex);
1818
out_free:
1819 1820
	kfree(opts.release_agent);
	kfree(opts.name);
A
Aristeu Rozanski 已提交
1821

T
Tejun Heo 已提交
1822
	if (ret)
1823
		return ERR_PTR(ret);
T
Tejun Heo 已提交
1824

1825 1826
	dentry = kernfs_mount(fs_type, flags, root->kf_root,
				CGROUP_SUPER_MAGIC, &new_sb);
L
Li Zefan 已提交
1827
	if (IS_ERR(dentry) || !new_sb)
1828
		cgroup_put(&root->cgrp);
T
Tejun Heo 已提交
1829 1830 1831 1832 1833 1834
	return dentry;
}

static void cgroup_kill_sb(struct super_block *sb)
{
	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
1835
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
T
Tejun Heo 已提交
1836

1837 1838 1839 1840
	/*
	 * If @root doesn't have any mounts or children, start killing it.
	 * This prevents new mounts by disabling percpu_ref_tryget_live().
	 * cgroup_mount() may wait for @root's release.
1841 1842
	 *
	 * And don't kill the default root.
1843
	 */
1844 1845
	if (css_has_online_children(&root->cgrp.self) ||
	    root == &cgrp_dfl_root)
1846 1847 1848 1849
		cgroup_put(&root->cgrp);
	else
		percpu_ref_kill(&root->cgrp.self.refcnt);

T
Tejun Heo 已提交
1850
	kernfs_kill_sb(sb);
1851 1852 1853 1854
}

static struct file_system_type cgroup_fs_type = {
	.name = "cgroup",
A
Al Viro 已提交
1855
	.mount = cgroup_mount,
1856 1857 1858
	.kill_sb = cgroup_kill_sb,
};

1859 1860
static struct kobject *cgroup_kobj;

1861
/**
1862
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1863 1864 1865 1866
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
1867 1868 1869 1870 1871
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
T
Tejun Heo 已提交
1872
 * Return value is the same as kernfs_path().
1873
 */
T
Tejun Heo 已提交
1874
char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1875
{
1876
	struct cgroup_root *root;
1877
	struct cgroup *cgrp;
T
Tejun Heo 已提交
1878 1879
	int hierarchy_id = 1;
	char *path = NULL;
1880 1881

	mutex_lock(&cgroup_mutex);
1882
	down_read(&css_set_rwsem);
1883

1884 1885
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

1886 1887
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
T
Tejun Heo 已提交
1888
		path = cgroup_path(cgrp, buf, buflen);
1889 1890
	} else {
		/* if no hierarchy exists, everyone is in "/" */
T
Tejun Heo 已提交
1891 1892
		if (strlcpy(buf, "/", buflen) < buflen)
			path = buf;
1893 1894
	}

1895
	up_read(&css_set_rwsem);
1896
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
1897
	return path;
1898
}
1899
EXPORT_SYMBOL_GPL(task_cgroup_path);
1900

1901
/* used to track tasks and other necessary states during migration */
1902
struct cgroup_taskset {
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	/* the src and dst cset list running through cset->mg_node */
	struct list_head	src_csets;
	struct list_head	dst_csets;

	/*
	 * Fields for cgroup_taskset_*() iteration.
	 *
	 * Before migration is committed, the target migration tasks are on
	 * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
	 * the csets on ->dst_csets.  ->csets point to either ->src_csets
	 * or ->dst_csets depending on whether migration is committed.
	 *
	 * ->cur_csets and ->cur_task point to the current task position
	 * during iteration.
	 */
	struct list_head	*csets;
	struct css_set		*cur_cset;
	struct task_struct	*cur_task;
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
};

/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
 *
 * @tset iteration is initialized and the first task is returned.
 */
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
{
1931 1932 1933 1934
	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
	tset->cur_task = NULL;

	return cgroup_taskset_next(tset);
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
}

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
{
1946 1947
	struct css_set *cset = tset->cur_cset;
	struct task_struct *task = tset->cur_task;
1948

1949 1950 1951 1952 1953 1954
	while (&cset->mg_node != tset->csets) {
		if (!task)
			task = list_first_entry(&cset->mg_tasks,
						struct task_struct, cg_list);
		else
			task = list_next_entry(task, cg_list);
1955

1956 1957 1958 1959 1960
		if (&task->cg_list != &cset->mg_tasks) {
			tset->cur_cset = cset;
			tset->cur_task = task;
			return task;
		}
1961

1962 1963 1964
		cset = list_next_entry(cset, mg_node);
		task = NULL;
	}
1965

1966
	return NULL;
1967 1968
}

1969
/**
B
Ben Blum 已提交
1970
 * cgroup_task_migrate - move a task from one cgroup to another.
1971
 * @old_cgrp: the cgroup @tsk is being migrated from
1972 1973
 * @tsk: the task being migrated
 * @new_cset: the new css_set @tsk is being attached to
B
Ben Blum 已提交
1974
 *
1975
 * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
B
Ben Blum 已提交
1976
 */
1977 1978 1979
static void cgroup_task_migrate(struct cgroup *old_cgrp,
				struct task_struct *tsk,
				struct css_set *new_cset)
B
Ben Blum 已提交
1980
{
1981
	struct css_set *old_cset;
B
Ben Blum 已提交
1982

1983 1984 1985
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

B
Ben Blum 已提交
1986
	/*
1987 1988 1989
	 * We are synchronized through threadgroup_lock() against PF_EXITING
	 * setting such that we can't race against cgroup_exit() changing the
	 * css_set to init_css_set and dropping the old one.
B
Ben Blum 已提交
1990
	 */
1991
	WARN_ON_ONCE(tsk->flags & PF_EXITING);
1992
	old_cset = task_css_set(tsk);
B
Ben Blum 已提交
1993

1994
	get_css_set(new_cset);
1995
	rcu_assign_pointer(tsk->cgroups, new_cset);
B
Ben Blum 已提交
1996

1997 1998 1999 2000 2001 2002 2003
	/*
	 * Use move_tail so that cgroup_taskset_first() still returns the
	 * leader after migration.  This works because cgroup_migrate()
	 * ensures that the dst_cset of the leader is the first on the
	 * tset's dst_csets list.
	 */
	list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
B
Ben Blum 已提交
2004 2005

	/*
2006 2007 2008
	 * We just gained a reference on old_cset by taking it from the
	 * task. As trading it for new_cset is protected by cgroup_mutex,
	 * we're safe to drop it here; it will be freed under RCU.
B
Ben Blum 已提交
2009
	 */
2010
	set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
2011
	put_css_set_locked(old_cset, false);
B
Ben Blum 已提交
2012 2013
}

L
Li Zefan 已提交
2014
/**
2015 2016
 * cgroup_migrate_finish - cleanup after attach
 * @preloaded_csets: list of preloaded css_sets
B
Ben Blum 已提交
2017
 *
2018 2019
 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
 * those functions for details.
B
Ben Blum 已提交
2020
 */
2021
static void cgroup_migrate_finish(struct list_head *preloaded_csets)
B
Ben Blum 已提交
2022
{
2023
	struct css_set *cset, *tmp_cset;
B
Ben Blum 已提交
2024

2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
	lockdep_assert_held(&cgroup_mutex);

	down_write(&css_set_rwsem);
	list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
		cset->mg_src_cgrp = NULL;
		cset->mg_dst_cset = NULL;
		list_del_init(&cset->mg_preload_node);
		put_css_set_locked(cset, false);
	}
	up_write(&css_set_rwsem);
}

/**
 * cgroup_migrate_add_src - add a migration source css_set
 * @src_cset: the source css_set to add
 * @dst_cgrp: the destination cgroup
 * @preloaded_csets: list of preloaded css_sets
 *
 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
 * @src_cset and add it to @preloaded_csets, which should later be cleaned
 * up by cgroup_migrate_finish().
 *
 * This function may be called without holding threadgroup_lock even if the
 * target is a process.  Threads may be created and destroyed but as long
 * as cgroup_mutex is not dropped, no new css_set can be put into play and
 * the preloaded css_sets are guaranteed to cover all migrations.
 */
static void cgroup_migrate_add_src(struct css_set *src_cset,
				   struct cgroup *dst_cgrp,
				   struct list_head *preloaded_csets)
{
	struct cgroup *src_cgrp;

	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);

	if (!list_empty(&src_cset->mg_preload_node))
		return;

	WARN_ON(src_cset->mg_src_cgrp);
	WARN_ON(!list_empty(&src_cset->mg_tasks));
	WARN_ON(!list_empty(&src_cset->mg_node));

	src_cset->mg_src_cgrp = src_cgrp;
	get_css_set(src_cset);
	list_add(&src_cset->mg_preload_node, preloaded_csets);
}

/**
 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2077
 * @dst_cgrp: the destination cgroup (may be %NULL)
2078 2079 2080 2081
 * @preloaded_csets: list of preloaded source css_sets
 *
 * Tasks are about to be moved to @dst_cgrp and all the source css_sets
 * have been preloaded to @preloaded_csets.  This function looks up and
2082 2083 2084
 * pins all destination css_sets, links each to its source, and append them
 * to @preloaded_csets.  If @dst_cgrp is %NULL, the destination of each
 * source css_set is assumed to be its cgroup on the default hierarchy.
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094
 *
 * This function must be called after cgroup_migrate_add_src() has been
 * called on each migration source css_set.  After migration is performed
 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
 * @preloaded_csets.
 */
static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
				      struct list_head *preloaded_csets)
{
	LIST_HEAD(csets);
2095
	struct css_set *src_cset, *tmp_cset;
2096 2097 2098

	lockdep_assert_held(&cgroup_mutex);

2099 2100 2101 2102
	/*
	 * Except for the root, child_subsys_mask must be zero for a cgroup
	 * with tasks so that child cgroups don't compete against tasks.
	 */
T
Tejun Heo 已提交
2103
	if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
2104 2105 2106
	    dst_cgrp->child_subsys_mask)
		return -EBUSY;

2107
	/* look up the dst cset for each src cset and link it to src */
2108
	list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2109 2110
		struct css_set *dst_cset;

2111 2112
		dst_cset = find_css_set(src_cset,
					dst_cgrp ?: src_cset->dfl_cgrp);
2113 2114 2115 2116
		if (!dst_cset)
			goto err;

		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130

		/*
		 * If src cset equals dst, it's noop.  Drop the src.
		 * cgroup_migrate() will skip the cset too.  Note that we
		 * can't handle src == dst as some nodes are used by both.
		 */
		if (src_cset == dst_cset) {
			src_cset->mg_src_cgrp = NULL;
			list_del_init(&src_cset->mg_preload_node);
			put_css_set(src_cset, false);
			put_css_set(dst_cset, false);
			continue;
		}

2131 2132 2133 2134 2135 2136 2137 2138
		src_cset->mg_dst_cset = dst_cset;

		if (list_empty(&dst_cset->mg_preload_node))
			list_add(&dst_cset->mg_preload_node, &csets);
		else
			put_css_set(dst_cset, false);
	}

2139
	list_splice_tail(&csets, preloaded_csets);
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
	return 0;
err:
	cgroup_migrate_finish(&csets);
	return -ENOMEM;
}

/**
 * cgroup_migrate - migrate a process or task to a cgroup
 * @cgrp: the destination cgroup
 * @leader: the leader of the process or the task to migrate
 * @threadgroup: whether @leader points to the whole process or a single task
 *
 * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
 * process, the caller must be holding threadgroup_lock of @leader.  The
 * caller is also responsible for invoking cgroup_migrate_add_src() and
 * cgroup_migrate_prepare_dst() on the targets before invoking this
 * function and following up with cgroup_migrate_finish().
 *
 * As long as a controller's ->can_attach() doesn't fail, this function is
 * guaranteed to succeed.  This means that, excluding ->can_attach()
 * failure, when migrating multiple targets, the success or failure can be
 * decided for all targets by invoking group_migrate_prepare_dst() before
 * actually starting migrating.
 */
static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
			  bool threadgroup)
B
Ben Blum 已提交
2166
{
2167 2168 2169 2170 2171
	struct cgroup_taskset tset = {
		.src_csets	= LIST_HEAD_INIT(tset.src_csets),
		.dst_csets	= LIST_HEAD_INIT(tset.dst_csets),
		.csets		= &tset.src_csets,
	};
T
Tejun Heo 已提交
2172
	struct cgroup_subsys_state *css, *failed_css = NULL;
2173 2174 2175
	struct css_set *cset, *tmp_cset;
	struct task_struct *task, *tmp_task;
	int i, ret;
B
Ben Blum 已提交
2176

2177 2178 2179 2180 2181
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
2182
	down_write(&css_set_rwsem);
2183
	rcu_read_lock();
2184
	task = leader;
B
Ben Blum 已提交
2185
	do {
2186 2187
		/* @task either already exited or can't exit until the end */
		if (task->flags & PF_EXITING)
2188
			goto next;
2189

2190 2191
		/* leave @task alone if post_fork() hasn't linked it yet */
		if (list_empty(&task->cg_list))
2192
			goto next;
2193

2194
		cset = task_css_set(task);
2195
		if (!cset->mg_src_cgrp)
2196
			goto next;
2197

2198
		/*
2199 2200
		 * cgroup_taskset_first() must always return the leader.
		 * Take care to avoid disturbing the ordering.
2201
		 */
2202 2203 2204 2205 2206 2207
		list_move_tail(&task->cg_list, &cset->mg_tasks);
		if (list_empty(&cset->mg_node))
			list_add_tail(&cset->mg_node, &tset.src_csets);
		if (list_empty(&cset->mg_dst_cset->mg_node))
			list_move_tail(&cset->mg_dst_cset->mg_node,
				       &tset.dst_csets);
2208
	next:
2209 2210
		if (!threadgroup)
			break;
2211
	} while_each_thread(leader, task);
2212
	rcu_read_unlock();
2213
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2214

2215
	/* methods shouldn't be called if no task is actually migrating */
2216 2217
	if (list_empty(&tset.src_csets))
		return 0;
2218

2219
	/* check that we can legitimately attach to the cgroup */
2220
	for_each_e_css(css, i, cgrp) {
T
Tejun Heo 已提交
2221
		if (css->ss->can_attach) {
2222 2223
			ret = css->ss->can_attach(css, &tset);
			if (ret) {
T
Tejun Heo 已提交
2224
				failed_css = css;
B
Ben Blum 已提交
2225 2226 2227 2228 2229 2230
				goto out_cancel_attach;
			}
		}
	}

	/*
2231 2232 2233
	 * Now that we're guaranteed success, proceed to move all tasks to
	 * the new cgroup.  There are no failure cases after here, so this
	 * is the commit point.
B
Ben Blum 已提交
2234
	 */
2235
	down_write(&css_set_rwsem);
2236 2237 2238 2239
	list_for_each_entry(cset, &tset.src_csets, mg_node) {
		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
			cgroup_task_migrate(cset->mg_src_cgrp, task,
					    cset->mg_dst_cset);
B
Ben Blum 已提交
2240
	}
2241
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2242 2243

	/*
2244 2245 2246
	 * Migration is committed, all target tasks are now on dst_csets.
	 * Nothing is sensitive to fork() after this point.  Notify
	 * controllers that migration is complete.
B
Ben Blum 已提交
2247
	 */
2248
	tset.csets = &tset.dst_csets;
B
Ben Blum 已提交
2249

2250
	for_each_e_css(css, i, cgrp)
T
Tejun Heo 已提交
2251 2252
		if (css->ss->attach)
			css->ss->attach(css, &tset);
B
Ben Blum 已提交
2253

2254
	ret = 0;
2255 2256
	goto out_release_tset;

B
Ben Blum 已提交
2257
out_cancel_attach:
2258
	for_each_e_css(css, i, cgrp) {
2259 2260 2261 2262
		if (css == failed_css)
			break;
		if (css->ss->cancel_attach)
			css->ss->cancel_attach(css, &tset);
B
Ben Blum 已提交
2263
	}
2264 2265 2266 2267
out_release_tset:
	down_write(&css_set_rwsem);
	list_splice_init(&tset.dst_csets, &tset.src_csets);
	list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
2268
		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2269 2270 2271
		list_del_init(&cset->mg_node);
	}
	up_write(&css_set_rwsem);
2272
	return ret;
B
Ben Blum 已提交
2273 2274
}

2275 2276 2277 2278 2279 2280
/**
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
 * @dst_cgrp: the cgroup to attach to
 * @leader: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
 *
2281
 * Call holding cgroup_mutex and threadgroup_lock of @leader.
2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309
 */
static int cgroup_attach_task(struct cgroup *dst_cgrp,
			      struct task_struct *leader, bool threadgroup)
{
	LIST_HEAD(preloaded_csets);
	struct task_struct *task;
	int ret;

	/* look up all src csets */
	down_read(&css_set_rwsem);
	rcu_read_lock();
	task = leader;
	do {
		cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
				       &preloaded_csets);
		if (!threadgroup)
			break;
	} while_each_thread(leader, task);
	rcu_read_unlock();
	up_read(&css_set_rwsem);

	/* prepare dst csets and commit */
	ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
	if (!ret)
		ret = cgroup_migrate(dst_cgrp, leader, threadgroup);

	cgroup_migrate_finish(&preloaded_csets);
	return ret;
B
Ben Blum 已提交
2310 2311 2312 2313
}

/*
 * Find the task_struct of the task to attach by vpid and pass it along to the
2314
 * function to attach either it or all tasks in its threadgroup. Will lock
2315
 * cgroup_mutex and threadgroup.
2316
 */
2317 2318
static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
				    size_t nbytes, loff_t off, bool threadgroup)
2319 2320
{
	struct task_struct *tsk;
2321
	const struct cred *cred = current_cred(), *tcred;
2322
	struct cgroup *cgrp;
2323
	pid_t pid;
2324 2325
	int ret;

2326 2327 2328
	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
		return -EINVAL;

2329 2330
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
B
Ben Blum 已提交
2331 2332
		return -ENODEV;

2333 2334
retry_find_task:
	rcu_read_lock();
2335
	if (pid) {
2336
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2337 2338
		if (!tsk) {
			rcu_read_unlock();
S
SeongJae Park 已提交
2339
			ret = -ESRCH;
2340
			goto out_unlock_cgroup;
2341
		}
B
Ben Blum 已提交
2342 2343 2344 2345
		/*
		 * even if we're attaching all tasks in the thread group, we
		 * only need to check permissions on one of them.
		 */
2346
		tcred = __task_cred(tsk);
2347 2348 2349
		if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
		    !uid_eq(cred->euid, tcred->uid) &&
		    !uid_eq(cred->euid, tcred->suid)) {
2350
			rcu_read_unlock();
2351 2352
			ret = -EACCES;
			goto out_unlock_cgroup;
2353
		}
2354 2355
	} else
		tsk = current;
2356 2357

	if (threadgroup)
2358
		tsk = tsk->group_leader;
2359 2360

	/*
2361
	 * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2362 2363 2364
	 * trapped in a cpuset, or RT worker may be born in a cgroup
	 * with no rt_runtime allocated.  Just say no.
	 */
2365
	if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2366 2367 2368 2369 2370
		ret = -EINVAL;
		rcu_read_unlock();
		goto out_unlock_cgroup;
	}

2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
	get_task_struct(tsk);
	rcu_read_unlock();

	threadgroup_lock(tsk);
	if (threadgroup) {
		if (!thread_group_leader(tsk)) {
			/*
			 * a race with de_thread from another thread's exec()
			 * may strip us of our leadership, if this happens,
			 * there is no choice but to throw this task away and
			 * try again; this is
			 * "double-double-toil-and-trouble-check locking".
			 */
			threadgroup_unlock(tsk);
			put_task_struct(tsk);
			goto retry_find_task;
		}
2388 2389 2390 2391
	}

	ret = cgroup_attach_task(cgrp, tsk, threadgroup);

2392 2393
	threadgroup_unlock(tsk);

2394
	put_task_struct(tsk);
2395
out_unlock_cgroup:
2396
	cgroup_kn_unlock(of->kn);
2397
	return ret ?: nbytes;
2398 2399
}

2400 2401 2402 2403 2404 2405 2406
/**
 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
 * @from: attach to all cgroups of a given task
 * @tsk: the task to be attached
 */
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
{
2407
	struct cgroup_root *root;
2408 2409
	int retval = 0;

T
Tejun Heo 已提交
2410
	mutex_lock(&cgroup_mutex);
2411
	for_each_root(root) {
2412 2413
		struct cgroup *from_cgrp;

2414
		if (root == &cgrp_dfl_root)
2415 2416
			continue;

2417 2418 2419
		down_read(&css_set_rwsem);
		from_cgrp = task_cgroup_from_root(from, root);
		up_read(&css_set_rwsem);
2420

L
Li Zefan 已提交
2421
		retval = cgroup_attach_task(from_cgrp, tsk, false);
2422 2423 2424
		if (retval)
			break;
	}
T
Tejun Heo 已提交
2425
	mutex_unlock(&cgroup_mutex);
2426 2427 2428 2429 2430

	return retval;
}
EXPORT_SYMBOL_GPL(cgroup_attach_task_all);

2431 2432
static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
B
Ben Blum 已提交
2433
{
2434
	return __cgroup_procs_write(of, buf, nbytes, off, false);
B
Ben Blum 已提交
2435 2436
}

2437 2438
static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
2439
{
2440
	return __cgroup_procs_write(of, buf, nbytes, off, true);
2441 2442
}

2443 2444
static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
					  char *buf, size_t nbytes, loff_t off)
2445
{
2446
	struct cgroup *cgrp;
2447

2448
	BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
2449

2450 2451
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
2452
		return -ENODEV;
2453
	spin_lock(&release_agent_path_lock);
2454 2455
	strlcpy(cgrp->root->release_agent_path, strstrip(buf),
		sizeof(cgrp->root->release_agent_path));
2456
	spin_unlock(&release_agent_path_lock);
2457
	cgroup_kn_unlock(of->kn);
2458
	return nbytes;
2459 2460
}

2461
static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2462
{
2463
	struct cgroup *cgrp = seq_css(seq)->cgroup;
2464

2465
	spin_lock(&release_agent_path_lock);
2466
	seq_puts(seq, cgrp->root->release_agent_path);
2467
	spin_unlock(&release_agent_path_lock);
2468 2469 2470 2471
	seq_putc(seq, '\n');
	return 0;
}

2472
static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2473
{
2474 2475 2476
	struct cgroup *cgrp = seq_css(seq)->cgroup;

	seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
2477 2478 2479
	return 0;
}

2480
static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask)
2481
{
2482 2483 2484
	struct cgroup_subsys *ss;
	bool printed = false;
	int ssid;
2485

2486 2487 2488 2489 2490 2491 2492
	for_each_subsys(ss, ssid) {
		if (ss_mask & (1 << ssid)) {
			if (printed)
				seq_putc(seq, ' ');
			seq_printf(seq, "%s", ss->name);
			printed = true;
		}
2493
	}
2494 2495
	if (printed)
		seq_putc(seq, '\n');
2496 2497
}

2498 2499
/* show controllers which are currently attached to the default hierarchy */
static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
2500
{
2501 2502
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2503 2504
	cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
			     ~cgrp_dfl_root_inhibit_ss_mask);
2505
	return 0;
2506 2507
}

2508 2509
/* show controllers which are enabled from the parent */
static int cgroup_controllers_show(struct seq_file *seq, void *v)
2510
{
2511 2512
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2513
	cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
2514
	return 0;
2515 2516
}

2517 2518
/* show controllers which are enabled for a given cgroup's children */
static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2519
{
2520 2521
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2522
	cgroup_print_ss_mask(seq, cgrp->subtree_control);
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
	return 0;
}

/**
 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
 * @cgrp: root of the subtree to update csses for
 *
 * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
 * css associations need to be updated accordingly.  This function looks up
 * all css_sets which are attached to the subtree, creates the matching
 * updated css_sets and migrates the tasks to the new ones.
 */
static int cgroup_update_dfl_csses(struct cgroup *cgrp)
{
	LIST_HEAD(preloaded_csets);
	struct cgroup_subsys_state *css;
	struct css_set *src_cset;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* look up all csses currently attached to @cgrp's subtree */
	down_read(&css_set_rwsem);
	css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
		struct cgrp_cset_link *link;

		/* self is not affected by child_subsys_mask change */
		if (css->cgroup == cgrp)
			continue;

		list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
			cgroup_migrate_add_src(link->cset, cgrp,
					       &preloaded_csets);
	}
	up_read(&css_set_rwsem);

	/* NULL dst indicates self on default hierarchy */
	ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
	if (ret)
		goto out_finish;

	list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
		struct task_struct *last_task = NULL, *task;

		/* src_csets precede dst_csets, break on the first dst_cset */
		if (!src_cset->mg_src_cgrp)
			break;

		/*
		 * All tasks in src_cset need to be migrated to the
		 * matching dst_cset.  Empty it process by process.  We
		 * walk tasks but migrate processes.  The leader might even
		 * belong to a different cset but such src_cset would also
		 * be among the target src_csets because the default
		 * hierarchy enforces per-process membership.
		 */
		while (true) {
			down_read(&css_set_rwsem);
			task = list_first_entry_or_null(&src_cset->tasks,
						struct task_struct, cg_list);
			if (task) {
				task = task->group_leader;
				WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp);
				get_task_struct(task);
			}
			up_read(&css_set_rwsem);

			if (!task)
				break;

			/* guard against possible infinite loop */
			if (WARN(last_task == task,
				 "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n"))
				goto out_finish;
			last_task = task;

			threadgroup_lock(task);
			/* raced against de_thread() from another thread? */
			if (!thread_group_leader(task)) {
				threadgroup_unlock(task);
				put_task_struct(task);
				continue;
			}

			ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);

			threadgroup_unlock(task);
			put_task_struct(task);

			if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
				goto out_finish;
		}
	}

out_finish:
	cgroup_migrate_finish(&preloaded_csets);
	return ret;
}

/* change the enabled child controllers for a cgroup in the default hierarchy */
2623 2624 2625
static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
2626
{
2627
	unsigned int enable = 0, disable = 0;
2628
	unsigned int css_enable, css_disable, old_ctrl, new_ctrl;
2629
	struct cgroup *cgrp, *child;
2630
	struct cgroup_subsys *ss;
2631
	char *tok;
2632 2633 2634
	int ssid, ret;

	/*
2635 2636
	 * Parse input - space separated list of subsystem names prefixed
	 * with either + or -.
2637
	 */
2638 2639
	buf = strstrip(buf);
	while ((tok = strsep(&buf, " "))) {
2640 2641
		if (tok[0] == '\0')
			continue;
2642
		for_each_subsys(ss, ssid) {
2643 2644
			if (ss->disabled || strcmp(tok + 1, ss->name) ||
			    ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask))
2645 2646 2647
				continue;

			if (*tok == '+') {
2648 2649
				enable |= 1 << ssid;
				disable &= ~(1 << ssid);
2650
			} else if (*tok == '-') {
2651 2652
				disable |= 1 << ssid;
				enable &= ~(1 << ssid);
2653 2654 2655 2656 2657 2658 2659 2660 2661
			} else {
				return -EINVAL;
			}
			break;
		}
		if (ssid == CGROUP_SUBSYS_COUNT)
			return -EINVAL;
	}

2662 2663 2664
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
		return -ENODEV;
2665 2666 2667

	for_each_subsys(ss, ssid) {
		if (enable & (1 << ssid)) {
2668
			if (cgrp->subtree_control & (1 << ssid)) {
2669 2670 2671 2672
				enable &= ~(1 << ssid);
				continue;
			}

2673 2674 2675
			/* unavailable or not enabled on the parent? */
			if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
			    (cgroup_parent(cgrp) &&
2676
			     !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
2677 2678 2679 2680
				ret = -ENOENT;
				goto out_unlock;
			}

2681 2682 2683 2684 2685 2686 2687
			/*
			 * @ss is already enabled through dependency and
			 * we'll just make it visible.  Skip draining.
			 */
			if (cgrp->child_subsys_mask & (1 << ssid))
				continue;

2688 2689 2690 2691 2692 2693 2694
			/*
			 * Because css offlining is asynchronous, userland
			 * might try to re-enable the same controller while
			 * the previous instance is still around.  In such
			 * cases, wait till it's gone using offline_waitq.
			 */
			cgroup_for_each_live_child(child, cgrp) {
2695
				DEFINE_WAIT(wait);
2696 2697 2698 2699

				if (!cgroup_css(child, ss))
					continue;

2700
				cgroup_get(child);
2701 2702
				prepare_to_wait(&child->offline_waitq, &wait,
						TASK_UNINTERRUPTIBLE);
2703
				cgroup_kn_unlock(of->kn);
2704 2705
				schedule();
				finish_wait(&child->offline_waitq, &wait);
2706
				cgroup_put(child);
2707

2708
				return restart_syscall();
2709 2710
			}
		} else if (disable & (1 << ssid)) {
2711
			if (!(cgrp->subtree_control & (1 << ssid))) {
2712 2713 2714 2715 2716 2717
				disable &= ~(1 << ssid);
				continue;
			}

			/* a child has it enabled? */
			cgroup_for_each_live_child(child, cgrp) {
2718
				if (child->subtree_control & (1 << ssid)) {
2719
					ret = -EBUSY;
2720
					goto out_unlock;
2721 2722 2723 2724 2725 2726 2727
				}
			}
		}
	}

	if (!enable && !disable) {
		ret = 0;
2728
		goto out_unlock;
2729 2730 2731
	}

	/*
2732
	 * Except for the root, subtree_control must be zero for a cgroup
2733 2734
	 * with tasks so that child cgroups don't compete against tasks.
	 */
T
Tejun Heo 已提交
2735
	if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
2736 2737 2738 2739
		ret = -EBUSY;
		goto out_unlock;
	}

2740 2741 2742 2743 2744
	/*
	 * Update subsys masks and calculate what needs to be done.  More
	 * subsystems than specified may need to be enabled or disabled
	 * depending on subsystem dependencies.
	 */
2745 2746
	cgrp->subtree_control |= enable;
	cgrp->subtree_control &= ~disable;
2747 2748

	old_ctrl = cgrp->child_subsys_mask;
2749
	cgroup_refresh_child_subsys_mask(cgrp);
2750 2751 2752 2753 2754 2755
	new_ctrl = cgrp->child_subsys_mask;

	css_enable = ~old_ctrl & new_ctrl;
	css_disable = old_ctrl & ~new_ctrl;
	enable |= css_enable;
	disable |= css_disable;
2756

2757 2758 2759 2760 2761 2762
	/*
	 * Create new csses or make the existing ones visible.  A css is
	 * created invisible if it's being implicitly enabled through
	 * dependency.  An invisible css is made visible when the userland
	 * explicitly enables it.
	 */
2763 2764 2765 2766 2767
	for_each_subsys(ss, ssid) {
		if (!(enable & (1 << ssid)))
			continue;

		cgroup_for_each_live_child(child, cgrp) {
2768 2769 2770 2771 2772
			if (css_enable & (1 << ssid))
				ret = create_css(child, ss,
					cgrp->subtree_control & (1 << ssid));
			else
				ret = cgroup_populate_dir(child, 1 << ssid);
2773 2774 2775 2776 2777
			if (ret)
				goto err_undo_css;
		}
	}

2778 2779 2780 2781 2782
	/*
	 * At this point, cgroup_e_css() results reflect the new csses
	 * making the following cgroup_update_dfl_csses() properly update
	 * css associations of all tasks in the subtree.
	 */
2783 2784 2785 2786
	ret = cgroup_update_dfl_csses(cgrp);
	if (ret)
		goto err_undo_css;

2787 2788 2789
	/*
	 * All tasks are migrated out of disabled csses.  Kill or hide
	 * them.  A css is hidden when the userland requests it to be
2790 2791 2792 2793
	 * disabled while other subsystems are still depending on it.  The
	 * css must not actively control resources and be in the vanilla
	 * state if it's made visible again later.  Controllers which may
	 * be depended upon should provide ->css_reset() for this purpose.
2794
	 */
2795 2796 2797 2798
	for_each_subsys(ss, ssid) {
		if (!(disable & (1 << ssid)))
			continue;

2799
		cgroup_for_each_live_child(child, cgrp) {
2800 2801 2802 2803 2804
			struct cgroup_subsys_state *css = cgroup_css(child, ss);

			if (css_disable & (1 << ssid)) {
				kill_css(css);
			} else {
2805
				cgroup_clear_dir(child, 1 << ssid);
2806 2807 2808
				if (ss->css_reset)
					ss->css_reset(css);
			}
2809
		}
2810 2811 2812 2813 2814
	}

	kernfs_activate(cgrp->kn);
	ret = 0;
out_unlock:
2815
	cgroup_kn_unlock(of->kn);
2816
	return ret ?: nbytes;
2817 2818

err_undo_css:
2819 2820 2821
	cgrp->subtree_control &= ~enable;
	cgrp->subtree_control |= disable;
	cgroup_refresh_child_subsys_mask(cgrp);
2822 2823 2824 2825 2826 2827 2828

	for_each_subsys(ss, ssid) {
		if (!(enable & (1 << ssid)))
			continue;

		cgroup_for_each_live_child(child, cgrp) {
			struct cgroup_subsys_state *css = cgroup_css(child, ss);
2829 2830 2831 2832 2833

			if (!css)
				continue;

			if (css_enable & (1 << ssid))
2834
				kill_css(css);
2835 2836
			else
				cgroup_clear_dir(child, 1 << ssid);
2837 2838 2839 2840 2841
		}
	}
	goto out_unlock;
}

2842 2843 2844 2845 2846 2847
static int cgroup_populated_show(struct seq_file *seq, void *v)
{
	seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt);
	return 0;
}

T
Tejun Heo 已提交
2848 2849
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
2850
{
T
Tejun Heo 已提交
2851 2852 2853
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = of->kn->priv;
	struct cgroup_subsys_state *css;
2854
	int ret;
2855

T
Tejun Heo 已提交
2856 2857 2858
	if (cft->write)
		return cft->write(of, buf, nbytes, off);

T
Tejun Heo 已提交
2859 2860 2861 2862 2863 2864 2865 2866 2867
	/*
	 * kernfs guarantees that a file isn't deleted with operations in
	 * flight, which means that the matching css is and stays alive and
	 * doesn't need to be pinned.  The RCU locking is not necessary
	 * either.  It's just for the convenience of using cgroup_css().
	 */
	rcu_read_lock();
	css = cgroup_css(cgrp, cft->ss);
	rcu_read_unlock();
2868

2869
	if (cft->write_u64) {
2870 2871 2872 2873 2874 2875 2876 2877 2878
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
2879
	} else {
2880
		ret = -EINVAL;
2881
	}
T
Tejun Heo 已提交
2882

2883
	return ret ?: nbytes;
2884 2885
}

2886
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
2887
{
T
Tejun Heo 已提交
2888
	return seq_cft(seq)->seq_start(seq, ppos);
2889 2890
}

2891
static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
2892
{
T
Tejun Heo 已提交
2893
	return seq_cft(seq)->seq_next(seq, v, ppos);
2894 2895
}

2896
static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
2897
{
T
Tejun Heo 已提交
2898
	seq_cft(seq)->seq_stop(seq, v);
2899 2900
}

2901
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2902
{
2903 2904
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
2905

2906 2907
	if (cft->seq_show)
		return cft->seq_show(m, arg);
2908

2909
	if (cft->read_u64)
2910 2911 2912 2913 2914 2915
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
2916 2917
}

T
Tejun Heo 已提交
2918 2919 2920 2921
static struct kernfs_ops cgroup_kf_single_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_show		= cgroup_seqfile_show,
2922 2923
};

T
Tejun Heo 已提交
2924 2925 2926 2927 2928 2929 2930 2931
static struct kernfs_ops cgroup_kf_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_start		= cgroup_seqfile_start,
	.seq_next		= cgroup_seqfile_next,
	.seq_stop		= cgroup_seqfile_stop,
	.seq_show		= cgroup_seqfile_show,
};
2932 2933 2934 2935

/*
 * cgroup_rename - Only allow simple rename of directories in place.
 */
T
Tejun Heo 已提交
2936 2937
static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
			 const char *new_name_str)
2938
{
T
Tejun Heo 已提交
2939
	struct cgroup *cgrp = kn->priv;
2940 2941
	int ret;

T
Tejun Heo 已提交
2942
	if (kernfs_type(kn) != KERNFS_DIR)
2943
		return -ENOTDIR;
T
Tejun Heo 已提交
2944
	if (kn->parent != new_parent)
2945
		return -EIO;
2946

2947 2948 2949 2950 2951 2952
	/*
	 * This isn't a proper migration and its usefulness is very
	 * limited.  Disallow if sane_behavior.
	 */
	if (cgroup_sane_behavior(cgrp))
		return -EPERM;
L
Li Zefan 已提交
2953

2954
	/*
T
Tejun Heo 已提交
2955
	 * We're gonna grab cgroup_mutex which nests outside kernfs
2956
	 * active_ref.  kernfs_rename() doesn't require active_ref
T
Tejun Heo 已提交
2957
	 * protection.  Break them before grabbing cgroup_mutex.
2958 2959 2960
	 */
	kernfs_break_active_protection(new_parent);
	kernfs_break_active_protection(kn);
L
Li Zefan 已提交
2961

T
Tejun Heo 已提交
2962
	mutex_lock(&cgroup_mutex);
L
Li Zefan 已提交
2963

T
Tejun Heo 已提交
2964
	ret = kernfs_rename(kn, new_parent, new_name_str);
L
Li Zefan 已提交
2965

T
Tejun Heo 已提交
2966
	mutex_unlock(&cgroup_mutex);
2967 2968 2969

	kernfs_unbreak_active_protection(kn);
	kernfs_unbreak_active_protection(new_parent);
T
Tejun Heo 已提交
2970
	return ret;
L
Li Zefan 已提交
2971 2972
}

2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986
/* set uid and gid of cgroup dirs and files to that of the creator */
static int cgroup_kn_set_ugid(struct kernfs_node *kn)
{
	struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
			       .ia_uid = current_fsuid(),
			       .ia_gid = current_fsgid(), };

	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
	    gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
		return 0;

	return kernfs_setattr(kn, &iattr);
}

2987
static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2988
{
T
Tejun Heo 已提交
2989
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
2990 2991
	struct kernfs_node *kn;
	struct lock_class_key *key = NULL;
2992
	int ret;
T
Tejun Heo 已提交
2993

T
Tejun Heo 已提交
2994 2995 2996 2997 2998 2999
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	key = &cft->lockdep_key;
#endif
	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
				  cgroup_file_mode(cft), 0, cft->kf_ops, cft,
				  NULL, false, key);
3000 3001 3002 3003
	if (IS_ERR(kn))
		return PTR_ERR(kn);

	ret = cgroup_kn_set_ugid(kn);
3004
	if (ret) {
3005
		kernfs_remove(kn);
3006 3007 3008
		return ret;
	}

T
Tejun Heo 已提交
3009
	if (cft->seq_show == cgroup_populated_show)
3010
		cgrp->populated_kn = kn;
3011
	return 0;
3012 3013
}

3014 3015 3016 3017 3018 3019 3020
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
 * @cgrp: the target cgroup
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3021 3022 3023
 * For removals, this function never fails.  If addition fails, this
 * function doesn't remove files already added.  The caller is responsible
 * for cleaning up.
3024
 */
3025 3026
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add)
3027
{
A
Aristeu Rozanski 已提交
3028
	struct cftype *cft;
3029 3030
	int ret;

3031
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
3032 3033

	for (cft = cfts; cft->name[0] != '\0'; cft++) {
3034
		/* does cft->flags tell us to skip this file on @cgrp? */
T
Tejun Heo 已提交
3035 3036
		if ((cft->flags & CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
			continue;
3037 3038
		if ((cft->flags & CFTYPE_INSANE) && cgroup_sane_behavior(cgrp))
			continue;
T
Tejun Heo 已提交
3039
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3040
			continue;
T
Tejun Heo 已提交
3041
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3042 3043
			continue;

3044
		if (is_add) {
3045
			ret = cgroup_add_file(cgrp, cft);
3046
			if (ret) {
3047 3048
				pr_warn("%s: failed to add %s, err=%d\n",
					__func__, cft->name, ret);
3049 3050
				return ret;
			}
3051 3052
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
3053
		}
3054
	}
3055
	return 0;
3056 3057
}

3058
static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3059 3060
{
	LIST_HEAD(pending);
3061
	struct cgroup_subsys *ss = cfts[0].ss;
3062
	struct cgroup *root = &ss->root->cgrp;
3063
	struct cgroup_subsys_state *css;
3064
	int ret = 0;
3065

3066
	lockdep_assert_held(&cgroup_mutex);
3067 3068

	/* add/rm files for all cgroups created before */
3069
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3070 3071
		struct cgroup *cgrp = css->cgroup;

3072 3073 3074
		if (cgroup_is_dead(cgrp))
			continue;

3075
		ret = cgroup_addrm_files(cgrp, cfts, is_add);
3076 3077
		if (ret)
			break;
3078
	}
3079 3080 3081

	if (is_add && !ret)
		kernfs_activate(root->kn);
3082
	return ret;
3083 3084
}

3085
static void cgroup_exit_cftypes(struct cftype *cfts)
3086
{
3087
	struct cftype *cft;
3088

T
Tejun Heo 已提交
3089 3090 3091 3092 3093
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		/* free copy for custom atomic_write_len, see init_cftypes() */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
			kfree(cft->kf_ops);
		cft->kf_ops = NULL;
3094
		cft->ss = NULL;
T
Tejun Heo 已提交
3095
	}
3096 3097
}

T
Tejun Heo 已提交
3098
static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3099 3100 3101
{
	struct cftype *cft;

T
Tejun Heo 已提交
3102 3103 3104
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		struct kernfs_ops *kf_ops;

T
Tejun Heo 已提交
3105 3106
		WARN_ON(cft->ss || cft->kf_ops);

T
Tejun Heo 已提交
3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
		if (cft->seq_start)
			kf_ops = &cgroup_kf_ops;
		else
			kf_ops = &cgroup_kf_single_ops;

		/*
		 * Ugh... if @cft wants a custom max_write_len, we need to
		 * make a copy of kf_ops to set its atomic_write_len.
		 */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
			kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
			if (!kf_ops) {
				cgroup_exit_cftypes(cfts);
				return -ENOMEM;
			}
			kf_ops->atomic_write_len = cft->max_write_len;
		}
3124

T
Tejun Heo 已提交
3125
		cft->kf_ops = kf_ops;
3126
		cft->ss = ss;
T
Tejun Heo 已提交
3127
	}
3128

T
Tejun Heo 已提交
3129
	return 0;
3130 3131
}

3132 3133
static int cgroup_rm_cftypes_locked(struct cftype *cfts)
{
3134
	lockdep_assert_held(&cgroup_mutex);
3135 3136 3137 3138 3139 3140 3141 3142

	if (!cfts || !cfts[0].ss)
		return -ENOENT;

	list_del(&cfts->node);
	cgroup_apply_cftypes(cfts, false);
	cgroup_exit_cftypes(cfts);
	return 0;
3143 3144
}

3145 3146 3147 3148
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
3149 3150 3151
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
3152 3153
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3154
 * registered.
3155
 */
3156
int cgroup_rm_cftypes(struct cftype *cfts)
3157
{
3158
	int ret;
3159

3160
	mutex_lock(&cgroup_mutex);
3161
	ret = cgroup_rm_cftypes_locked(cfts);
3162
	mutex_unlock(&cgroup_mutex);
3163
	return ret;
T
Tejun Heo 已提交
3164 3165
}

3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
A
Aristeu Rozanski 已提交
3180
int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3181
{
3182
	int ret;
3183

3184 3185 3186
	if (ss->disabled)
		return 0;

3187 3188
	if (!cfts || cfts[0].name[0] == '\0')
		return 0;
3189

T
Tejun Heo 已提交
3190 3191 3192
	ret = cgroup_init_cftypes(ss, cfts);
	if (ret)
		return ret;
3193

3194
	mutex_lock(&cgroup_mutex);
3195

T
Tejun Heo 已提交
3196
	list_add_tail(&cfts->node, &ss->cfts);
3197
	ret = cgroup_apply_cftypes(cfts, true);
3198
	if (ret)
3199
		cgroup_rm_cftypes_locked(cfts);
3200

3201
	mutex_unlock(&cgroup_mutex);
3202
	return ret;
3203 3204
}

L
Li Zefan 已提交
3205 3206 3207 3208 3209 3210
/**
 * cgroup_task_count - count the number of tasks in a cgroup.
 * @cgrp: the cgroup in question
 *
 * Return the number of tasks in the cgroup.
 */
3211
static int cgroup_task_count(const struct cgroup *cgrp)
3212 3213
{
	int count = 0;
3214
	struct cgrp_cset_link *link;
3215

3216
	down_read(&css_set_rwsem);
3217 3218
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		count += atomic_read(&link->cset->refcount);
3219
	up_read(&css_set_rwsem);
3220 3221 3222
	return count;
}

3223
/**
3224
 * css_next_child - find the next child of a given css
3225 3226
 * @pos: the current position (%NULL to initiate traversal)
 * @parent: css whose children to walk
3227
 *
3228
 * This function returns the next child of @parent and should be called
3229
 * under either cgroup_mutex or RCU read lock.  The only requirement is
3230 3231 3232 3233 3234 3235 3236 3237 3238
 * that @parent and @pos are accessible.  The next sibling is guaranteed to
 * be returned regardless of their states.
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3239
 */
3240 3241
struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
					   struct cgroup_subsys_state *parent)
3242
{
3243
	struct cgroup_subsys_state *next;
3244

T
Tejun Heo 已提交
3245
	cgroup_assert_mutex_or_rcu_locked();
3246 3247

	/*
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257
	 * @pos could already have been unlinked from the sibling list.
	 * Once a cgroup is removed, its ->sibling.next is no longer
	 * updated when its next sibling changes.  CSS_RELEASED is set when
	 * @pos is taken off list, at which time its next pointer is valid,
	 * and, as releases are serialized, the one pointed to by the next
	 * pointer is guaranteed to not have started release yet.  This
	 * implies that if we observe !CSS_RELEASED on @pos in this RCU
	 * critical section, the one pointed to by its next pointer is
	 * guaranteed to not have finished its RCU grace period even if we
	 * have dropped rcu_read_lock() inbetween iterations.
3258
	 *
3259 3260 3261 3262 3263 3264 3265
	 * If @pos has CSS_RELEASED set, its next pointer can't be
	 * dereferenced; however, as each css is given a monotonically
	 * increasing unique serial number and always appended to the
	 * sibling list, the next one can be found by walking the parent's
	 * children until the first css with higher serial number than
	 * @pos's.  While this path can be slower, it happens iff iteration
	 * races against release and the race window is very small.
3266
	 */
3267
	if (!pos) {
3268 3269 3270
		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
	} else if (likely(!(pos->flags & CSS_RELEASED))) {
		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3271
	} else {
3272
		list_for_each_entry_rcu(next, &parent->children, sibling)
3273 3274
			if (next->serial_nr > pos->serial_nr)
				break;
3275 3276
	}

3277 3278
	/*
	 * @next, if not pointing to the head, can be dereferenced and is
3279
	 * the next sibling.
3280
	 */
3281 3282
	if (&next->sibling != &parent->children)
		return next;
3283
	return NULL;
3284 3285
}

3286
/**
3287
 * css_next_descendant_pre - find the next descendant for pre-order walk
3288
 * @pos: the current position (%NULL to initiate traversal)
3289
 * @root: css whose descendants to walk
3290
 *
3291
 * To be used by css_for_each_descendant_pre().  Find the next descendant
3292 3293
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
3294
 *
3295 3296 3297 3298
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
3299 3300 3301 3302 3303 3304 3305
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3306
 */
3307 3308 3309
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
3310
{
3311
	struct cgroup_subsys_state *next;
3312

T
Tejun Heo 已提交
3313
	cgroup_assert_mutex_or_rcu_locked();
3314

3315
	/* if first iteration, visit @root */
3316
	if (!pos)
3317
		return root;
3318 3319

	/* visit the first child if exists */
3320
	next = css_next_child(NULL, pos);
3321 3322 3323 3324
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
3325
	while (pos != root) {
T
Tejun Heo 已提交
3326
		next = css_next_child(pos, pos->parent);
3327
		if (next)
3328
			return next;
T
Tejun Heo 已提交
3329
		pos = pos->parent;
3330
	}
3331 3332 3333 3334

	return NULL;
}

3335
/**
3336 3337
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
3338
 *
3339 3340
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
3341
 * subtree of @pos.
3342
 *
3343 3344 3345 3346
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
3347
 */
3348 3349
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
3350
{
3351
	struct cgroup_subsys_state *last, *tmp;
3352

T
Tejun Heo 已提交
3353
	cgroup_assert_mutex_or_rcu_locked();
3354 3355 3356 3357 3358

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
3359
		css_for_each_child(tmp, last)
3360 3361 3362 3363 3364 3365
			pos = tmp;
	} while (pos);

	return last;
}

3366 3367
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
3368
{
3369
	struct cgroup_subsys_state *last;
3370 3371 3372

	do {
		last = pos;
3373
		pos = css_next_child(NULL, pos);
3374 3375 3376 3377 3378 3379
	} while (pos);

	return last;
}

/**
3380
 * css_next_descendant_post - find the next descendant for post-order walk
3381
 * @pos: the current position (%NULL to initiate traversal)
3382
 * @root: css whose descendants to walk
3383
 *
3384
 * To be used by css_for_each_descendant_post().  Find the next descendant
3385 3386
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
3387
 *
3388 3389 3390 3391 3392
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
3393 3394 3395 3396 3397 3398 3399
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3400
 */
3401 3402 3403
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
3404
{
3405
	struct cgroup_subsys_state *next;
3406

T
Tejun Heo 已提交
3407
	cgroup_assert_mutex_or_rcu_locked();
3408

3409 3410 3411
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
3412

3413 3414 3415 3416
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

3417
	/* if there's an unvisited sibling, visit its leftmost descendant */
T
Tejun Heo 已提交
3418
	next = css_next_child(pos, pos->parent);
3419
	if (next)
3420
		return css_leftmost_descendant(next);
3421 3422

	/* no sibling left, visit parent */
T
Tejun Heo 已提交
3423
	return pos->parent;
3424 3425
}

3426 3427 3428 3429 3430 3431 3432 3433 3434
/**
 * css_has_online_children - does a css have online children
 * @css: the target css
 *
 * Returns %true if @css has any online children; otherwise, %false.  This
 * function can be called from any context but the caller is responsible
 * for synchronizing against on/offlining as necessary.
 */
bool css_has_online_children(struct cgroup_subsys_state *css)
3435
{
3436 3437
	struct cgroup_subsys_state *child;
	bool ret = false;
3438 3439

	rcu_read_lock();
3440 3441 3442 3443
	css_for_each_child(child, css) {
		if (css->flags & CSS_ONLINE) {
			ret = true;
			break;
3444 3445 3446
		}
	}
	rcu_read_unlock();
3447
	return ret;
3448 3449
}

3450
/**
3451
 * css_advance_task_iter - advance a task itererator to the next css_set
3452 3453 3454
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
3455
 */
3456
static void css_advance_task_iter(struct css_task_iter *it)
3457
{
T
Tejun Heo 已提交
3458
	struct list_head *l = it->cset_pos;
3459 3460 3461 3462 3463 3464
	struct cgrp_cset_link *link;
	struct css_set *cset;

	/* Advance to the next non-empty css_set */
	do {
		l = l->next;
T
Tejun Heo 已提交
3465 3466
		if (l == it->cset_head) {
			it->cset_pos = NULL;
3467 3468
			return;
		}
3469 3470 3471 3472 3473 3474 3475 3476

		if (it->ss) {
			cset = container_of(l, struct css_set,
					    e_cset_node[it->ss->id]);
		} else {
			link = list_entry(l, struct cgrp_cset_link, cset_link);
			cset = link->cset;
		}
T
Tejun Heo 已提交
3477 3478
	} while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));

T
Tejun Heo 已提交
3479
	it->cset_pos = l;
T
Tejun Heo 已提交
3480 3481

	if (!list_empty(&cset->tasks))
T
Tejun Heo 已提交
3482
		it->task_pos = cset->tasks.next;
T
Tejun Heo 已提交
3483
	else
T
Tejun Heo 已提交
3484 3485 3486 3487
		it->task_pos = cset->mg_tasks.next;

	it->tasks_head = &cset->tasks;
	it->mg_tasks_head = &cset->mg_tasks;
3488 3489
}

3490
/**
3491 3492
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
3493 3494
 * @it: the task iterator to use
 *
3495 3496 3497 3498
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
3499 3500 3501 3502 3503
 *
 * Note that this function acquires a lock which is released when the
 * iteration finishes.  The caller can't sleep while iteration is in
 * progress.
 */
3504 3505
void css_task_iter_start(struct cgroup_subsys_state *css,
			 struct css_task_iter *it)
3506
	__acquires(css_set_rwsem)
3507
{
3508 3509
	/* no one should try to iterate before mounting cgroups */
	WARN_ON_ONCE(!use_task_css_set_links);
3510

3511
	down_read(&css_set_rwsem);
3512

3513 3514 3515 3516 3517 3518 3519
	it->ss = css->ss;

	if (it->ss)
		it->cset_pos = &css->cgroup->e_csets[css->ss->id];
	else
		it->cset_pos = &css->cgroup->cset_links;

T
Tejun Heo 已提交
3520
	it->cset_head = it->cset_pos;
3521

3522
	css_advance_task_iter(it);
3523 3524
}

3525
/**
3526
 * css_task_iter_next - return the next task for the iterator
3527 3528 3529
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
3530 3531
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
3532
 */
3533
struct task_struct *css_task_iter_next(struct css_task_iter *it)
3534 3535
{
	struct task_struct *res;
T
Tejun Heo 已提交
3536
	struct list_head *l = it->task_pos;
3537 3538

	/* If the iterator cg is NULL, we have no tasks */
T
Tejun Heo 已提交
3539
	if (!it->cset_pos)
3540 3541
		return NULL;
	res = list_entry(l, struct task_struct, cg_list);
T
Tejun Heo 已提交
3542 3543 3544 3545 3546 3547

	/*
	 * Advance iterator to find next entry.  cset->tasks is consumed
	 * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
	 * next cset.
	 */
3548
	l = l->next;
T
Tejun Heo 已提交
3549

T
Tejun Heo 已提交
3550 3551
	if (l == it->tasks_head)
		l = it->mg_tasks_head->next;
T
Tejun Heo 已提交
3552

T
Tejun Heo 已提交
3553
	if (l == it->mg_tasks_head)
3554
		css_advance_task_iter(it);
T
Tejun Heo 已提交
3555
	else
T
Tejun Heo 已提交
3556
		it->task_pos = l;
T
Tejun Heo 已提交
3557

3558 3559 3560
	return res;
}

3561
/**
3562
 * css_task_iter_end - finish task iteration
3563 3564
 * @it: the task iterator to finish
 *
3565
 * Finish task iteration started by css_task_iter_start().
3566
 */
3567
void css_task_iter_end(struct css_task_iter *it)
3568
	__releases(css_set_rwsem)
3569
{
3570
	up_read(&css_set_rwsem);
3571 3572 3573
}

/**
3574 3575 3576
 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
 * @to: cgroup to which the tasks will be moved
 * @from: cgroup in which the tasks currently reside
3577
 *
3578 3579 3580 3581 3582
 * Locking rules between cgroup_post_fork() and the migration path
 * guarantee that, if a task is forking while being migrated, the new child
 * is guaranteed to be either visible in the source cgroup after the
 * parent's migration is complete or put into the target cgroup.  No task
 * can slip out of migration through forking.
3583
 */
3584
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
3585
{
3586 3587
	LIST_HEAD(preloaded_csets);
	struct cgrp_cset_link *link;
3588
	struct css_task_iter it;
3589
	struct task_struct *task;
3590
	int ret;
3591

3592
	mutex_lock(&cgroup_mutex);
3593

3594 3595 3596 3597 3598
	/* all tasks in @from are being moved, all csets are source */
	down_read(&css_set_rwsem);
	list_for_each_entry(link, &from->cset_links, cset_link)
		cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
	up_read(&css_set_rwsem);
3599

3600 3601 3602
	ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
	if (ret)
		goto out_err;
3603

3604 3605 3606 3607
	/*
	 * Migrate tasks one-by-one until @form is empty.  This fails iff
	 * ->can_attach() fails.
	 */
3608
	do {
3609
		css_task_iter_start(&from->self, &it);
3610 3611 3612 3613 3614 3615
		task = css_task_iter_next(&it);
		if (task)
			get_task_struct(task);
		css_task_iter_end(&it);

		if (task) {
3616
			ret = cgroup_migrate(to, task, false);
3617 3618 3619
			put_task_struct(task);
		}
	} while (task && !ret);
3620 3621
out_err:
	cgroup_migrate_finish(&preloaded_csets);
T
Tejun Heo 已提交
3622
	mutex_unlock(&cgroup_mutex);
3623
	return ret;
3624 3625
}

3626
/*
3627
 * Stuff for reading the 'tasks'/'procs' files.
3628 3629 3630 3631 3632 3633 3634 3635
 *
 * Reading this file can return large amounts of data if a cgroup has
 * *lots* of attached tasks. So it may need several calls to read(),
 * but we cannot guarantee that the information we produce is correct
 * unless we produce it entirely atomically.
 *
 */

3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661
/* which pidlist file are we talking about? */
enum cgroup_filetype {
	CGROUP_FILE_PROCS,
	CGROUP_FILE_TASKS,
};

/*
 * A pidlist is a list of pids that virtually represents the contents of one
 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
 * a pair (one each for procs, tasks) for each pid namespace that's relevant
 * to the cgroup.
 */
struct cgroup_pidlist {
	/*
	 * used to find which pidlist is wanted. doesn't change as long as
	 * this particular list stays in the list.
	*/
	struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
	/* array of xids */
	pid_t *list;
	/* how many elements the above list has */
	int length;
	/* each of these stored in a list by its cgroup */
	struct list_head links;
	/* pointer to the cgroup we belong to, for list removal purposes */
	struct cgroup *owner;
3662 3663
	/* for delayed destruction */
	struct delayed_work destroy_dwork;
3664 3665
};

3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678
/*
 * The following two functions "fix" the issue where there are more pids
 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
 * TODO: replace with a kernel-wide solution to this problem
 */
#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
static void *pidlist_allocate(int count)
{
	if (PIDLIST_TOO_LARGE(count))
		return vmalloc(count * sizeof(pid_t));
	else
		return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
}
3679

3680 3681 3682 3683 3684 3685 3686 3687
static void pidlist_free(void *p)
{
	if (is_vmalloc_addr(p))
		vfree(p);
	else
		kfree(p);
}

3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
/*
 * Used to destroy all pidlists lingering waiting for destroy timer.  None
 * should be left afterwards.
 */
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
{
	struct cgroup_pidlist *l, *tmp_l;

	mutex_lock(&cgrp->pidlist_mutex);
	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
	mutex_unlock(&cgrp->pidlist_mutex);

	flush_workqueue(cgroup_pidlist_destroy_wq);
	BUG_ON(!list_empty(&cgrp->pidlists));
}

static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
						destroy_dwork);
	struct cgroup_pidlist *tofree = NULL;

	mutex_lock(&l->owner->pidlist_mutex);

	/*
3715 3716
	 * Destroy iff we didn't get queued again.  The state won't change
	 * as destroy_dwork can only be queued while locked.
3717
	 */
3718
	if (!delayed_work_pending(dwork)) {
3719 3720 3721 3722 3723 3724 3725 3726 3727 3728
		list_del(&l->links);
		pidlist_free(l->list);
		put_pid_ns(l->key.ns);
		tofree = l;
	}

	mutex_unlock(&l->owner->pidlist_mutex);
	kfree(tofree);
}

3729
/*
3730
 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3731
 * Returns the number of unique elements.
3732
 */
3733
static int pidlist_uniq(pid_t *list, int length)
3734
{
3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758
	int src, dest = 1;

	/*
	 * we presume the 0th element is unique, so i starts at 1. trivial
	 * edge cases first; no work needs to be done for either
	 */
	if (length == 0 || length == 1)
		return length;
	/* src and dest walk down the list; dest counts unique elements */
	for (src = 1; src < length; src++) {
		/* find next unique element */
		while (list[src] == list[src-1]) {
			src++;
			if (src == length)
				goto after;
		}
		/* dest always points to where the next unique element goes */
		list[dest] = list[src];
		dest++;
	}
after:
	return dest;
}

3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791
/*
 * The two pid files - task and cgroup.procs - guaranteed that the result
 * is sorted, which forced this whole pidlist fiasco.  As pid order is
 * different per namespace, each namespace needs differently sorted list,
 * making it impossible to use, for example, single rbtree of member tasks
 * sorted by task pointer.  As pidlists can be fairly large, allocating one
 * per open file is dangerous, so cgroup had to implement shared pool of
 * pidlists keyed by cgroup and namespace.
 *
 * All this extra complexity was caused by the original implementation
 * committing to an entirely unnecessary property.  In the long term, we
 * want to do away with it.  Explicitly scramble sort order if
 * sane_behavior so that no such expectation exists in the new interface.
 *
 * Scrambling is done by swapping every two consecutive bits, which is
 * non-identity one-to-one mapping which disturbs sort order sufficiently.
 */
static pid_t pid_fry(pid_t pid)
{
	unsigned a = pid & 0x55555555;
	unsigned b = pid & 0xAAAAAAAA;

	return (a << 1) | (b >> 1);
}

static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
{
	if (cgroup_sane_behavior(cgrp))
		return pid_fry(pid);
	else
		return pid;
}

3792 3793 3794 3795 3796
static int cmppid(const void *a, const void *b)
{
	return *(pid_t *)a - *(pid_t *)b;
}

3797 3798 3799 3800 3801
static int fried_cmppid(const void *a, const void *b)
{
	return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
}

T
Tejun Heo 已提交
3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
						  enum cgroup_filetype type)
{
	struct cgroup_pidlist *l;
	/* don't need task_nsproxy() if we're looking at ourself */
	struct pid_namespace *ns = task_active_pid_ns(current);

	lockdep_assert_held(&cgrp->pidlist_mutex);

	list_for_each_entry(l, &cgrp->pidlists, links)
		if (l->key.type == type && l->key.ns == ns)
			return l;
	return NULL;
}

3817 3818 3819 3820 3821 3822
/*
 * find the appropriate pidlist for our purpose (given procs vs tasks)
 * returns with the lock on that pidlist already held, and takes care
 * of the use count, or returns NULL with no locks held if we're out of
 * memory.
 */
T
Tejun Heo 已提交
3823 3824
static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
						enum cgroup_filetype type)
3825 3826
{
	struct cgroup_pidlist *l;
3827

T
Tejun Heo 已提交
3828 3829 3830 3831 3832 3833
	lockdep_assert_held(&cgrp->pidlist_mutex);

	l = cgroup_pidlist_find(cgrp, type);
	if (l)
		return l;

3834
	/* entry not found; create a new one */
3835
	l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
T
Tejun Heo 已提交
3836
	if (!l)
3837
		return l;
T
Tejun Heo 已提交
3838

3839
	INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3840
	l->key.type = type;
T
Tejun Heo 已提交
3841 3842
	/* don't need task_nsproxy() if we're looking at ourself */
	l->key.ns = get_pid_ns(task_active_pid_ns(current));
3843 3844 3845 3846 3847
	l->owner = cgrp;
	list_add(&l->links, &cgrp->pidlists);
	return l;
}

3848 3849 3850
/*
 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
 */
3851 3852
static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
			      struct cgroup_pidlist **lp)
3853 3854 3855 3856
{
	pid_t *array;
	int length;
	int pid, n = 0; /* used for populating the array */
3857
	struct css_task_iter it;
3858
	struct task_struct *tsk;
3859 3860
	struct cgroup_pidlist *l;

3861 3862
	lockdep_assert_held(&cgrp->pidlist_mutex);

3863 3864 3865 3866 3867 3868 3869
	/*
	 * If cgroup gets more users after we read count, we won't have
	 * enough space - tough.  This race is indistinguishable to the
	 * caller from the case that the additional cgroup users didn't
	 * show up until sometime later on.
	 */
	length = cgroup_task_count(cgrp);
3870
	array = pidlist_allocate(length);
3871 3872 3873
	if (!array)
		return -ENOMEM;
	/* now, populate the array */
3874
	css_task_iter_start(&cgrp->self, &it);
3875
	while ((tsk = css_task_iter_next(&it))) {
3876
		if (unlikely(n == length))
3877
			break;
3878
		/* get tgid or pid for procs or tasks file respectively */
3879 3880 3881 3882
		if (type == CGROUP_FILE_PROCS)
			pid = task_tgid_vnr(tsk);
		else
			pid = task_pid_vnr(tsk);
3883 3884
		if (pid > 0) /* make sure to only use valid results */
			array[n++] = pid;
3885
	}
3886
	css_task_iter_end(&it);
3887 3888
	length = n;
	/* now sort & (if procs) strip out duplicates */
3889 3890 3891 3892
	if (cgroup_sane_behavior(cgrp))
		sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
	else
		sort(array, length, sizeof(pid_t), cmppid, NULL);
3893
	if (type == CGROUP_FILE_PROCS)
3894
		length = pidlist_uniq(array, length);
T
Tejun Heo 已提交
3895 3896

	l = cgroup_pidlist_find_create(cgrp, type);
3897
	if (!l) {
T
Tejun Heo 已提交
3898
		mutex_unlock(&cgrp->pidlist_mutex);
3899
		pidlist_free(array);
3900
		return -ENOMEM;
3901
	}
T
Tejun Heo 已提交
3902 3903

	/* store array, freeing old if necessary */
3904
	pidlist_free(l->list);
3905 3906
	l->list = array;
	l->length = length;
3907
	*lp = l;
3908
	return 0;
3909 3910
}

B
Balbir Singh 已提交
3911
/**
L
Li Zefan 已提交
3912
 * cgroupstats_build - build and fill cgroupstats
B
Balbir Singh 已提交
3913 3914 3915
 * @stats: cgroupstats to fill information into
 * @dentry: A dentry entry belonging to the cgroup for which stats have
 * been requested.
L
Li Zefan 已提交
3916 3917 3918
 *
 * Build and fill cgroupstats so that taskstats can export it to user
 * space.
B
Balbir Singh 已提交
3919 3920 3921
 */
int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
{
T
Tejun Heo 已提交
3922
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
3923
	struct cgroup *cgrp;
3924
	struct css_task_iter it;
B
Balbir Singh 已提交
3925
	struct task_struct *tsk;
3926

T
Tejun Heo 已提交
3927 3928 3929 3930 3931
	/* it should be kernfs_node belonging to cgroupfs and is a directory */
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
		return -EINVAL;

3932 3933
	mutex_lock(&cgroup_mutex);

B
Balbir Singh 已提交
3934
	/*
T
Tejun Heo 已提交
3935
	 * We aren't being called from kernfs and there's no guarantee on
3936
	 * @kn->priv's validity.  For this and css_tryget_online_from_dir(),
T
Tejun Heo 已提交
3937
	 * @kn->priv is RCU safe.  Let's do the RCU dancing.
B
Balbir Singh 已提交
3938
	 */
T
Tejun Heo 已提交
3939 3940
	rcu_read_lock();
	cgrp = rcu_dereference(kn->priv);
3941
	if (!cgrp || cgroup_is_dead(cgrp)) {
T
Tejun Heo 已提交
3942
		rcu_read_unlock();
3943
		mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3944 3945
		return -ENOENT;
	}
3946
	rcu_read_unlock();
B
Balbir Singh 已提交
3947

3948
	css_task_iter_start(&cgrp->self, &it);
3949
	while ((tsk = css_task_iter_next(&it))) {
B
Balbir Singh 已提交
3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
		switch (tsk->state) {
		case TASK_RUNNING:
			stats->nr_running++;
			break;
		case TASK_INTERRUPTIBLE:
			stats->nr_sleeping++;
			break;
		case TASK_UNINTERRUPTIBLE:
			stats->nr_uninterruptible++;
			break;
		case TASK_STOPPED:
			stats->nr_stopped++;
			break;
		default:
			if (delayacct_is_task_waiting_on_io(tsk))
				stats->nr_io_wait++;
			break;
		}
	}
3969
	css_task_iter_end(&it);
B
Balbir Singh 已提交
3970

3971
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3972
	return 0;
B
Balbir Singh 已提交
3973 3974
}

3975

3976
/*
3977
 * seq_file methods for the tasks/procs files. The seq_file position is the
3978
 * next pid to display; the seq_file iterator is a pointer to the pid
3979
 * in the cgroup->l->list array.
3980
 */
3981

3982
static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
3983
{
3984 3985 3986 3987 3988 3989
	/*
	 * Initially we receive a position value that corresponds to
	 * one more than the last pid shown (or 0 on the first call or
	 * after a seek to the start). Use a binary-search to find the
	 * next pid to display, if any
	 */
T
Tejun Heo 已提交
3990
	struct kernfs_open_file *of = s->private;
3991
	struct cgroup *cgrp = seq_css(s)->cgroup;
3992
	struct cgroup_pidlist *l;
3993
	enum cgroup_filetype type = seq_cft(s)->private;
3994
	int index = 0, pid = *pos;
3995 3996 3997 3998 3999
	int *iter, ret;

	mutex_lock(&cgrp->pidlist_mutex);

	/*
4000
	 * !NULL @of->priv indicates that this isn't the first start()
4001
	 * after open.  If the matching pidlist is around, we can use that.
4002
	 * Look for it.  Note that @of->priv can't be used directly.  It
4003 4004
	 * could already have been destroyed.
	 */
4005 4006
	if (of->priv)
		of->priv = cgroup_pidlist_find(cgrp, type);
4007 4008 4009 4010 4011

	/*
	 * Either this is the first start() after open or the matching
	 * pidlist has been destroyed inbetween.  Create a new one.
	 */
4012 4013 4014
	if (!of->priv) {
		ret = pidlist_array_load(cgrp, type,
					 (struct cgroup_pidlist **)&of->priv);
4015 4016 4017
		if (ret)
			return ERR_PTR(ret);
	}
4018
	l = of->priv;
4019 4020

	if (pid) {
4021
		int end = l->length;
S
Stephen Rothwell 已提交
4022

4023 4024
		while (index < end) {
			int mid = (index + end) / 2;
4025
			if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
4026 4027
				index = mid;
				break;
4028
			} else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
4029 4030 4031 4032 4033 4034
				index = mid + 1;
			else
				end = mid;
		}
	}
	/* If we're off the end of the array, we're done */
4035
	if (index >= l->length)
4036 4037
		return NULL;
	/* Update the abstract position to be the actual pid that we found */
4038
	iter = l->list + index;
4039
	*pos = cgroup_pid_fry(cgrp, *iter);
4040 4041 4042
	return iter;
}

4043
static void cgroup_pidlist_stop(struct seq_file *s, void *v)
4044
{
T
Tejun Heo 已提交
4045
	struct kernfs_open_file *of = s->private;
4046
	struct cgroup_pidlist *l = of->priv;
4047

4048 4049
	if (l)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
4050
				 CGROUP_PIDLIST_DESTROY_DELAY);
4051
	mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
4052 4053
}

4054
static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
4055
{
T
Tejun Heo 已提交
4056
	struct kernfs_open_file *of = s->private;
4057
	struct cgroup_pidlist *l = of->priv;
4058 4059
	pid_t *p = v;
	pid_t *end = l->list + l->length;
4060 4061 4062 4063 4064 4065 4066 4067
	/*
	 * Advance to the next pid in the array. If this goes off the
	 * end, we're done
	 */
	p++;
	if (p >= end) {
		return NULL;
	} else {
4068
		*pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
4069 4070 4071 4072
		return p;
	}
}

4073
static int cgroup_pidlist_show(struct seq_file *s, void *v)
4074 4075 4076
{
	return seq_printf(s, "%d\n", *(int *)v);
}
4077

4078 4079
static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
					 struct cftype *cft)
4080
{
4081
	return notify_on_release(css->cgroup);
4082 4083
}

4084 4085
static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
					  struct cftype *cft, u64 val)
4086
{
4087
	clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
4088
	if (val)
4089
		set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4090
	else
4091
		clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4092 4093 4094
	return 0;
}

4095 4096
static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
4097
{
4098
	return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4099 4100
}

4101 4102
static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
4103 4104
{
	if (val)
4105
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4106
	else
4107
		clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4108 4109 4110
	return 0;
}

4111
static struct cftype cgroup_base_files[] = {
4112
	{
4113
		.name = "cgroup.procs",
4114 4115 4116 4117
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
4118
		.private = CGROUP_FILE_PROCS,
4119
		.write = cgroup_procs_write,
B
Ben Blum 已提交
4120
		.mode = S_IRUGO | S_IWUSR,
4121
	},
4122 4123
	{
		.name = "cgroup.clone_children",
4124
		.flags = CFTYPE_INSANE,
4125 4126 4127
		.read_u64 = cgroup_clone_children_read,
		.write_u64 = cgroup_clone_children_write,
	},
4128 4129 4130
	{
		.name = "cgroup.sane_behavior",
		.flags = CFTYPE_ONLY_ON_ROOT,
4131
		.seq_show = cgroup_sane_behavior_show,
4132
	},
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146
	{
		.name = "cgroup.controllers",
		.flags = CFTYPE_ONLY_ON_DFL | CFTYPE_ONLY_ON_ROOT,
		.seq_show = cgroup_root_controllers_show,
	},
	{
		.name = "cgroup.controllers",
		.flags = CFTYPE_ONLY_ON_DFL | CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_controllers_show,
	},
	{
		.name = "cgroup.subtree_control",
		.flags = CFTYPE_ONLY_ON_DFL,
		.seq_show = cgroup_subtree_control_show,
4147
		.write = cgroup_subtree_control_write,
4148
	},
4149 4150 4151 4152 4153
	{
		.name = "cgroup.populated",
		.flags = CFTYPE_ONLY_ON_DFL | CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_populated_show,
	},
4154 4155 4156 4157 4158 4159 4160 4161 4162

	/*
	 * Historical crazy stuff.  These don't have "cgroup."  prefix and
	 * don't exist if sane_behavior.  If you're depending on these, be
	 * prepared to be burned.
	 */
	{
		.name = "tasks",
		.flags = CFTYPE_INSANE,		/* use "procs" instead */
4163 4164 4165 4166
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
4167
		.private = CGROUP_FILE_TASKS,
4168
		.write = cgroup_tasks_write,
4169 4170 4171 4172 4173 4174 4175 4176
		.mode = S_IRUGO | S_IWUSR,
	},
	{
		.name = "notify_on_release",
		.flags = CFTYPE_INSANE,
		.read_u64 = cgroup_read_notify_on_release,
		.write_u64 = cgroup_write_notify_on_release,
	},
4177 4178
	{
		.name = "release_agent",
4179
		.flags = CFTYPE_INSANE | CFTYPE_ONLY_ON_ROOT,
4180
		.seq_show = cgroup_release_agent_show,
4181
		.write = cgroup_release_agent_write,
4182
		.max_write_len = PATH_MAX - 1,
4183
	},
T
Tejun Heo 已提交
4184
	{ }	/* terminate */
4185 4186
};

4187
/**
4188
 * cgroup_populate_dir - create subsys files in a cgroup directory
4189 4190
 * @cgrp: target cgroup
 * @subsys_mask: mask of the subsystem ids whose files should be added
4191 4192
 *
 * On failure, no file is added.
4193
 */
4194
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask)
4195 4196
{
	struct cgroup_subsys *ss;
4197
	int i, ret = 0;
4198

4199
	/* process cftsets of each subsystem */
4200
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
4201
		struct cftype *cfts;
4202

4203
		if (!(subsys_mask & (1 << i)))
4204
			continue;
4205

T
Tejun Heo 已提交
4206 4207
		list_for_each_entry(cfts, &ss->cfts, node) {
			ret = cgroup_addrm_files(cgrp, cfts, true);
4208 4209 4210
			if (ret < 0)
				goto err;
		}
4211 4212
	}
	return 0;
4213 4214 4215
err:
	cgroup_clear_dir(cgrp, subsys_mask);
	return ret;
4216 4217
}

4218 4219 4220 4221 4222 4223 4224
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4225 4226 4227
 *    and thus css_tryget_online() is guaranteed to fail, the css can be
 *    offlined by invoking offline_css().  After offlining, the base ref is
 *    put.  Implemented in css_killed_work_fn().
4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
4240
static void css_free_work_fn(struct work_struct *work)
4241 4242
{
	struct cgroup_subsys_state *css =
4243
		container_of(work, struct cgroup_subsys_state, destroy_work);
4244
	struct cgroup *cgrp = css->cgroup;
4245

4246 4247 4248 4249
	if (css->ss) {
		/* css free path */
		if (css->parent)
			css_put(css->parent);
4250

4251 4252 4253 4254 4255 4256 4257
		css->ss->css_free(css);
		cgroup_put(cgrp);
	} else {
		/* cgroup free path */
		atomic_dec(&cgrp->root->nr_cgrps);
		cgroup_pidlist_destroy_all(cgrp);

T
Tejun Heo 已提交
4258
		if (cgroup_parent(cgrp)) {
4259 4260 4261 4262 4263 4264
			/*
			 * We get a ref to the parent, and put the ref when
			 * this cgroup is being freed, so it's guaranteed
			 * that the parent won't be destroyed before its
			 * children.
			 */
T
Tejun Heo 已提交
4265
			cgroup_put(cgroup_parent(cgrp));
4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276
			kernfs_put(cgrp->kn);
			kfree(cgrp);
		} else {
			/*
			 * This is root cgroup's refcnt reaching zero,
			 * which indicates that the root should be
			 * released.
			 */
			cgroup_destroy_root(cgrp->root);
		}
	}
4277 4278
}

4279
static void css_free_rcu_fn(struct rcu_head *rcu_head)
4280 4281
{
	struct cgroup_subsys_state *css =
4282
		container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4283

4284
	INIT_WORK(&css->destroy_work, css_free_work_fn);
4285
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4286 4287
}

4288
static void css_release_work_fn(struct work_struct *work)
4289 4290
{
	struct cgroup_subsys_state *css =
4291
		container_of(work, struct cgroup_subsys_state, destroy_work);
4292
	struct cgroup_subsys *ss = css->ss;
4293
	struct cgroup *cgrp = css->cgroup;
4294

4295 4296
	mutex_lock(&cgroup_mutex);

4297
	css->flags |= CSS_RELEASED;
4298 4299
	list_del_rcu(&css->sibling);

4300 4301 4302 4303 4304 4305 4306 4307
	if (ss) {
		/* css release path */
		cgroup_idr_remove(&ss->css_idr, css->id);
	} else {
		/* cgroup release path */
		cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
		cgrp->id = -1;
	}
4308

4309 4310
	mutex_unlock(&cgroup_mutex);

4311
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4312 4313 4314 4315 4316 4317 4318
}

static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4319 4320
	INIT_WORK(&css->destroy_work, css_release_work_fn);
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4321 4322
}

4323 4324
static void init_and_link_css(struct cgroup_subsys_state *css,
			      struct cgroup_subsys *ss, struct cgroup *cgrp)
4325
{
4326 4327
	lockdep_assert_held(&cgroup_mutex);

4328 4329
	cgroup_get(cgrp);

4330
	memset(css, 0, sizeof(*css));
4331
	css->cgroup = cgrp;
4332
	css->ss = ss;
4333 4334
	INIT_LIST_HEAD(&css->sibling);
	INIT_LIST_HEAD(&css->children);
4335
	css->serial_nr = css_serial_nr_next++;
4336

T
Tejun Heo 已提交
4337 4338
	if (cgroup_parent(cgrp)) {
		css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4339 4340
		css_get(css->parent);
	}
4341

4342
	BUG_ON(cgroup_css(cgrp, ss));
4343 4344
}

4345
/* invoke ->css_online() on a new CSS and mark it online if successful */
4346
static int online_css(struct cgroup_subsys_state *css)
4347
{
4348
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
4349 4350
	int ret = 0;

4351 4352
	lockdep_assert_held(&cgroup_mutex);

4353
	if (ss->css_online)
4354
		ret = ss->css_online(css);
4355
	if (!ret) {
4356
		css->flags |= CSS_ONLINE;
4357
		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4358
	}
T
Tejun Heo 已提交
4359
	return ret;
4360 4361
}

4362
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4363
static void offline_css(struct cgroup_subsys_state *css)
4364
{
4365
	struct cgroup_subsys *ss = css->ss;
4366 4367 4368 4369 4370 4371

	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

4372
	if (ss->css_offline)
4373
		ss->css_offline(css);
4374

4375
	css->flags &= ~CSS_ONLINE;
4376
	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4377 4378

	wake_up_all(&css->cgroup->offline_waitq);
4379 4380
}

4381 4382 4383 4384
/**
 * create_css - create a cgroup_subsys_state
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
4385
 * @visible: whether to create control knobs for the new css or not
4386 4387
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
4388 4389
 * css is online and installed in @cgrp with all interface files created if
 * @visible.  Returns 0 on success, -errno on failure.
4390
 */
4391 4392
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
		      bool visible)
4393
{
T
Tejun Heo 已提交
4394
	struct cgroup *parent = cgroup_parent(cgrp);
4395
	struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4396 4397 4398 4399 4400
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgroup_mutex);

4401
	css = ss->css_alloc(parent_css);
4402 4403 4404
	if (IS_ERR(css))
		return PTR_ERR(css);

4405
	init_and_link_css(css, ss, cgrp);
4406

4407 4408
	err = percpu_ref_init(&css->refcnt, css_release);
	if (err)
4409
		goto err_free_css;
4410

4411 4412 4413 4414
	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT);
	if (err < 0)
		goto err_free_percpu_ref;
	css->id = err;
4415

4416 4417 4418 4419 4420
	if (visible) {
		err = cgroup_populate_dir(cgrp, 1 << ss->id);
		if (err)
			goto err_free_id;
	}
4421 4422

	/* @css is ready to be brought online now, make it visible */
4423
	list_add_tail_rcu(&css->sibling, &parent_css->children);
4424
	cgroup_idr_replace(&ss->css_idr, css, css->id);
4425 4426 4427

	err = online_css(css);
	if (err)
4428
		goto err_list_del;
4429

4430
	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
T
Tejun Heo 已提交
4431
	    cgroup_parent(parent)) {
4432
		pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4433
			current->comm, current->pid, ss->name);
4434
		if (!strcmp(ss->name, "memory"))
4435
			pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4436 4437 4438 4439 4440
		ss->warned_broken_hierarchy = true;
	}

	return 0;

4441 4442
err_list_del:
	list_del_rcu(&css->sibling);
4443
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4444 4445
err_free_id:
	cgroup_idr_remove(&ss->css_idr, css->id);
4446
err_free_percpu_ref:
4447
	percpu_ref_cancel_init(&css->refcnt);
4448
err_free_css:
4449
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4450 4451 4452
	return err;
}

4453 4454
static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
			umode_t mode)
4455
{
4456 4457
	struct cgroup *parent, *cgrp;
	struct cgroup_root *root;
4458
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
4459
	struct kernfs_node *kn;
4460
	int ssid, ret;
4461

4462 4463 4464 4465
	parent = cgroup_kn_lock_live(parent_kn);
	if (!parent)
		return -ENODEV;
	root = parent->root;
4466

T
Tejun Heo 已提交
4467
	/* allocate the cgroup and its ID, 0 is reserved for the root */
4468
	cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
T
Tejun Heo 已提交
4469 4470 4471
	if (!cgrp) {
		ret = -ENOMEM;
		goto out_unlock;
4472 4473
	}

4474 4475 4476 4477
	ret = percpu_ref_init(&cgrp->self.refcnt, css_release);
	if (ret)
		goto out_free_cgrp;

4478 4479 4480 4481
	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
4482
	cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT);
4483
	if (cgrp->id < 0) {
T
Tejun Heo 已提交
4484
		ret = -ENOMEM;
4485
		goto out_cancel_ref;
4486 4487
	}

4488
	init_cgroup_housekeeping(cgrp);
4489

4490
	cgrp->self.parent = &parent->self;
T
Tejun Heo 已提交
4491
	cgrp->root = root;
4492

4493 4494 4495
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

4496 4497
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4498

T
Tejun Heo 已提交
4499
	/* create the directory */
T
Tejun Heo 已提交
4500
	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
T
Tejun Heo 已提交
4501
	if (IS_ERR(kn)) {
T
Tejun Heo 已提交
4502 4503
		ret = PTR_ERR(kn);
		goto out_free_id;
T
Tejun Heo 已提交
4504 4505
	}
	cgrp->kn = kn;
4506

4507
	/*
4508 4509
	 * This extra ref will be put in cgroup_free_fn() and guarantees
	 * that @cgrp->kn is always accessible.
4510
	 */
4511
	kernfs_get(kn);
4512

4513
	cgrp->self.serial_nr = css_serial_nr_next++;
4514

4515
	/* allocation complete, commit to creation */
4516
	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4517
	atomic_inc(&root->nr_cgrps);
4518
	cgroup_get(parent);
4519

4520 4521 4522 4523
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
4524
	cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4525

T
Tejun Heo 已提交
4526 4527 4528
	ret = cgroup_kn_set_ugid(kn);
	if (ret)
		goto out_destroy;
4529

T
Tejun Heo 已提交
4530 4531 4532
	ret = cgroup_addrm_files(cgrp, cgroup_base_files, true);
	if (ret)
		goto out_destroy;
4533

4534
	/* let's create and online css's */
T
Tejun Heo 已提交
4535
	for_each_subsys(ss, ssid) {
4536
		if (parent->child_subsys_mask & (1 << ssid)) {
4537 4538
			ret = create_css(cgrp, ss,
					 parent->subtree_control & (1 << ssid));
T
Tejun Heo 已提交
4539 4540
			if (ret)
				goto out_destroy;
T
Tejun Heo 已提交
4541
		}
4542
	}
4543

4544 4545
	/*
	 * On the default hierarchy, a child doesn't automatically inherit
4546
	 * subtree_control from the parent.  Each is configured manually.
4547
	 */
4548 4549 4550 4551
	if (!cgroup_on_dfl(cgrp)) {
		cgrp->subtree_control = parent->subtree_control;
		cgroup_refresh_child_subsys_mask(cgrp);
	}
T
Tejun Heo 已提交
4552 4553

	kernfs_activate(kn);
4554

T
Tejun Heo 已提交
4555 4556
	ret = 0;
	goto out_unlock;
4557

T
Tejun Heo 已提交
4558
out_free_id:
4559
	cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
4560 4561
out_cancel_ref:
	percpu_ref_cancel_init(&cgrp->self.refcnt);
T
Tejun Heo 已提交
4562
out_free_cgrp:
4563
	kfree(cgrp);
T
Tejun Heo 已提交
4564
out_unlock:
4565
	cgroup_kn_unlock(parent_kn);
T
Tejun Heo 已提交
4566
	return ret;
4567

T
Tejun Heo 已提交
4568
out_destroy:
4569
	cgroup_destroy_locked(cgrp);
T
Tejun Heo 已提交
4570
	goto out_unlock;
4571 4572
}

4573 4574
/*
 * This is called when the refcnt of a css is confirmed to be killed.
4575 4576
 * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
 * initate destruction and put the css ref from kill_css().
4577 4578
 */
static void css_killed_work_fn(struct work_struct *work)
4579
{
4580 4581
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
4582

4583
	mutex_lock(&cgroup_mutex);
4584
	offline_css(css);
4585
	mutex_unlock(&cgroup_mutex);
4586 4587

	css_put(css);
4588 4589
}

4590 4591
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
4592 4593 4594 4595
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4596
	INIT_WORK(&css->destroy_work, css_killed_work_fn);
4597
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4598 4599
}

4600 4601 4602 4603 4604 4605
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
4606 4607
 * asynchronously once css_tryget_online() is guaranteed to fail and when
 * the reference count reaches zero, @css will be released.
4608 4609
 */
static void kill_css(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
4610
{
4611
	lockdep_assert_held(&cgroup_mutex);
4612

T
Tejun Heo 已提交
4613 4614 4615 4616
	/*
	 * This must happen before css is disassociated with its cgroup.
	 * See seq_css() for details.
	 */
4617
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4618

T
Tejun Heo 已提交
4619 4620 4621 4622 4623 4624 4625 4626 4627
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
4628
	 * css_tryget_online().  We can't simply call percpu_ref_kill() and
T
Tejun Heo 已提交
4629 4630 4631 4632 4633 4634 4635
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4636 4637 4638 4639 4640 4641 4642 4643
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
4644 4645 4646
 * guarantee that css_tryget_online() won't succeed by the time
 * ->css_offline() is invoked.  To satisfy all the requirements,
 * destruction is implemented in the following two steps.
4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
4662 4663
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4664
{
T
Tejun Heo 已提交
4665
	struct cgroup_subsys_state *css;
4666
	bool empty;
T
Tejun Heo 已提交
4667
	int ssid;
4668

4669 4670
	lockdep_assert_held(&cgroup_mutex);

4671
	/*
4672
	 * css_set_rwsem synchronizes access to ->cset_links and prevents
4673
	 * @cgrp from being removed while put_css_set() is in progress.
4674
	 */
4675
	down_read(&css_set_rwsem);
4676
	empty = list_empty(&cgrp->cset_links);
4677
	up_read(&css_set_rwsem);
4678
	if (!empty)
4679
		return -EBUSY;
L
Li Zefan 已提交
4680

4681
	/*
4682 4683 4684
	 * Make sure there's no live children.  We can't test emptiness of
	 * ->self.children as dead children linger on it while being
	 * drained; otherwise, "rmdir parent/child parent" may fail.
4685
	 */
4686
	if (css_has_online_children(&cgrp->self))
4687 4688
		return -EBUSY;

4689 4690
	/*
	 * Mark @cgrp dead.  This prevents further task migration and child
4691
	 * creation by disabling cgroup_lock_live_group().
4692
	 */
4693
	cgrp->self.flags &= ~CSS_ONLINE;
4694

4695
	/* initiate massacre of all css's */
T
Tejun Heo 已提交
4696 4697
	for_each_css(css, ssid, cgrp)
		kill_css(css);
4698

4699
	/* CSS_ONLINE is clear, remove from ->release_list for the last time */
4700 4701 4702 4703 4704 4705
	raw_spin_lock(&release_list_lock);
	if (!list_empty(&cgrp->release_list))
		list_del_init(&cgrp->release_list);
	raw_spin_unlock(&release_list_lock);

	/*
4706 4707
	 * Remove @cgrp directory along with the base files.  @cgrp has an
	 * extra ref on its kn.
4708
	 */
4709
	kernfs_remove(cgrp->kn);
4710

T
Tejun Heo 已提交
4711 4712
	set_bit(CGRP_RELEASABLE, &cgroup_parent(cgrp)->flags);
	check_for_release(cgroup_parent(cgrp));
T
Tejun Heo 已提交
4713

4714
	/* put the base reference */
4715
	percpu_ref_kill(&cgrp->self.refcnt);
4716

4717 4718 4719
	return 0;
};

T
Tejun Heo 已提交
4720
static int cgroup_rmdir(struct kernfs_node *kn)
4721
{
4722
	struct cgroup *cgrp;
T
Tejun Heo 已提交
4723
	int ret = 0;
4724

4725 4726 4727 4728
	cgrp = cgroup_kn_lock_live(kn);
	if (!cgrp)
		return 0;
	cgroup_get(cgrp);	/* for @kn->priv clearing */
4729

4730
	ret = cgroup_destroy_locked(cgrp);
4731

4732
	cgroup_kn_unlock(kn);
4733 4734

	/*
4735 4736 4737 4738 4739
	 * There are two control paths which try to determine cgroup from
	 * dentry without going through kernfs - cgroupstats_build() and
	 * css_tryget_online_from_dir().  Those are supported by RCU
	 * protecting clearing of cgrp->kn->priv backpointer, which should
	 * happen after all files under it have been removed.
4740
	 */
4741 4742
	if (!ret)
		RCU_INIT_POINTER(*(void __rcu __force **)&kn->priv, NULL);
4743

T
Tejun Heo 已提交
4744
	cgroup_put(cgrp);
4745
	return ret;
4746 4747
}

T
Tejun Heo 已提交
4748 4749 4750 4751 4752 4753 4754 4755
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
	.remount_fs		= cgroup_remount,
	.show_options		= cgroup_show_options,
	.mkdir			= cgroup_mkdir,
	.rmdir			= cgroup_rmdir,
	.rename			= cgroup_rename,
};

4756
static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
4757 4758
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
4759 4760

	printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4761

4762 4763
	mutex_lock(&cgroup_mutex);

4764
	idr_init(&ss->css_idr);
T
Tejun Heo 已提交
4765
	INIT_LIST_HEAD(&ss->cfts);
4766

4767 4768 4769
	/* Create the root cgroup state for this subsystem */
	ss->root = &cgrp_dfl_root;
	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
4770 4771
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
4772
	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
4773 4774 4775 4776 4777 4778 4779

	/*
	 * Root csses are never destroyed and we can't initialize
	 * percpu_ref during early init.  Disable refcnting.
	 */
	css->flags |= CSS_NO_REF;

4780
	if (early) {
4781
		/* allocation can't be done safely during early init */
4782 4783 4784 4785 4786
		css->id = 1;
	} else {
		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
		BUG_ON(css->id < 0);
	}
4787

L
Li Zefan 已提交
4788
	/* Update the init_css_set to contain a subsys
4789
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
4790
	 * newly registered, all tasks and hence the
4791
	 * init_css_set is in the subsystem's root cgroup. */
4792
	init_css_set.subsys[ss->id] = css;
4793 4794 4795

	need_forkexit_callback |= ss->fork || ss->exit;

L
Li Zefan 已提交
4796 4797 4798 4799 4800
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

4801
	BUG_ON(online_css(css));
4802

B
Ben Blum 已提交
4803 4804 4805
	mutex_unlock(&cgroup_mutex);
}

4806
/**
L
Li Zefan 已提交
4807 4808 4809 4810
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
4811 4812 4813
 */
int __init cgroup_init_early(void)
{
T
Tejun Heo 已提交
4814 4815
	static struct cgroup_sb_opts __initdata opts =
		{ .flags = CGRP_ROOT_SANE_BEHAVIOR };
4816
	struct cgroup_subsys *ss;
4817
	int i;
4818

4819
	init_cgroup_root(&cgrp_dfl_root, &opts);
4820 4821
	cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;

4822
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4823

T
Tejun Heo 已提交
4824
	for_each_subsys(ss, i) {
4825
		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
4826 4827
		     "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
4828
		     ss->id, ss->name);
4829 4830 4831
		WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
		     "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);

4832
		ss->id = i;
4833
		ss->name = cgroup_subsys_name[i];
4834 4835

		if (ss->early_init)
4836
			cgroup_init_subsys(ss, true);
4837 4838 4839 4840 4841
	}
	return 0;
}

/**
L
Li Zefan 已提交
4842 4843 4844 4845
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
4846 4847 4848
 */
int __init cgroup_init(void)
{
4849
	struct cgroup_subsys *ss;
4850
	unsigned long key;
4851
	int ssid, err;
4852

T
Tejun Heo 已提交
4853
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
4854

T
Tejun Heo 已提交
4855 4856
	mutex_lock(&cgroup_mutex);

4857 4858 4859 4860
	/* Add init_css_set to the hash table */
	key = css_set_hash(init_css_set.subsys);
	hash_add(css_set_table, &init_css_set.hlist, key);

4861
	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4862

T
Tejun Heo 已提交
4863 4864
	mutex_unlock(&cgroup_mutex);

4865
	for_each_subsys(ss, ssid) {
4866 4867 4868 4869 4870 4871 4872 4873 4874 4875
		if (ss->early_init) {
			struct cgroup_subsys_state *css =
				init_css_set.subsys[ss->id];

			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
						   GFP_KERNEL);
			BUG_ON(css->id < 0);
		} else {
			cgroup_init_subsys(ss, false);
		}
4876

T
Tejun Heo 已提交
4877 4878
		list_add_tail(&init_css_set.e_cset_node[ssid],
			      &cgrp_dfl_root.cgrp.e_csets[ssid]);
4879 4880

		/*
4881 4882 4883
		 * Setting dfl_root subsys_mask needs to consider the
		 * disabled flag and cftype registration needs kmalloc,
		 * both of which aren't available during early_init.
4884
		 */
4885 4886
		if (!ss->disabled) {
			cgrp_dfl_root.subsys_mask |= 1 << ss->id;
4887
			WARN_ON(cgroup_add_cftypes(ss, ss->base_cftypes));
4888
		}
4889 4890 4891
	}

	cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
T
Tejun Heo 已提交
4892 4893
	if (!cgroup_kobj)
		return -ENOMEM;
4894

4895
	err = register_filesystem(&cgroup_fs_type);
4896 4897
	if (err < 0) {
		kobject_put(cgroup_kobj);
T
Tejun Heo 已提交
4898
		return err;
4899
	}
4900

L
Li Zefan 已提交
4901
	proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
T
Tejun Heo 已提交
4902
	return 0;
4903
}
4904

4905 4906 4907 4908 4909
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
4910
	 * Use 1 for @max_active.
4911 4912 4913 4914
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
4915
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
4916
	BUG_ON(!cgroup_destroy_wq);
4917 4918 4919 4920 4921 4922 4923 4924 4925

	/*
	 * Used to destroy pidlists and separate to serve as flush domain.
	 * Cap @max_active to 1 too.
	 */
	cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
						    0, 1);
	BUG_ON(!cgroup_pidlist_destroy_wq);

4926 4927 4928 4929
	return 0;
}
core_initcall(cgroup_wq_init);

4930 4931 4932 4933 4934 4935 4936
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 */

/* TODO: Use a proper seq_file iterator */
4937
int proc_cgroup_show(struct seq_file *m, void *v)
4938 4939 4940
{
	struct pid *pid;
	struct task_struct *tsk;
T
Tejun Heo 已提交
4941
	char *buf, *path;
4942
	int retval;
4943
	struct cgroup_root *root;
4944 4945

	retval = -ENOMEM;
T
Tejun Heo 已提交
4946
	buf = kmalloc(PATH_MAX, GFP_KERNEL);
4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958
	if (!buf)
		goto out;

	retval = -ESRCH;
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
	if (!tsk)
		goto out_free;

	retval = 0;

	mutex_lock(&cgroup_mutex);
4959
	down_read(&css_set_rwsem);
4960

4961
	for_each_root(root) {
4962
		struct cgroup_subsys *ss;
4963
		struct cgroup *cgrp;
T
Tejun Heo 已提交
4964
		int ssid, count = 0;
4965

T
Tejun Heo 已提交
4966
		if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
4967 4968
			continue;

4969
		seq_printf(m, "%d:", root->hierarchy_id);
T
Tejun Heo 已提交
4970
		for_each_subsys(ss, ssid)
4971
			if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
4972
				seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
4973 4974 4975
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
4976
		seq_putc(m, ':');
4977
		cgrp = task_cgroup_from_root(tsk, root);
T
Tejun Heo 已提交
4978 4979 4980
		path = cgroup_path(cgrp, buf, PATH_MAX);
		if (!path) {
			retval = -ENAMETOOLONG;
4981
			goto out_unlock;
T
Tejun Heo 已提交
4982 4983
		}
		seq_puts(m, path);
4984 4985 4986 4987
		seq_putc(m, '\n');
	}

out_unlock:
4988
	up_read(&css_set_rwsem);
4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999
	mutex_unlock(&cgroup_mutex);
	put_task_struct(tsk);
out_free:
	kfree(buf);
out:
	return retval;
}

/* Display information about each subsystem and each hierarchy */
static int proc_cgroupstats_show(struct seq_file *m, void *v)
{
5000
	struct cgroup_subsys *ss;
5001 5002
	int i;

5003
	seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
B
Ben Blum 已提交
5004 5005 5006 5007 5008
	/*
	 * ideally we don't want subsystems moving around while we do this.
	 * cgroup_mutex is also necessary to guarantee an atomic snapshot of
	 * subsys/hierarchy state.
	 */
5009
	mutex_lock(&cgroup_mutex);
5010 5011

	for_each_subsys(ss, i)
5012 5013
		seq_printf(m, "%s\t%d\t%d\t%d\n",
			   ss->name, ss->root->hierarchy_id,
5014
			   atomic_read(&ss->root->nr_cgrps), !ss->disabled);
5015

5016 5017 5018 5019 5020 5021
	mutex_unlock(&cgroup_mutex);
	return 0;
}

static int cgroupstats_open(struct inode *inode, struct file *file)
{
A
Al Viro 已提交
5022
	return single_open(file, proc_cgroupstats_show, NULL);
5023 5024
}

5025
static const struct file_operations proc_cgroupstats_operations = {
5026 5027 5028 5029 5030 5031
	.open = cgroupstats_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

5032
/**
5033
 * cgroup_fork - initialize cgroup related fields during copy_process()
L
Li Zefan 已提交
5034
 * @child: pointer to task_struct of forking parent process.
5035
 *
5036 5037 5038
 * A task is associated with the init_css_set until cgroup_post_fork()
 * attaches it to the parent's css_set.  Empty cg_list indicates that
 * @child isn't holding reference to its css_set.
5039 5040 5041
 */
void cgroup_fork(struct task_struct *child)
{
5042
	RCU_INIT_POINTER(child->cgroups, &init_css_set);
5043
	INIT_LIST_HEAD(&child->cg_list);
5044 5045
}

5046
/**
L
Li Zefan 已提交
5047 5048 5049
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
5050 5051 5052
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
5053
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5054
 * list.
L
Li Zefan 已提交
5055
 */
5056 5057
void cgroup_post_fork(struct task_struct *child)
{
5058
	struct cgroup_subsys *ss;
5059 5060
	int i;

5061
	/*
5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080
	 * This may race against cgroup_enable_task_cg_links().  As that
	 * function sets use_task_css_set_links before grabbing
	 * tasklist_lock and we just went through tasklist_lock to add
	 * @child, it's guaranteed that either we see the set
	 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
	 * @child during its iteration.
	 *
	 * If we won the race, @child is associated with %current's
	 * css_set.  Grabbing css_set_rwsem guarantees both that the
	 * association is stable, and, on completion of the parent's
	 * migration, @child is visible in the source of migration or
	 * already in the destination cgroup.  This guarantee is necessary
	 * when implementing operations which need to migrate all tasks of
	 * a cgroup to another.
	 *
	 * Note that if we lose to cgroup_enable_task_cg_links(), @child
	 * will remain in init_css_set.  This is safe because all tasks are
	 * in the init_css_set before cg_links is enabled and there's no
	 * operation which transfers all tasks out of init_css_set.
5081
	 */
5082
	if (use_task_css_set_links) {
5083 5084
		struct css_set *cset;

5085
		down_write(&css_set_rwsem);
5086
		cset = task_css_set(current);
5087 5088 5089 5090 5091
		if (list_empty(&child->cg_list)) {
			rcu_assign_pointer(child->cgroups, cset);
			list_add(&child->cg_list, &cset->tasks);
			get_css_set(cset);
		}
5092
		up_write(&css_set_rwsem);
5093
	}
5094 5095 5096 5097 5098 5099 5100

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
5101
		for_each_subsys(ss, i)
5102 5103 5104
			if (ss->fork)
				ss->fork(child);
	}
5105
}
5106

5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
5119 5120 5121 5122 5123
 * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
 * call cgroup_exit() while the task is still competent to handle
 * notify_on_release(), then leave the task attached to the root cgroup in
 * each hierarchy for the remainder of its exit.  No need to bother with
 * init_css_set refcnting.  init_css_set never goes away and we can't race
5124
 * with migration path - PF_EXITING is visible to migration path.
5125
 */
5126
void cgroup_exit(struct task_struct *tsk)
5127
{
5128
	struct cgroup_subsys *ss;
5129
	struct css_set *cset;
5130
	bool put_cset = false;
5131
	int i;
5132 5133

	/*
5134 5135
	 * Unlink from @tsk from its css_set.  As migration path can't race
	 * with us, we can check cg_list without grabbing css_set_rwsem.
5136 5137
	 */
	if (!list_empty(&tsk->cg_list)) {
5138
		down_write(&css_set_rwsem);
5139
		list_del_init(&tsk->cg_list);
5140
		up_write(&css_set_rwsem);
5141
		put_cset = true;
5142 5143
	}

5144
	/* Reassign the task to the init_css_set. */
5145 5146
	cset = task_css_set(tsk);
	RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5147

5148
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
5149 5150
		/* see cgroup_post_fork() for details */
		for_each_subsys(ss, i) {
5151
			if (ss->exit) {
5152 5153
				struct cgroup_subsys_state *old_css = cset->subsys[i];
				struct cgroup_subsys_state *css = task_css(tsk, i);
5154

5155
				ss->exit(css, old_css, tsk);
5156 5157 5158 5159
			}
		}
	}

5160 5161
	if (put_cset)
		put_css_set(cset, true);
5162
}
5163

5164
static void check_for_release(struct cgroup *cgrp)
5165
{
5166 5167
	if (cgroup_is_releasable(cgrp) && list_empty(&cgrp->cset_links) &&
	    !css_has_online_children(&cgrp->self)) {
5168 5169
		/*
		 * Control Group is currently removeable. If it's not
5170
		 * already queued for a userspace notification, queue
5171 5172
		 * it now
		 */
5173
		int need_schedule_work = 0;
5174

5175
		raw_spin_lock(&release_list_lock);
5176
		if (!cgroup_is_dead(cgrp) &&
5177 5178
		    list_empty(&cgrp->release_list)) {
			list_add(&cgrp->release_list, &release_list);
5179 5180
			need_schedule_work = 1;
		}
5181
		raw_spin_unlock(&release_list_lock);
5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213
		if (need_schedule_work)
			schedule_work(&release_agent_work);
	}
}

/*
 * Notify userspace when a cgroup is released, by running the
 * configured release agent with the name of the cgroup (path
 * relative to the root of cgroup file system) as the argument.
 *
 * Most likely, this user command will try to rmdir this cgroup.
 *
 * This races with the possibility that some other task will be
 * attached to this cgroup before it is removed, or that some other
 * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
 * unused, and this cgroup will be reprieved from its death sentence,
 * to continue to serve a useful existence.  Next time it's released,
 * we will get notified again, if it still has 'notify_on_release' set.
 *
 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
 * means only wait until the task is successfully execve()'d.  The
 * separate release agent task is forked by call_usermodehelper(),
 * then control in this thread returns here, without waiting for the
 * release agent task.  We don't bother to wait because the caller of
 * this routine has no use for the exit status of the release agent
 * task, so no sense holding our caller up for that.
 */
static void cgroup_release_agent(struct work_struct *work)
{
	BUG_ON(work != &release_agent_work);
	mutex_lock(&cgroup_mutex);
5214
	raw_spin_lock(&release_list_lock);
5215 5216 5217
	while (!list_empty(&release_list)) {
		char *argv[3], *envp[3];
		int i;
T
Tejun Heo 已提交
5218
		char *pathbuf = NULL, *agentbuf = NULL, *path;
5219
		struct cgroup *cgrp = list_entry(release_list.next,
5220 5221
						    struct cgroup,
						    release_list);
5222
		list_del_init(&cgrp->release_list);
5223
		raw_spin_unlock(&release_list_lock);
T
Tejun Heo 已提交
5224
		pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5225 5226
		if (!pathbuf)
			goto continue_free;
T
Tejun Heo 已提交
5227 5228
		path = cgroup_path(cgrp, pathbuf, PATH_MAX);
		if (!path)
5229 5230 5231 5232
			goto continue_free;
		agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
		if (!agentbuf)
			goto continue_free;
5233 5234

		i = 0;
5235
		argv[i++] = agentbuf;
T
Tejun Heo 已提交
5236
		argv[i++] = path;
5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250
		argv[i] = NULL;

		i = 0;
		/* minimal command environment */
		envp[i++] = "HOME=/";
		envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
		envp[i] = NULL;

		/* Drop the lock while we invoke the usermode helper,
		 * since the exec could involve hitting disk and hence
		 * be a slow process */
		mutex_unlock(&cgroup_mutex);
		call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
		mutex_lock(&cgroup_mutex);
5251 5252 5253
 continue_free:
		kfree(pathbuf);
		kfree(agentbuf);
5254
		raw_spin_lock(&release_list_lock);
5255
	}
5256
	raw_spin_unlock(&release_list_lock);
5257 5258
	mutex_unlock(&cgroup_mutex);
}
5259 5260 5261

static int __init cgroup_disable(char *str)
{
5262
	struct cgroup_subsys *ss;
5263
	char *token;
5264
	int i;
5265 5266 5267 5268

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
5269

T
Tejun Heo 已提交
5270
		for_each_subsys(ss, i) {
5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281
			if (!strcmp(token, ss->name)) {
				ss->disabled = 1;
				printk(KERN_INFO "Disabling %s control group"
					" subsystem\n", ss->name);
				break;
			}
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
5282

5283
/**
5284
 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5285 5286
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
5287
 *
5288 5289 5290
 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
 * to get the corresponding css and return it.  If such css doesn't exist
 * or can't be pinned, an ERR_PTR value is returned.
S
Stephane Eranian 已提交
5291
 */
5292 5293
struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
						       struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
5294
{
T
Tejun Heo 已提交
5295 5296
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
	struct cgroup_subsys_state *css = NULL;
S
Stephane Eranian 已提交
5297 5298
	struct cgroup *cgrp;

5299
	/* is @dentry a cgroup dir? */
T
Tejun Heo 已提交
5300 5301
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
S
Stephane Eranian 已提交
5302 5303
		return ERR_PTR(-EBADF);

5304 5305
	rcu_read_lock();

T
Tejun Heo 已提交
5306 5307 5308
	/*
	 * This path doesn't originate from kernfs and @kn could already
	 * have been or be removed at any point.  @kn->priv is RCU
5309
	 * protected for this access.  See cgroup_rmdir() for details.
T
Tejun Heo 已提交
5310 5311 5312 5313
	 */
	cgrp = rcu_dereference(kn->priv);
	if (cgrp)
		css = cgroup_css(cgrp, ss);
5314

5315
	if (!css || !css_tryget_online(css))
5316 5317 5318 5319
		css = ERR_PTR(-ENOENT);

	rcu_read_unlock();
	return css;
S
Stephane Eranian 已提交
5320 5321
}

5322 5323 5324 5325 5326 5327 5328 5329 5330 5331
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
5332
	WARN_ON_ONCE(!rcu_read_lock_held());
5333
	return idr_find(&ss->css_idr, id);
S
Stephane Eranian 已提交
5334 5335
}

5336
#ifdef CONFIG_CGROUP_DEBUG
5337 5338
static struct cgroup_subsys_state *
debug_css_alloc(struct cgroup_subsys_state *parent_css)
5339 5340 5341 5342 5343 5344 5345 5346 5347
{
	struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);

	if (!css)
		return ERR_PTR(-ENOMEM);

	return css;
}

5348
static void debug_css_free(struct cgroup_subsys_state *css)
5349
{
5350
	kfree(css);
5351 5352
}

5353 5354
static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5355
{
5356
	return cgroup_task_count(css->cgroup);
5357 5358
}

5359 5360
static u64 current_css_set_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5361 5362 5363 5364
{
	return (u64)(unsigned long)current->cgroups;
}

5365
static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
L
Li Zefan 已提交
5366
					 struct cftype *cft)
5367 5368 5369 5370
{
	u64 count;

	rcu_read_lock();
5371
	count = atomic_read(&task_css_set(current)->refcount);
5372 5373 5374 5375
	rcu_read_unlock();
	return count;
}

5376
static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5377
{
5378
	struct cgrp_cset_link *link;
5379
	struct css_set *cset;
T
Tejun Heo 已提交
5380 5381 5382 5383 5384
	char *name_buf;

	name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
	if (!name_buf)
		return -ENOMEM;
5385

5386
	down_read(&css_set_rwsem);
5387
	rcu_read_lock();
5388
	cset = rcu_dereference(current->cgroups);
5389
	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5390 5391
		struct cgroup *c = link->cgrp;

T
Tejun Heo 已提交
5392
		cgroup_name(c, name_buf, NAME_MAX + 1);
5393
		seq_printf(seq, "Root %d group %s\n",
T
Tejun Heo 已提交
5394
			   c->root->hierarchy_id, name_buf);
5395 5396
	}
	rcu_read_unlock();
5397
	up_read(&css_set_rwsem);
T
Tejun Heo 已提交
5398
	kfree(name_buf);
5399 5400 5401 5402
	return 0;
}

#define MAX_TASKS_SHOWN_PER_CSS 25
5403
static int cgroup_css_links_read(struct seq_file *seq, void *v)
5404
{
5405
	struct cgroup_subsys_state *css = seq_css(seq);
5406
	struct cgrp_cset_link *link;
5407

5408
	down_read(&css_set_rwsem);
5409
	list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5410
		struct css_set *cset = link->cset;
5411 5412
		struct task_struct *task;
		int count = 0;
T
Tejun Heo 已提交
5413

5414
		seq_printf(seq, "css_set %p\n", cset);
T
Tejun Heo 已提交
5415

5416
		list_for_each_entry(task, &cset->tasks, cg_list) {
T
Tejun Heo 已提交
5417 5418 5419 5420 5421 5422 5423 5424 5425
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
		}

		list_for_each_entry(task, &cset->mg_tasks, cg_list) {
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5426
		}
T
Tejun Heo 已提交
5427 5428 5429
		continue;
	overflow:
		seq_puts(seq, "  ...\n");
5430
	}
5431
	up_read(&css_set_rwsem);
5432 5433 5434
	return 0;
}

5435
static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5436
{
5437
	return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455
}

static struct cftype debug_files[] =  {
	{
		.name = "taskcount",
		.read_u64 = debug_taskcount_read,
	},

	{
		.name = "current_css_set",
		.read_u64 = current_css_set_read,
	},

	{
		.name = "current_css_set_refcount",
		.read_u64 = current_css_set_refcount_read,
	},

5456 5457
	{
		.name = "current_css_set_cg_links",
5458
		.seq_show = current_css_set_cg_links_read,
5459 5460 5461 5462
	},

	{
		.name = "cgroup_css_links",
5463
		.seq_show = cgroup_css_links_read,
5464 5465
	},

5466 5467 5468 5469 5470
	{
		.name = "releasable",
		.read_u64 = releasable_read,
	},

5471 5472
	{ }	/* terminate */
};
5473

5474
struct cgroup_subsys debug_cgrp_subsys = {
5475 5476
	.css_alloc = debug_css_alloc,
	.css_free = debug_css_free,
5477
	.base_cftypes = debug_files,
5478 5479
};
#endif /* CONFIG_CGROUP_DEBUG */