cgroup.c 148.9 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/cgroup.h>
32
#include <linux/cred.h>
33
#include <linux/ctype.h>
34
#include <linux/errno.h>
35
#include <linux/init_task.h>
36 37
#include <linux/kernel.h>
#include <linux/list.h>
38
#include <linux/magic.h>
39 40 41 42
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
43
#include <linux/proc_fs.h>
44 45 46 47
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
48
#include <linux/rwsem.h>
49
#include <linux/string.h>
50
#include <linux/sort.h>
51
#include <linux/kmod.h>
B
Balbir Singh 已提交
52 53
#include <linux/delayacct.h>
#include <linux/cgroupstats.h>
54
#include <linux/hashtable.h>
L
Li Zefan 已提交
55
#include <linux/pid_namespace.h>
56
#include <linux/idr.h>
57
#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
58
#include <linux/kthread.h>
T
Tejun Heo 已提交
59
#include <linux/delay.h>
B
Balbir Singh 已提交
60

A
Arun Sharma 已提交
61
#include <linux/atomic.h>
62

63 64 65 66 67 68 69 70
/*
 * pidlists linger the following amount before being destroyed.  The goal
 * is avoiding frequent destruction in the middle of consecutive read calls
 * Expiring in the middle is a performance problem not a correctness one.
 * 1 sec should be enough.
 */
#define CGROUP_PIDLIST_DESTROY_DELAY	HZ

T
Tejun Heo 已提交
71 72 73
#define CGROUP_FILE_NAME_MAX		(MAX_CGROUP_TYPE_NAMELEN +	\
					 MAX_CFTYPE_NAME + 2)

T
Tejun Heo 已提交
74 75 76 77
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
78 79
 * css_set_rwsem protects task->cgroups pointer, the list of css_set
 * objects, and the chain of tasks off each css_set.
T
Tejun Heo 已提交
80
 *
81 82
 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 * cgroup.h can use them for lockdep annotations.
T
Tejun Heo 已提交
83
 */
T
Tejun Heo 已提交
84 85
#ifdef CONFIG_PROVE_RCU
DEFINE_MUTEX(cgroup_mutex);
86 87 88
DECLARE_RWSEM(css_set_rwsem);
EXPORT_SYMBOL_GPL(cgroup_mutex);
EXPORT_SYMBOL_GPL(css_set_rwsem);
T
Tejun Heo 已提交
89
#else
90
static DEFINE_MUTEX(cgroup_mutex);
91
static DECLARE_RWSEM(css_set_rwsem);
T
Tejun Heo 已提交
92 93
#endif

94
/*
95 96
 * Protects cgroup_idr and css_idr so that IDs can be released without
 * grabbing cgroup_mutex.
97 98 99
 */
static DEFINE_SPINLOCK(cgroup_idr_lock);

100 101 102 103 104
/*
 * Protects cgroup_subsys->release_agent_path.  Modifying it also requires
 * cgroup_mutex.  Reading requires either cgroup_mutex or this spinlock.
 */
static DEFINE_SPINLOCK(release_agent_path_lock);
105

T
Tejun Heo 已提交
106
#define cgroup_assert_mutex_or_rcu_locked()				\
107 108
	rcu_lockdep_assert(rcu_read_lock_held() ||			\
			   lockdep_is_held(&cgroup_mutex),		\
T
Tejun Heo 已提交
109
			   "cgroup_mutex or RCU read lock required");
110

111 112 113 114 115 116 117 118
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

119 120 121 122 123 124
/*
 * pidlist destructions need to be flushed on cgroup destruction.  Use a
 * separate workqueue as flush domain.
 */
static struct workqueue_struct *cgroup_pidlist_destroy_wq;

T
Tejun Heo 已提交
125
/* generate an array of cgroup subsystem pointers */
126
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
T
Tejun Heo 已提交
127
static struct cgroup_subsys *cgroup_subsys[] = {
128 129
#include <linux/cgroup_subsys.h>
};
130 131 132 133 134
#undef SUBSYS

/* array of cgroup subsystem names */
#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
static const char *cgroup_subsys_name[] = {
135 136
#include <linux/cgroup_subsys.h>
};
137
#undef SUBSYS
138 139

/*
140
 * The default hierarchy, reserved for the subsystems that are otherwise
141 142
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
143
 */
T
Tejun Heo 已提交
144
struct cgroup_root cgrp_dfl_root;
145

T
Tejun Heo 已提交
146 147 148 149 150
/*
 * The default hierarchy always exists but is hidden until mounted for the
 * first time.  This is for backward compatibility.
 */
static bool cgrp_dfl_root_visible;
151

152 153 154 155 156 157
/*
 * Set by the boot param of the same name and makes subsystems with NULL
 * ->dfl_files to use ->legacy_files on the default hierarchy.
 */
static bool cgroup_legacy_files_on_dfl;

158 159 160 161 162 163 164
/* some controllers are not supported in the default hierarchy */
static const unsigned int cgrp_dfl_root_inhibit_ss_mask = 0
#ifdef CONFIG_CGROUP_DEBUG
	| (1 << debug_cgrp_id)
#endif
	;

165 166
/* The list of hierarchy roots */

167 168
static LIST_HEAD(cgroup_roots);
static int cgroup_root_count;
169

T
Tejun Heo 已提交
170
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
171
static DEFINE_IDR(cgroup_hierarchy_idr);
172

173
/*
174 175 176 177 178
 * Assign a monotonically increasing serial number to csses.  It guarantees
 * cgroups with bigger numbers are newer than those with smaller numbers.
 * Also, as csses are always appended to the parent's ->children list, it
 * guarantees that sibling csses are always sorted in the ascending serial
 * number order on the list.  Protected by cgroup_mutex.
179
 */
180
static u64 css_serial_nr_next = 1;
181

182
/* This flag indicates whether tasks in the fork and exit paths should
L
Li Zefan 已提交
183 184 185
 * check for fork/exit handlers to call. This avoids us having to do
 * extra work in the fork/exit path if none of the subsystems need to
 * be called.
186
 */
187
static int need_forkexit_callback __read_mostly;
188

189 190
static struct cftype cgroup_dfl_base_files[];
static struct cftype cgroup_legacy_base_files[];
191

192
static void cgroup_put(struct cgroup *cgrp);
193
static int rebind_subsystems(struct cgroup_root *dst_root,
194
			     unsigned int ss_mask);
195
static int cgroup_destroy_locked(struct cgroup *cgrp);
196 197
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
		      bool visible);
198
static void css_release(struct percpu_ref *ref);
199
static void kill_css(struct cgroup_subsys_state *css);
200 201
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add);
202
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
203

204 205 206 207 208 209 210
/* IDR wrappers which synchronize using cgroup_idr_lock */
static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
			    gfp_t gfp_mask)
{
	int ret;

	idr_preload(gfp_mask);
T
Tejun Heo 已提交
211
	spin_lock_bh(&cgroup_idr_lock);
212
	ret = idr_alloc(idr, ptr, start, end, gfp_mask);
T
Tejun Heo 已提交
213
	spin_unlock_bh(&cgroup_idr_lock);
214 215 216 217 218 219 220 221
	idr_preload_end();
	return ret;
}

static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
{
	void *ret;

T
Tejun Heo 已提交
222
	spin_lock_bh(&cgroup_idr_lock);
223
	ret = idr_replace(idr, ptr, id);
T
Tejun Heo 已提交
224
	spin_unlock_bh(&cgroup_idr_lock);
225 226 227 228 229
	return ret;
}

static void cgroup_idr_remove(struct idr *idr, int id)
{
T
Tejun Heo 已提交
230
	spin_lock_bh(&cgroup_idr_lock);
231
	idr_remove(idr, id);
T
Tejun Heo 已提交
232
	spin_unlock_bh(&cgroup_idr_lock);
233 234
}

T
Tejun Heo 已提交
235 236 237 238 239 240 241 242 243
static struct cgroup *cgroup_parent(struct cgroup *cgrp)
{
	struct cgroup_subsys_state *parent_css = cgrp->self.parent;

	if (parent_css)
		return container_of(parent_css, struct cgroup, self);
	return NULL;
}

T
Tejun Heo 已提交
244 245 246
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
247
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
T
Tejun Heo 已提交
248
 *
249 250 251 252 253
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
254 255
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
256
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
257
{
258
	if (ss)
259
		return rcu_dereference_check(cgrp->subsys[ss->id],
T
Tejun Heo 已提交
260
					lockdep_is_held(&cgroup_mutex));
261
	else
262
		return &cgrp->self;
T
Tejun Heo 已提交
263
}
264

265 266 267
/**
 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
268
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
269 270 271 272 273 274 275 276 277 278 279 280
 *
 * Similar to cgroup_css() but returns the effctive css, which is defined
 * as the matching css of the nearest ancestor including self which has @ss
 * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
 * function is guaranteed to return non-NULL css.
 */
static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
						struct cgroup_subsys *ss)
{
	lockdep_assert_held(&cgroup_mutex);

	if (!ss)
281
		return &cgrp->self;
282 283 284 285

	if (!(cgrp->root->subsys_mask & (1 << ss->id)))
		return NULL;

T
Tejun Heo 已提交
286 287 288
	while (cgroup_parent(cgrp) &&
	       !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id)))
		cgrp = cgroup_parent(cgrp);
289 290

	return cgroup_css(cgrp, ss);
T
Tejun Heo 已提交
291
}
292

293
/* convenient tests for these bits */
294
static inline bool cgroup_is_dead(const struct cgroup *cgrp)
295
{
296
	return !(cgrp->self.flags & CSS_ONLINE);
297 298
}

T
Tejun Heo 已提交
299
struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
300
{
T
Tejun Heo 已提交
301
	struct cgroup *cgrp = of->kn->parent->priv;
T
Tejun Heo 已提交
302
	struct cftype *cft = of_cft(of);
T
Tejun Heo 已提交
303 304 305 306 307 308 309 310 311 312 313 314

	/*
	 * This is open and unprotected implementation of cgroup_css().
	 * seq_css() is only called from a kernfs file operation which has
	 * an active reference on the file.  Because all the subsystem
	 * files are drained before a css is disassociated with a cgroup,
	 * the matching css from the cgroup's subsys table is guaranteed to
	 * be and stay valid until the enclosing operation is complete.
	 */
	if (cft->ss)
		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
	else
315
		return &cgrp->self;
316
}
T
Tejun Heo 已提交
317
EXPORT_SYMBOL_GPL(of_css);
318

319 320 321 322 323 324 325 326 327 328 329 330 331 332
/**
 * cgroup_is_descendant - test ancestry
 * @cgrp: the cgroup to be tested
 * @ancestor: possible ancestor of @cgrp
 *
 * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
 * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
 * and @ancestor are accessible.
 */
bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
{
	while (cgrp) {
		if (cgrp == ancestor)
			return true;
T
Tejun Heo 已提交
333
		cgrp = cgroup_parent(cgrp);
334 335 336
	}
	return false;
}
337

338
static int cgroup_is_releasable(const struct cgroup *cgrp)
339 340
{
	const int bits =
341 342 343
		(1 << CGRP_RELEASABLE) |
		(1 << CGRP_NOTIFY_ON_RELEASE);
	return (cgrp->flags & bits) == bits;
344 345
}

346
static int notify_on_release(const struct cgroup *cgrp)
347
{
348
	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
349 350
}

T
Tejun Heo 已提交
351 352 353 354 355 356
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
357
 * Should be called under cgroup_[tree_]mutex.
T
Tejun Heo 已提交
358 359 360 361 362 363 364 365
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

366 367 368 369 370 371 372 373 374 375 376 377 378 379
/**
 * for_each_e_css - iterate all effective css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_[tree_]mutex.
 */
#define for_each_e_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
			;						\
		else

380
/**
T
Tejun Heo 已提交
381
 * for_each_subsys - iterate all enabled cgroup subsystems
382
 * @ss: the iteration cursor
383
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
384
 */
385
#define for_each_subsys(ss, ssid)					\
T
Tejun Heo 已提交
386 387
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT &&		\
	     (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
388

389 390
/* iterate across the hierarchies */
#define for_each_root(root)						\
391
	list_for_each_entry((root), &cgroup_roots, root_list)
392

393 394
/* iterate over child cgrps, lock should be held throughout iteration */
#define cgroup_for_each_live_child(child, cgrp)				\
395
	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
T
Tejun Heo 已提交
396
		if (({ lockdep_assert_held(&cgroup_mutex);		\
397 398 399
		       cgroup_is_dead(child); }))			\
			;						\
		else
400

401 402 403
/* the list of cgroups eligible for automatic release. Protected by
 * release_list_lock */
static LIST_HEAD(release_list);
404
static DEFINE_RAW_SPINLOCK(release_list_lock);
405 406
static void cgroup_release_agent(struct work_struct *work);
static DECLARE_WORK(release_agent_work, cgroup_release_agent);
407
static void check_for_release(struct cgroup *cgrp);
408

409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
/*
 * A cgroup can be associated with multiple css_sets as different tasks may
 * belong to different cgroups on different hierarchies.  In the other
 * direction, a css_set is naturally associated with multiple cgroups.
 * This M:N relationship is represented by the following link structure
 * which exists for each association and allows traversing the associations
 * from both sides.
 */
struct cgrp_cset_link {
	/* the cgroup and css_set this link associates */
	struct cgroup		*cgrp;
	struct css_set		*cset;

	/* list of cgrp_cset_links anchored at cgrp->cset_links */
	struct list_head	cset_link;

	/* list of cgrp_cset_links anchored at css_set->cgrp_links */
	struct list_head	cgrp_link;
427 428
};

429 430
/*
 * The default css_set - used by init and its children prior to any
431 432 433 434 435
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */
T
Tejun Heo 已提交
436
struct css_set init_css_set = {
437 438 439 440 441 442 443
	.refcount		= ATOMIC_INIT(1),
	.cgrp_links		= LIST_HEAD_INIT(init_css_set.cgrp_links),
	.tasks			= LIST_HEAD_INIT(init_css_set.tasks),
	.mg_tasks		= LIST_HEAD_INIT(init_css_set.mg_tasks),
	.mg_preload_node	= LIST_HEAD_INIT(init_css_set.mg_preload_node),
	.mg_node		= LIST_HEAD_INIT(init_css_set.mg_node),
};
444

445
static int css_set_count	= 1;	/* 1 for init_css_set */
446

447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
/**
 * cgroup_update_populated - updated populated count of a cgroup
 * @cgrp: the target cgroup
 * @populated: inc or dec populated count
 *
 * @cgrp is either getting the first task (css_set) or losing the last.
 * Update @cgrp->populated_cnt accordingly.  The count is propagated
 * towards root so that a given cgroup's populated_cnt is zero iff the
 * cgroup and all its descendants are empty.
 *
 * @cgrp's interface file "cgroup.populated" is zero if
 * @cgrp->populated_cnt is zero and 1 otherwise.  When @cgrp->populated_cnt
 * changes from or to zero, userland is notified that the content of the
 * interface file has changed.  This can be used to detect when @cgrp and
 * its descendants become populated or empty.
 */
static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
{
	lockdep_assert_held(&css_set_rwsem);

	do {
		bool trigger;

		if (populated)
			trigger = !cgrp->populated_cnt++;
		else
			trigger = !--cgrp->populated_cnt;

		if (!trigger)
			break;

		if (cgrp->populated_kn)
			kernfs_notify(cgrp->populated_kn);
T
Tejun Heo 已提交
480
		cgrp = cgroup_parent(cgrp);
481 482 483
	} while (cgrp);
}

484 485 486 487 488
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
489
#define CSS_SET_HASH_BITS	7
490
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
491

492
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
493
{
494
	unsigned long key = 0UL;
495 496
	struct cgroup_subsys *ss;
	int i;
497

498
	for_each_subsys(ss, i)
499 500
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
501

502
	return key;
503 504
}

505
static void put_css_set_locked(struct css_set *cset, bool taskexit)
506
{
507
	struct cgrp_cset_link *link, *tmp_link;
T
Tejun Heo 已提交
508 509
	struct cgroup_subsys *ss;
	int ssid;
510

511 512 513
	lockdep_assert_held(&css_set_rwsem);

	if (!atomic_dec_and_test(&cset->refcount))
514
		return;
515

516
	/* This css_set is dead. unlink it and release cgroup refcounts */
T
Tejun Heo 已提交
517 518
	for_each_subsys(ss, ssid)
		list_del(&cset->e_cset_node[ssid]);
519
	hash_del(&cset->hlist);
520 521
	css_set_count--;

522
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
523
		struct cgroup *cgrp = link->cgrp;
524

525 526
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
L
Li Zefan 已提交
527

528
		/* @cgrp can't go away while we're holding css_set_rwsem */
529 530 531 532 533 534 535
		if (list_empty(&cgrp->cset_links)) {
			cgroup_update_populated(cgrp, false);
			if (notify_on_release(cgrp)) {
				if (taskexit)
					set_bit(CGRP_RELEASABLE, &cgrp->flags);
				check_for_release(cgrp);
			}
536
		}
537 538

		kfree(link);
539
	}
540

541
	kfree_rcu(cset, rcu_head);
542 543
}

544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
static void put_css_set(struct css_set *cset, bool taskexit)
{
	/*
	 * Ensure that the refcount doesn't hit zero while any readers
	 * can see it. Similar to atomic_dec_and_lock(), but for an
	 * rwlock
	 */
	if (atomic_add_unless(&cset->refcount, -1, 1))
		return;

	down_write(&css_set_rwsem);
	put_css_set_locked(cset, taskexit);
	up_write(&css_set_rwsem);
}

559 560 561
/*
 * refcounted get/put for css_set objects
 */
562
static inline void get_css_set(struct css_set *cset)
563
{
564
	atomic_inc(&cset->refcount);
565 566
}

567
/**
568
 * compare_css_sets - helper function for find_existing_css_set().
569 570
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
571 572 573
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
574
 * Returns true if "cset" matches "old_cset" except for the hierarchy
575 576
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
577 578
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
579 580 581 582 583
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
	struct list_head *l1, *l2;

584 585 586 587 588 589
	/*
	 * On the default hierarchy, there can be csets which are
	 * associated with the same set of cgroups but different csses.
	 * Let's first ensure that csses match.
	 */
	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
590 591 592 593
		return false;

	/*
	 * Compare cgroup pointers in order to distinguish between
594 595 596
	 * different cgroups in hierarchies.  As different cgroups may
	 * share the same effective css, this comparison is always
	 * necessary.
597
	 */
598 599
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
600
	while (1) {
601
		struct cgrp_cset_link *link1, *link2;
602
		struct cgroup *cgrp1, *cgrp2;
603 604 605 606

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
607 608
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
609 610
			break;
		} else {
611
			BUG_ON(l2 == &old_cset->cgrp_links);
612 613
		}
		/* Locate the cgroups associated with these links. */
614 615 616 617
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
618
		/* Hierarchies should be linked in the same order. */
619
		BUG_ON(cgrp1->root != cgrp2->root);
620 621 622 623 624 625 626 627

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
628 629
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
630 631
				return false;
		} else {
632
			if (cgrp1 != cgrp2)
633 634 635 636 637 638
				return false;
		}
	}
	return true;
}

639 640 641 642 643
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
644
 */
645 646 647
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
648
{
649
	struct cgroup_root *root = cgrp->root;
650
	struct cgroup_subsys *ss;
651
	struct css_set *cset;
652
	unsigned long key;
653
	int i;
654

B
Ben Blum 已提交
655 656 657 658 659
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
660
	for_each_subsys(ss, i) {
661
		if (root->subsys_mask & (1UL << i)) {
662 663 664 665 666
			/*
			 * @ss is in this hierarchy, so we want the
			 * effective css from @cgrp.
			 */
			template[i] = cgroup_e_css(cgrp, ss);
667
		} else {
668 669 670 671
			/*
			 * @ss is not in this hierarchy, so we don't want
			 * to change the css.
			 */
672
			template[i] = old_cset->subsys[i];
673 674 675
		}
	}

676
	key = css_set_hash(template);
677 678
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
679 680 681
			continue;

		/* This css_set matches what we need */
682
		return cset;
683
	}
684 685 686 687 688

	/* No existing cgroup group matched */
	return NULL;
}

689
static void free_cgrp_cset_links(struct list_head *links_to_free)
690
{
691
	struct cgrp_cset_link *link, *tmp_link;
692

693 694
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
695 696 697 698
		kfree(link);
	}
}

699 700 701 702 703 704 705
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
706
 */
707
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
708
{
709
	struct cgrp_cset_link *link;
710
	int i;
711 712 713

	INIT_LIST_HEAD(tmp_links);

714
	for (i = 0; i < count; i++) {
715
		link = kzalloc(sizeof(*link), GFP_KERNEL);
716
		if (!link) {
717
			free_cgrp_cset_links(tmp_links);
718 719
			return -ENOMEM;
		}
720
		list_add(&link->cset_link, tmp_links);
721 722 723 724
	}
	return 0;
}

725 726
/**
 * link_css_set - a helper function to link a css_set to a cgroup
727
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
728
 * @cset: the css_set to be linked
729 730
 * @cgrp: the destination cgroup
 */
731 732
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
733
{
734
	struct cgrp_cset_link *link;
735

736
	BUG_ON(list_empty(tmp_links));
T
Tejun Heo 已提交
737 738 739 740

	if (cgroup_on_dfl(cgrp))
		cset->dfl_cgrp = cgrp;

741 742
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
743
	link->cgrp = cgrp;
744 745 746

	if (list_empty(&cgrp->cset_links))
		cgroup_update_populated(cgrp, true);
747
	list_move(&link->cset_link, &cgrp->cset_links);
748

749 750 751 752
	/*
	 * Always add links to the tail of the list so that the list
	 * is sorted by order of hierarchy creation
	 */
753
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
754 755
}

756 757 758 759 760 761 762
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
763
 */
764 765
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
766
{
767
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
768
	struct css_set *cset;
769 770
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
771
	struct cgroup_subsys *ss;
772
	unsigned long key;
T
Tejun Heo 已提交
773
	int ssid;
774

775 776
	lockdep_assert_held(&cgroup_mutex);

777 778
	/* First see if we already have a cgroup group that matches
	 * the desired set */
779
	down_read(&css_set_rwsem);
780 781 782
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
783
	up_read(&css_set_rwsem);
784

785 786
	if (cset)
		return cset;
787

788
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
789
	if (!cset)
790 791
		return NULL;

792
	/* Allocate all the cgrp_cset_link objects that we'll need */
793
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
794
		kfree(cset);
795 796 797
		return NULL;
	}

798
	atomic_set(&cset->refcount, 1);
799
	INIT_LIST_HEAD(&cset->cgrp_links);
800
	INIT_LIST_HEAD(&cset->tasks);
T
Tejun Heo 已提交
801
	INIT_LIST_HEAD(&cset->mg_tasks);
802
	INIT_LIST_HEAD(&cset->mg_preload_node);
803
	INIT_LIST_HEAD(&cset->mg_node);
804
	INIT_HLIST_NODE(&cset->hlist);
805 806 807

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
808
	memcpy(cset->subsys, template, sizeof(cset->subsys));
809

810
	down_write(&css_set_rwsem);
811
	/* Add reference counts and links from the new css_set. */
812
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
813
		struct cgroup *c = link->cgrp;
814

815 816
		if (c->root == cgrp->root)
			c = cgrp;
817
		link_css_set(&tmp_links, cset, c);
818
	}
819

820
	BUG_ON(!list_empty(&tmp_links));
821 822

	css_set_count++;
823

T
Tejun Heo 已提交
824
	/* Add @cset to the hash table */
825 826
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
827

T
Tejun Heo 已提交
828 829 830 831
	for_each_subsys(ss, ssid)
		list_add_tail(&cset->e_cset_node[ssid],
			      &cset->subsys[ssid]->cgroup->e_csets[ssid]);

832
	up_write(&css_set_rwsem);
833

834
	return cset;
835 836
}

837
static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
838
{
839
	struct cgroup *root_cgrp = kf_root->kn->priv;
T
Tejun Heo 已提交
840

841
	return root_cgrp->root;
T
Tejun Heo 已提交
842 843
}

844
static int cgroup_init_root_id(struct cgroup_root *root)
845 846 847 848 849
{
	int id;

	lockdep_assert_held(&cgroup_mutex);

850
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
851 852 853 854 855 856 857
	if (id < 0)
		return id;

	root->hierarchy_id = id;
	return 0;
}

858
static void cgroup_exit_root_id(struct cgroup_root *root)
859 860 861 862 863 864 865 866 867
{
	lockdep_assert_held(&cgroup_mutex);

	if (root->hierarchy_id) {
		idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
		root->hierarchy_id = 0;
	}
}

868
static void cgroup_free_root(struct cgroup_root *root)
869 870 871 872 873 874 875 876 877 878
{
	if (root) {
		/* hierarhcy ID shoulid already have been released */
		WARN_ON_ONCE(root->hierarchy_id);

		idr_destroy(&root->cgroup_idr);
		kfree(root);
	}
}

879
static void cgroup_destroy_root(struct cgroup_root *root)
880
{
881
	struct cgroup *cgrp = &root->cgrp;
882 883
	struct cgrp_cset_link *link, *tmp_link;

T
Tejun Heo 已提交
884
	mutex_lock(&cgroup_mutex);
885

T
Tejun Heo 已提交
886
	BUG_ON(atomic_read(&root->nr_cgrps));
887
	BUG_ON(!list_empty(&cgrp->self.children));
888 889

	/* Rebind all subsystems back to the default hierarchy */
890
	rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
891 892

	/*
893 894
	 * Release all the links from cset_links to this hierarchy's
	 * root cgroup
895
	 */
896
	down_write(&css_set_rwsem);
897 898 899 900 901 902

	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
		kfree(link);
	}
903
	up_write(&css_set_rwsem);
904 905 906 907 908 909 910 911 912 913

	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
		cgroup_root_count--;
	}

	cgroup_exit_root_id(root);

	mutex_unlock(&cgroup_mutex);

T
Tejun Heo 已提交
914
	kernfs_destroy_root(root->kf_root);
915 916 917
	cgroup_free_root(root);
}

918 919
/* look up cgroup associated with given css_set on the specified hierarchy */
static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
920
					    struct cgroup_root *root)
921 922 923
{
	struct cgroup *res = NULL;

924 925 926
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

927
	if (cset == &init_css_set) {
928
		res = &root->cgrp;
929
	} else {
930 931 932
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
933
			struct cgroup *c = link->cgrp;
934

935 936 937 938 939 940
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
941

942 943 944 945
	BUG_ON(!res);
	return res;
}

946
/*
947 948 949 950
 * Return the cgroup for "task" from the given hierarchy. Must be
 * called with cgroup_mutex and css_set_rwsem held.
 */
static struct cgroup *task_cgroup_from_root(struct task_struct *task,
951
					    struct cgroup_root *root)
952 953 954 955 956 957 958 959 960
{
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
	return cset_cgroup_from_root(task_css_set(task), root);
}

961 962 963 964 965 966
/*
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
967
 * cgroup_attach_task() can increment it again.  Because a count of zero
968 969 970 971 972 973 974 975 976 977 978 979 980
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
 * (usually) take cgroup_mutex.  These are the two most performance
 * critical pieces of code here.  The exception occurs on cgroup_exit(),
 * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
 * is taken, and if the cgroup count is zero, a usermode call made
L
Li Zefan 已提交
981 982
 * to the release agent with the name of the cgroup (path relative to
 * the root of cgroup file system) as the argument.
983 984 985 986
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
987
 * least one task in the system (init, pid == 1), therefore, root cgroup
988
 * always has either children cgroups and/or using tasks.  So we don't
989
 * need a special hack to ensure that root cgroup cannot be deleted.
990 991
 *
 * P.S.  One more locking exception.  RCU is used to guard the
992
 * update of a tasks cgroup pointer by cgroup_attach_task()
993 994
 */

995
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask);
T
Tejun Heo 已提交
996
static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
997
static const struct file_operations proc_cgroupstats_operations;
998

T
Tejun Heo 已提交
999 1000
static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
			      char *buf)
1001
{
T
Tejun Heo 已提交
1002 1003 1004 1005 1006 1007 1008
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
		snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
			 cft->ss->name, cft->name);
	else
		strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
	return buf;
1009 1010
}

1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
 * returns cft->mode if ->mode is not 0
 * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
 * returns S_IRUGO if it has only a read handler
 * returns S_IWUSR if it has only a write hander
 */
static umode_t cgroup_file_mode(const struct cftype *cft)
1021
{
1022
	umode_t mode = 0;
1023

1024 1025 1026 1027 1028 1029
	if (cft->mode)
		return cft->mode;

	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
		mode |= S_IRUGO;

1030
	if (cft->write_u64 || cft->write_s64 || cft->write)
1031 1032 1033
		mode |= S_IWUSR;

	return mode;
1034 1035
}

1036
static void cgroup_get(struct cgroup *cgrp)
1037
{
T
Tejun Heo 已提交
1038
	WARN_ON_ONCE(cgroup_is_dead(cgrp));
1039
	css_get(&cgrp->self);
1040 1041
}

1042
static void cgroup_put(struct cgroup *cgrp)
1043
{
1044
	css_put(&cgrp->self);
1045 1046
}

1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
/**
 * cgroup_refresh_child_subsys_mask - update child_subsys_mask
 * @cgrp: the target cgroup
 *
 * On the default hierarchy, a subsystem may request other subsystems to be
 * enabled together through its ->depends_on mask.  In such cases, more
 * subsystems than specified in "cgroup.subtree_control" may be enabled.
 *
 * This function determines which subsystems need to be enabled given the
 * current @cgrp->subtree_control and records it in
 * @cgrp->child_subsys_mask.  The resulting mask is always a superset of
 * @cgrp->subtree_control and follows the usual hierarchy rules.
 */
1060 1061
static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp)
{
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
	struct cgroup *parent = cgroup_parent(cgrp);
	unsigned int cur_ss_mask = cgrp->subtree_control;
	struct cgroup_subsys *ss;
	int ssid;

	lockdep_assert_held(&cgroup_mutex);

	if (!cgroup_on_dfl(cgrp)) {
		cgrp->child_subsys_mask = cur_ss_mask;
		return;
	}

	while (true) {
		unsigned int new_ss_mask = cur_ss_mask;

		for_each_subsys(ss, ssid)
			if (cur_ss_mask & (1 << ssid))
				new_ss_mask |= ss->depends_on;

		/*
		 * Mask out subsystems which aren't available.  This can
		 * happen only if some depended-upon subsystems were bound
		 * to non-default hierarchies.
		 */
		if (parent)
			new_ss_mask &= parent->child_subsys_mask;
		else
			new_ss_mask &= cgrp->root->subsys_mask;

		if (new_ss_mask == cur_ss_mask)
			break;
		cur_ss_mask = new_ss_mask;
	}

	cgrp->child_subsys_mask = cur_ss_mask;
1097 1098
}

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
/**
 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper undoes cgroup_kn_lock_live() and should be invoked before
 * the method finishes if locking succeeded.  Note that once this function
 * returns the cgroup returned by cgroup_kn_lock_live() may become
 * inaccessible any time.  If the caller intends to continue to access the
 * cgroup, it should pin it before invoking this function.
 */
static void cgroup_kn_unlock(struct kernfs_node *kn)
1110
{
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;

	mutex_unlock(&cgroup_mutex);

	kernfs_unbreak_active_protection(kn);
	cgroup_put(cgrp);
1122 1123
}

1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
/**
 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper is to be used by a cgroup kernfs method currently servicing
 * @kn.  It breaks the active protection, performs cgroup locking and
 * verifies that the associated cgroup is alive.  Returns the cgroup if
 * alive; otherwise, %NULL.  A successful return should be undone by a
 * matching cgroup_kn_unlock() invocation.
 *
 * Any cgroup kernfs method implementation which requires locking the
 * associated cgroup should use this helper.  It avoids nesting cgroup
 * locking under kernfs active protection and allows all kernfs operations
 * including self-removal.
 */
static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
T
Tejun Heo 已提交
1140
{
1141 1142 1143 1144 1145 1146
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;
T
Tejun Heo 已提交
1147

1148
	/*
1149
	 * We're gonna grab cgroup_mutex which nests outside kernfs
1150 1151 1152
	 * active_ref.  cgroup liveliness check alone provides enough
	 * protection against removal.  Ensure @cgrp stays accessible and
	 * break the active_ref protection.
1153
	 */
1154 1155 1156
	cgroup_get(cgrp);
	kernfs_break_active_protection(kn);

T
Tejun Heo 已提交
1157
	mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1158

1159 1160 1161 1162 1163
	if (!cgroup_is_dead(cgrp))
		return cgrp;

	cgroup_kn_unlock(kn);
	return NULL;
1164
}
T
Tejun Heo 已提交
1165

1166
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
1167
{
T
Tejun Heo 已提交
1168
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
1169

1170
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
1171
	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
T
Tejun Heo 已提交
1172 1173
}

1174
/**
1175
 * cgroup_clear_dir - remove subsys files in a cgroup directory
1176
 * @cgrp: target cgroup
1177 1178
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
1179
static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask)
T
Tejun Heo 已提交
1180
{
1181
	struct cgroup_subsys *ss;
1182
	int i;
T
Tejun Heo 已提交
1183

1184
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
1185
		struct cftype *cfts;
1186

1187
		if (!(subsys_mask & (1 << i)))
1188
			continue;
T
Tejun Heo 已提交
1189 1190
		list_for_each_entry(cfts, &ss->cfts, node)
			cgroup_addrm_files(cgrp, cfts, false);
1191
	}
1192 1193
}

1194
static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask)
1195
{
1196
	struct cgroup_subsys *ss;
1197
	unsigned int tmp_ss_mask;
T
Tejun Heo 已提交
1198
	int ssid, i, ret;
1199

T
Tejun Heo 已提交
1200
	lockdep_assert_held(&cgroup_mutex);
1201

1202 1203 1204
	for_each_subsys(ss, ssid) {
		if (!(ss_mask & (1 << ssid)))
			continue;
B
Ben Blum 已提交
1205

1206 1207
		/* if @ss has non-root csses attached to it, can't move */
		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
T
Tejun Heo 已提交
1208
			return -EBUSY;
1209

1210
		/* can't move between two non-dummy roots either */
1211
		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1212
			return -EBUSY;
1213 1214
	}

1215 1216 1217 1218 1219 1220
	/* skip creating root files on dfl_root for inhibited subsystems */
	tmp_ss_mask = ss_mask;
	if (dst_root == &cgrp_dfl_root)
		tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask;

	ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask);
T
Tejun Heo 已提交
1221 1222
	if (ret) {
		if (dst_root != &cgrp_dfl_root)
1223
			return ret;
1224

T
Tejun Heo 已提交
1225 1226 1227 1228 1229 1230 1231
		/*
		 * Rebinding back to the default root is not allowed to
		 * fail.  Using both default and non-default roots should
		 * be rare.  Moving subsystems back and forth even more so.
		 * Just warn about it and continue.
		 */
		if (cgrp_dfl_root_visible) {
1232
			pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
1233
				ret, ss_mask);
1234
			pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
T
Tejun Heo 已提交
1235
		}
1236
	}
1237 1238 1239 1240 1241

	/*
	 * Nothing can fail from this point on.  Remove files for the
	 * removed subsystems and rebind each subsystem.
	 */
1242
	for_each_subsys(ss, ssid)
T
Tejun Heo 已提交
1243
		if (ss_mask & (1 << ssid))
1244
			cgroup_clear_dir(&ss->root->cgrp, 1 << ssid);
1245

1246
	for_each_subsys(ss, ssid) {
1247
		struct cgroup_root *src_root;
1248
		struct cgroup_subsys_state *css;
T
Tejun Heo 已提交
1249
		struct css_set *cset;
1250

1251 1252
		if (!(ss_mask & (1 << ssid)))
			continue;
1253

1254
		src_root = ss->root;
1255
		css = cgroup_css(&src_root->cgrp, ss);
1256

1257
		WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss));
1258

1259 1260
		RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL);
		rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css);
1261
		ss->root = dst_root;
1262
		css->cgroup = &dst_root->cgrp;
1263

T
Tejun Heo 已提交
1264 1265 1266 1267 1268 1269
		down_write(&css_set_rwsem);
		hash_for_each(css_set_table, i, cset, hlist)
			list_move_tail(&cset->e_cset_node[ss->id],
				       &dst_root->cgrp.e_csets[ss->id]);
		up_write(&css_set_rwsem);

1270
		src_root->subsys_mask &= ~(1 << ssid);
1271 1272
		src_root->cgrp.subtree_control &= ~(1 << ssid);
		cgroup_refresh_child_subsys_mask(&src_root->cgrp);
1273

1274
		/* default hierarchy doesn't enable controllers by default */
1275
		dst_root->subsys_mask |= 1 << ssid;
1276 1277 1278 1279
		if (dst_root != &cgrp_dfl_root) {
			dst_root->cgrp.subtree_control |= 1 << ssid;
			cgroup_refresh_child_subsys_mask(&dst_root->cgrp);
		}
1280

1281 1282
		if (ss->bind)
			ss->bind(css);
1283 1284
	}

T
Tejun Heo 已提交
1285
	kernfs_activate(dst_root->cgrp.kn);
1286 1287 1288
	return 0;
}

T
Tejun Heo 已提交
1289 1290
static int cgroup_show_options(struct seq_file *seq,
			       struct kernfs_root *kf_root)
1291
{
1292
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1293
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1294
	int ssid;
1295

T
Tejun Heo 已提交
1296
	for_each_subsys(ss, ssid)
1297
		if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
1298
			seq_printf(seq, ",%s", ss->name);
1299
	if (root->flags & CGRP_ROOT_NOPREFIX)
1300
		seq_puts(seq, ",noprefix");
1301
	if (root->flags & CGRP_ROOT_XATTR)
A
Aristeu Rozanski 已提交
1302
		seq_puts(seq, ",xattr");
1303 1304

	spin_lock(&release_agent_path_lock);
1305 1306
	if (strlen(root->release_agent_path))
		seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1307 1308
	spin_unlock(&release_agent_path_lock);

1309
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
1310
		seq_puts(seq, ",clone_children");
1311 1312
	if (strlen(root->name))
		seq_printf(seq, ",name=%s", root->name);
1313 1314 1315 1316
	return 0;
}

struct cgroup_sb_opts {
1317 1318
	unsigned int subsys_mask;
	unsigned int flags;
1319
	char *release_agent;
1320
	bool cpuset_clone_children;
1321
	char *name;
1322 1323
	/* User explicitly requested empty subsystem */
	bool none;
1324 1325
};

B
Ben Blum 已提交
1326
static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1327
{
1328 1329
	char *token, *o = data;
	bool all_ss = false, one_ss = false;
1330
	unsigned int mask = -1U;
1331
	struct cgroup_subsys *ss;
1332
	int nr_opts = 0;
1333
	int i;
1334 1335

#ifdef CONFIG_CPUSETS
1336
	mask = ~(1U << cpuset_cgrp_id);
1337
#endif
1338

1339
	memset(opts, 0, sizeof(*opts));
1340 1341

	while ((token = strsep(&o, ",")) != NULL) {
1342 1343
		nr_opts++;

1344 1345
		if (!*token)
			return -EINVAL;
1346
		if (!strcmp(token, "none")) {
1347 1348
			/* Explicitly have no subsystems */
			opts->none = true;
1349 1350 1351 1352 1353 1354 1355 1356 1357
			continue;
		}
		if (!strcmp(token, "all")) {
			/* Mutually exclusive option 'all' + subsystem name */
			if (one_ss)
				return -EINVAL;
			all_ss = true;
			continue;
		}
1358 1359 1360 1361
		if (!strcmp(token, "__DEVEL__sane_behavior")) {
			opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
			continue;
		}
1362
		if (!strcmp(token, "noprefix")) {
1363
			opts->flags |= CGRP_ROOT_NOPREFIX;
1364 1365 1366
			continue;
		}
		if (!strcmp(token, "clone_children")) {
1367
			opts->cpuset_clone_children = true;
1368 1369
			continue;
		}
A
Aristeu Rozanski 已提交
1370
		if (!strcmp(token, "xattr")) {
1371
			opts->flags |= CGRP_ROOT_XATTR;
A
Aristeu Rozanski 已提交
1372 1373
			continue;
		}
1374
		if (!strncmp(token, "release_agent=", 14)) {
1375 1376 1377
			/* Specifying two release agents is forbidden */
			if (opts->release_agent)
				return -EINVAL;
1378
			opts->release_agent =
1379
				kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1380 1381
			if (!opts->release_agent)
				return -ENOMEM;
1382 1383 1384
			continue;
		}
		if (!strncmp(token, "name=", 5)) {
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
			const char *name = token + 5;
			/* Can't specify an empty name */
			if (!strlen(name))
				return -EINVAL;
			/* Must match [\w.-]+ */
			for (i = 0; i < strlen(name); i++) {
				char c = name[i];
				if (isalnum(c))
					continue;
				if ((c == '.') || (c == '-') || (c == '_'))
					continue;
				return -EINVAL;
			}
			/* Specifying two names is forbidden */
			if (opts->name)
				return -EINVAL;
			opts->name = kstrndup(name,
1402
					      MAX_CGROUP_ROOT_NAMELEN - 1,
1403 1404 1405
					      GFP_KERNEL);
			if (!opts->name)
				return -ENOMEM;
1406 1407 1408 1409

			continue;
		}

1410
		for_each_subsys(ss, i) {
1411 1412 1413 1414 1415 1416 1417 1418
			if (strcmp(token, ss->name))
				continue;
			if (ss->disabled)
				continue;

			/* Mutually exclusive option 'all' + subsystem name */
			if (all_ss)
				return -EINVAL;
1419
			opts->subsys_mask |= (1 << i);
1420 1421 1422 1423 1424 1425 1426 1427
			one_ss = true;

			break;
		}
		if (i == CGROUP_SUBSYS_COUNT)
			return -ENOENT;
	}

1428
	if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
1429
		pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");
1430 1431
		if (nr_opts != 1) {
			pr_err("sane_behavior: no other mount options allowed\n");
1432 1433
			return -EINVAL;
		}
1434
		return 0;
1435 1436
	}

1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453
	/*
	 * If the 'all' option was specified select all the subsystems,
	 * otherwise if 'none', 'name=' and a subsystem name options were
	 * not specified, let's default to 'all'
	 */
	if (all_ss || (!one_ss && !opts->none && !opts->name))
		for_each_subsys(ss, i)
			if (!ss->disabled)
				opts->subsys_mask |= (1 << i);

	/*
	 * We either have to specify by name or by subsystems. (So all
	 * empty hierarchies must have a name).
	 */
	if (!opts->subsys_mask && !opts->name)
		return -EINVAL;

1454 1455 1456 1457 1458
	/*
	 * Option noprefix was introduced just for backward compatibility
	 * with the old cpuset, so we allow noprefix only if mounting just
	 * the cpuset subsystem.
	 */
1459
	if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1460 1461
		return -EINVAL;

1462
	/* Can't specify "none" and some subsystems */
1463
	if (opts->subsys_mask && opts->none)
1464 1465
		return -EINVAL;

1466 1467 1468
	return 0;
}

T
Tejun Heo 已提交
1469
static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1470 1471
{
	int ret = 0;
1472
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
1473
	struct cgroup_sb_opts opts;
1474
	unsigned int added_mask, removed_mask;
1475

1476 1477
	if (root == &cgrp_dfl_root) {
		pr_err("remount is not allowed\n");
1478 1479 1480
		return -EINVAL;
	}

1481 1482 1483 1484 1485 1486 1487
	mutex_lock(&cgroup_mutex);

	/* See what subsystems are wanted */
	ret = parse_cgroupfs_options(data, &opts);
	if (ret)
		goto out_unlock;

1488
	if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1489
		pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
1490
			task_tgid_nr(current), current->comm);
1491

1492 1493
	added_mask = opts.subsys_mask & ~root->subsys_mask;
	removed_mask = root->subsys_mask & ~opts.subsys_mask;
1494

B
Ben Blum 已提交
1495
	/* Don't allow flags or name to change at remount */
T
Tejun Heo 已提交
1496
	if ((opts.flags ^ root->flags) ||
B
Ben Blum 已提交
1497
	    (opts.name && strcmp(opts.name, root->name))) {
1498
		pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
T
Tejun Heo 已提交
1499
		       opts.flags, opts.name ?: "", root->flags, root->name);
1500 1501 1502 1503
		ret = -EINVAL;
		goto out_unlock;
	}

1504
	/* remounting is not allowed for populated hierarchies */
1505
	if (!list_empty(&root->cgrp.self.children)) {
1506
		ret = -EBUSY;
1507
		goto out_unlock;
B
Ben Blum 已提交
1508
	}
1509

1510
	ret = rebind_subsystems(root, added_mask);
1511
	if (ret)
1512
		goto out_unlock;
1513

1514
	rebind_subsystems(&cgrp_dfl_root, removed_mask);
1515

1516 1517
	if (opts.release_agent) {
		spin_lock(&release_agent_path_lock);
1518
		strcpy(root->release_agent_path, opts.release_agent);
1519 1520
		spin_unlock(&release_agent_path_lock);
	}
1521
 out_unlock:
1522
	kfree(opts.release_agent);
1523
	kfree(opts.name);
1524 1525 1526 1527
	mutex_unlock(&cgroup_mutex);
	return ret;
}

1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
/*
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
 * words after the first mount.
 */
static bool use_task_css_set_links __read_mostly;

static void cgroup_enable_task_cg_lists(void)
{
	struct task_struct *p, *g;

1540
	down_write(&css_set_rwsem);
1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562

	if (use_task_css_set_links)
		goto out_unlock;

	use_task_css_set_links = true;

	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		WARN_ON_ONCE(!list_empty(&p->cg_list) ||
			     task_css_set(p) != &init_css_set);

		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
1563 1564
		 * Do it while holding siglock so that we don't end up
		 * racing against cgroup_exit().
1565
		 */
1566
		spin_lock_irq(&p->sighand->siglock);
1567 1568 1569 1570 1571 1572
		if (!(p->flags & PF_EXITING)) {
			struct css_set *cset = task_css_set(p);

			list_add(&p->cg_list, &cset->tasks);
			get_css_set(cset);
		}
1573
		spin_unlock_irq(&p->sighand->siglock);
1574 1575 1576
	} while_each_thread(g, p);
	read_unlock(&tasklist_lock);
out_unlock:
1577
	up_write(&css_set_rwsem);
1578
}
1579

1580 1581
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
T
Tejun Heo 已提交
1582 1583 1584
	struct cgroup_subsys *ss;
	int ssid;

1585 1586
	INIT_LIST_HEAD(&cgrp->self.sibling);
	INIT_LIST_HEAD(&cgrp->self.children);
1587
	INIT_LIST_HEAD(&cgrp->cset_links);
1588
	INIT_LIST_HEAD(&cgrp->release_list);
1589 1590
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
1591
	cgrp->self.cgroup = cgrp;
1592
	cgrp->self.flags |= CSS_ONLINE;
T
Tejun Heo 已提交
1593 1594 1595

	for_each_subsys(ss, ssid)
		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1596 1597

	init_waitqueue_head(&cgrp->offline_waitq);
1598
}
1599

1600
static void init_cgroup_root(struct cgroup_root *root,
1601
			     struct cgroup_sb_opts *opts)
1602
{
1603
	struct cgroup *cgrp = &root->cgrp;
1604

1605
	INIT_LIST_HEAD(&root->root_list);
1606
	atomic_set(&root->nr_cgrps, 1);
1607
	cgrp->root = root;
1608
	init_cgroup_housekeeping(cgrp);
1609
	idr_init(&root->cgroup_idr);
1610 1611 1612 1613 1614 1615

	root->flags = opts->flags;
	if (opts->release_agent)
		strcpy(root->release_agent_path, opts->release_agent);
	if (opts->name)
		strcpy(root->name, opts->name);
1616
	if (opts->cpuset_clone_children)
1617
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1618 1619
}

1620
static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
1621
{
1622
	LIST_HEAD(tmp_links);
1623
	struct cgroup *root_cgrp = &root->cgrp;
1624
	struct cftype *base_files;
1625 1626
	struct css_set *cset;
	int i, ret;
1627

1628
	lockdep_assert_held(&cgroup_mutex);
1629

1630
	ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT);
1631
	if (ret < 0)
T
Tejun Heo 已提交
1632
		goto out;
1633
	root_cgrp->id = ret;
1634

1635 1636 1637 1638
	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release);
	if (ret)
		goto out;

1639
	/*
1640
	 * We're accessing css_set_count without locking css_set_rwsem here,
1641 1642 1643 1644 1645 1646
	 * but that's OK - it can only be increased by someone holding
	 * cgroup_lock, and that's us. The worst that can happen is that we
	 * have some link structures left over
	 */
	ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
	if (ret)
1647
		goto cancel_ref;
1648

1649
	ret = cgroup_init_root_id(root);
1650
	if (ret)
1651
		goto cancel_ref;
1652

T
Tejun Heo 已提交
1653 1654 1655 1656 1657 1658 1659 1660
	root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
					   KERNFS_ROOT_CREATE_DEACTIVATED,
					   root_cgrp);
	if (IS_ERR(root->kf_root)) {
		ret = PTR_ERR(root->kf_root);
		goto exit_root_id;
	}
	root_cgrp->kn = root->kf_root->kn;
1661

1662 1663 1664 1665 1666 1667
	if (root == &cgrp_dfl_root)
		base_files = cgroup_dfl_base_files;
	else
		base_files = cgroup_legacy_base_files;

	ret = cgroup_addrm_files(root_cgrp, base_files, true);
1668
	if (ret)
T
Tejun Heo 已提交
1669
		goto destroy_root;
1670

1671
	ret = rebind_subsystems(root, ss_mask);
1672
	if (ret)
T
Tejun Heo 已提交
1673
		goto destroy_root;
1674

1675 1676 1677 1678 1679 1680 1681
	/*
	 * There must be no failure case after here, since rebinding takes
	 * care of subsystems' refcounts, which are explicitly dropped in
	 * the failure exit path.
	 */
	list_add(&root->root_list, &cgroup_roots);
	cgroup_root_count++;
A
Al Viro 已提交
1682

1683
	/*
1684
	 * Link the root cgroup in this hierarchy into all the css_set
1685 1686
	 * objects.
	 */
1687
	down_write(&css_set_rwsem);
1688 1689
	hash_for_each(css_set_table, i, cset, hlist)
		link_css_set(&tmp_links, cset, root_cgrp);
1690
	up_write(&css_set_rwsem);
1691

1692
	BUG_ON(!list_empty(&root_cgrp->self.children));
1693
	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1694

T
Tejun Heo 已提交
1695
	kernfs_activate(root_cgrp->kn);
1696
	ret = 0;
T
Tejun Heo 已提交
1697
	goto out;
1698

T
Tejun Heo 已提交
1699 1700 1701 1702
destroy_root:
	kernfs_destroy_root(root->kf_root);
	root->kf_root = NULL;
exit_root_id:
1703
	cgroup_exit_root_id(root);
1704 1705
cancel_ref:
	percpu_ref_cancel_init(&root_cgrp->self.refcnt);
T
Tejun Heo 已提交
1706
out:
1707 1708
	free_cgrp_cset_links(&tmp_links);
	return ret;
1709 1710
}

A
Al Viro 已提交
1711
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1712
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
1713
			 void *data)
1714
{
1715
	struct cgroup_root *root;
1716
	struct cgroup_sb_opts opts;
T
Tejun Heo 已提交
1717
	struct dentry *dentry;
1718
	int ret;
L
Li Zefan 已提交
1719
	bool new_sb;
1720

1721 1722 1723 1724 1725 1726
	/*
	 * The first time anyone tries to mount a cgroup, enable the list
	 * linking each css_set to its tasks and fix up all existing tasks.
	 */
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();
1727

B
Ben Blum 已提交
1728
	mutex_lock(&cgroup_mutex);
1729 1730

	/* First find the desired set of subsystems */
1731
	ret = parse_cgroupfs_options(data, &opts);
1732
	if (ret)
1733
		goto out_unlock;
1734

T
Tejun Heo 已提交
1735
	/* look for a matching existing root */
1736
	if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) {
T
Tejun Heo 已提交
1737 1738 1739 1740 1741
		cgrp_dfl_root_visible = true;
		root = &cgrp_dfl_root;
		cgroup_get(&root->cgrp);
		ret = 0;
		goto out_unlock;
1742 1743
	}

1744
	for_each_root(root) {
T
Tejun Heo 已提交
1745
		bool name_match = false;
1746

1747
		if (root == &cgrp_dfl_root)
1748
			continue;
1749

B
Ben Blum 已提交
1750
		/*
T
Tejun Heo 已提交
1751 1752 1753
		 * If we asked for a name then it must match.  Also, if
		 * name matches but sybsys_mask doesn't, we should fail.
		 * Remember whether name matched.
B
Ben Blum 已提交
1754
		 */
T
Tejun Heo 已提交
1755 1756 1757 1758 1759
		if (opts.name) {
			if (strcmp(opts.name, root->name))
				continue;
			name_match = true;
		}
1760

1761
		/*
T
Tejun Heo 已提交
1762 1763
		 * If we asked for subsystems (or explicitly for no
		 * subsystems) then they must match.
1764
		 */
T
Tejun Heo 已提交
1765
		if ((opts.subsys_mask || opts.none) &&
1766
		    (opts.subsys_mask != root->subsys_mask)) {
T
Tejun Heo 已提交
1767 1768 1769 1770 1771
			if (!name_match)
				continue;
			ret = -EBUSY;
			goto out_unlock;
		}
1772

1773 1774
		if (root->flags ^ opts.flags)
			pr_warn("new mount options do not match the existing superblock, will be ignored\n");
1775

T
Tejun Heo 已提交
1776
		/*
1777 1778 1779 1780 1781 1782
		 * A root's lifetime is governed by its root cgroup.
		 * tryget_live failure indicate that the root is being
		 * destroyed.  Wait for destruction to complete so that the
		 * subsystems are free.  We can use wait_queue for the wait
		 * but this path is super cold.  Let's just sleep for a bit
		 * and retry.
T
Tejun Heo 已提交
1783
		 */
1784
		if (!percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
T
Tejun Heo 已提交
1785 1786
			mutex_unlock(&cgroup_mutex);
			msleep(10);
1787 1788
			ret = restart_syscall();
			goto out_free;
T
Tejun Heo 已提交
1789
		}
1790

T
Tejun Heo 已提交
1791
		ret = 0;
T
Tejun Heo 已提交
1792
		goto out_unlock;
1793 1794
	}

1795
	/*
1796 1797 1798
	 * No such thing, create a new one.  name= matching without subsys
	 * specification is allowed for already existing hierarchies but we
	 * can't create new one without subsys specification.
1799
	 */
1800 1801 1802
	if (!opts.subsys_mask && !opts.none) {
		ret = -EINVAL;
		goto out_unlock;
1803 1804
	}

1805 1806 1807
	root = kzalloc(sizeof(*root), GFP_KERNEL);
	if (!root) {
		ret = -ENOMEM;
T
Tejun Heo 已提交
1808
		goto out_unlock;
1809
	}
1810

1811 1812
	init_cgroup_root(root, &opts);

T
Tejun Heo 已提交
1813
	ret = cgroup_setup_root(root, opts.subsys_mask);
T
Tejun Heo 已提交
1814 1815
	if (ret)
		cgroup_free_root(root);
1816

1817
out_unlock:
1818
	mutex_unlock(&cgroup_mutex);
1819
out_free:
1820 1821
	kfree(opts.release_agent);
	kfree(opts.name);
A
Aristeu Rozanski 已提交
1822

T
Tejun Heo 已提交
1823
	if (ret)
1824
		return ERR_PTR(ret);
T
Tejun Heo 已提交
1825

1826 1827
	dentry = kernfs_mount(fs_type, flags, root->kf_root,
				CGROUP_SUPER_MAGIC, &new_sb);
L
Li Zefan 已提交
1828
	if (IS_ERR(dentry) || !new_sb)
1829
		cgroup_put(&root->cgrp);
T
Tejun Heo 已提交
1830 1831 1832 1833 1834 1835
	return dentry;
}

static void cgroup_kill_sb(struct super_block *sb)
{
	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
1836
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
T
Tejun Heo 已提交
1837

1838 1839 1840 1841
	/*
	 * If @root doesn't have any mounts or children, start killing it.
	 * This prevents new mounts by disabling percpu_ref_tryget_live().
	 * cgroup_mount() may wait for @root's release.
1842 1843
	 *
	 * And don't kill the default root.
1844
	 */
1845 1846
	if (css_has_online_children(&root->cgrp.self) ||
	    root == &cgrp_dfl_root)
1847 1848 1849 1850
		cgroup_put(&root->cgrp);
	else
		percpu_ref_kill(&root->cgrp.self.refcnt);

T
Tejun Heo 已提交
1851
	kernfs_kill_sb(sb);
1852 1853 1854 1855
}

static struct file_system_type cgroup_fs_type = {
	.name = "cgroup",
A
Al Viro 已提交
1856
	.mount = cgroup_mount,
1857 1858 1859
	.kill_sb = cgroup_kill_sb,
};

1860 1861
static struct kobject *cgroup_kobj;

1862
/**
1863
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1864 1865 1866 1867
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
1868 1869 1870 1871 1872
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
T
Tejun Heo 已提交
1873
 * Return value is the same as kernfs_path().
1874
 */
T
Tejun Heo 已提交
1875
char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1876
{
1877
	struct cgroup_root *root;
1878
	struct cgroup *cgrp;
T
Tejun Heo 已提交
1879 1880
	int hierarchy_id = 1;
	char *path = NULL;
1881 1882

	mutex_lock(&cgroup_mutex);
1883
	down_read(&css_set_rwsem);
1884

1885 1886
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

1887 1888
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
T
Tejun Heo 已提交
1889
		path = cgroup_path(cgrp, buf, buflen);
1890 1891
	} else {
		/* if no hierarchy exists, everyone is in "/" */
T
Tejun Heo 已提交
1892 1893
		if (strlcpy(buf, "/", buflen) < buflen)
			path = buf;
1894 1895
	}

1896
	up_read(&css_set_rwsem);
1897
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
1898
	return path;
1899
}
1900
EXPORT_SYMBOL_GPL(task_cgroup_path);
1901

1902
/* used to track tasks and other necessary states during migration */
1903
struct cgroup_taskset {
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921
	/* the src and dst cset list running through cset->mg_node */
	struct list_head	src_csets;
	struct list_head	dst_csets;

	/*
	 * Fields for cgroup_taskset_*() iteration.
	 *
	 * Before migration is committed, the target migration tasks are on
	 * ->mg_tasks of the csets on ->src_csets.  After, on ->mg_tasks of
	 * the csets on ->dst_csets.  ->csets point to either ->src_csets
	 * or ->dst_csets depending on whether migration is committed.
	 *
	 * ->cur_csets and ->cur_task point to the current task position
	 * during iteration.
	 */
	struct list_head	*csets;
	struct css_set		*cur_cset;
	struct task_struct	*cur_task;
1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
};

/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
 *
 * @tset iteration is initialized and the first task is returned.
 */
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
{
1932 1933 1934 1935
	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
	tset->cur_task = NULL;

	return cgroup_taskset_next(tset);
1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
}

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
{
1947 1948
	struct css_set *cset = tset->cur_cset;
	struct task_struct *task = tset->cur_task;
1949

1950 1951 1952 1953 1954 1955
	while (&cset->mg_node != tset->csets) {
		if (!task)
			task = list_first_entry(&cset->mg_tasks,
						struct task_struct, cg_list);
		else
			task = list_next_entry(task, cg_list);
1956

1957 1958 1959 1960 1961
		if (&task->cg_list != &cset->mg_tasks) {
			tset->cur_cset = cset;
			tset->cur_task = task;
			return task;
		}
1962

1963 1964 1965
		cset = list_next_entry(cset, mg_node);
		task = NULL;
	}
1966

1967
	return NULL;
1968 1969
}

1970
/**
B
Ben Blum 已提交
1971
 * cgroup_task_migrate - move a task from one cgroup to another.
1972
 * @old_cgrp: the cgroup @tsk is being migrated from
1973 1974
 * @tsk: the task being migrated
 * @new_cset: the new css_set @tsk is being attached to
B
Ben Blum 已提交
1975
 *
1976
 * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked.
B
Ben Blum 已提交
1977
 */
1978 1979 1980
static void cgroup_task_migrate(struct cgroup *old_cgrp,
				struct task_struct *tsk,
				struct css_set *new_cset)
B
Ben Blum 已提交
1981
{
1982
	struct css_set *old_cset;
B
Ben Blum 已提交
1983

1984 1985 1986
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

B
Ben Blum 已提交
1987
	/*
1988 1989 1990
	 * We are synchronized through threadgroup_lock() against PF_EXITING
	 * setting such that we can't race against cgroup_exit() changing the
	 * css_set to init_css_set and dropping the old one.
B
Ben Blum 已提交
1991
	 */
1992
	WARN_ON_ONCE(tsk->flags & PF_EXITING);
1993
	old_cset = task_css_set(tsk);
B
Ben Blum 已提交
1994

1995
	get_css_set(new_cset);
1996
	rcu_assign_pointer(tsk->cgroups, new_cset);
B
Ben Blum 已提交
1997

1998 1999 2000 2001 2002 2003 2004
	/*
	 * Use move_tail so that cgroup_taskset_first() still returns the
	 * leader after migration.  This works because cgroup_migrate()
	 * ensures that the dst_cset of the leader is the first on the
	 * tset's dst_csets list.
	 */
	list_move_tail(&tsk->cg_list, &new_cset->mg_tasks);
B
Ben Blum 已提交
2005 2006

	/*
2007 2008 2009
	 * We just gained a reference on old_cset by taking it from the
	 * task. As trading it for new_cset is protected by cgroup_mutex,
	 * we're safe to drop it here; it will be freed under RCU.
B
Ben Blum 已提交
2010
	 */
2011
	set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
2012
	put_css_set_locked(old_cset, false);
B
Ben Blum 已提交
2013 2014
}

L
Li Zefan 已提交
2015
/**
2016 2017
 * cgroup_migrate_finish - cleanup after attach
 * @preloaded_csets: list of preloaded css_sets
B
Ben Blum 已提交
2018
 *
2019 2020
 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
 * those functions for details.
B
Ben Blum 已提交
2021
 */
2022
static void cgroup_migrate_finish(struct list_head *preloaded_csets)
B
Ben Blum 已提交
2023
{
2024
	struct css_set *cset, *tmp_cset;
B
Ben Blum 已提交
2025

2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
	lockdep_assert_held(&cgroup_mutex);

	down_write(&css_set_rwsem);
	list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
		cset->mg_src_cgrp = NULL;
		cset->mg_dst_cset = NULL;
		list_del_init(&cset->mg_preload_node);
		put_css_set_locked(cset, false);
	}
	up_write(&css_set_rwsem);
}

/**
 * cgroup_migrate_add_src - add a migration source css_set
 * @src_cset: the source css_set to add
 * @dst_cgrp: the destination cgroup
 * @preloaded_csets: list of preloaded css_sets
 *
 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
 * @src_cset and add it to @preloaded_csets, which should later be cleaned
 * up by cgroup_migrate_finish().
 *
 * This function may be called without holding threadgroup_lock even if the
 * target is a process.  Threads may be created and destroyed but as long
 * as cgroup_mutex is not dropped, no new css_set can be put into play and
 * the preloaded css_sets are guaranteed to cover all migrations.
 */
static void cgroup_migrate_add_src(struct css_set *src_cset,
				   struct cgroup *dst_cgrp,
				   struct list_head *preloaded_csets)
{
	struct cgroup *src_cgrp;

	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&css_set_rwsem);

	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);

	if (!list_empty(&src_cset->mg_preload_node))
		return;

	WARN_ON(src_cset->mg_src_cgrp);
	WARN_ON(!list_empty(&src_cset->mg_tasks));
	WARN_ON(!list_empty(&src_cset->mg_node));

	src_cset->mg_src_cgrp = src_cgrp;
	get_css_set(src_cset);
	list_add(&src_cset->mg_preload_node, preloaded_csets);
}

/**
 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2078
 * @dst_cgrp: the destination cgroup (may be %NULL)
2079 2080 2081 2082
 * @preloaded_csets: list of preloaded source css_sets
 *
 * Tasks are about to be moved to @dst_cgrp and all the source css_sets
 * have been preloaded to @preloaded_csets.  This function looks up and
2083 2084 2085
 * pins all destination css_sets, links each to its source, and append them
 * to @preloaded_csets.  If @dst_cgrp is %NULL, the destination of each
 * source css_set is assumed to be its cgroup on the default hierarchy.
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
 *
 * This function must be called after cgroup_migrate_add_src() has been
 * called on each migration source css_set.  After migration is performed
 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
 * @preloaded_csets.
 */
static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
				      struct list_head *preloaded_csets)
{
	LIST_HEAD(csets);
2096
	struct css_set *src_cset, *tmp_cset;
2097 2098 2099

	lockdep_assert_held(&cgroup_mutex);

2100 2101 2102 2103
	/*
	 * Except for the root, child_subsys_mask must be zero for a cgroup
	 * with tasks so that child cgroups don't compete against tasks.
	 */
T
Tejun Heo 已提交
2104
	if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
2105 2106 2107
	    dst_cgrp->child_subsys_mask)
		return -EBUSY;

2108
	/* look up the dst cset for each src cset and link it to src */
2109
	list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
2110 2111
		struct css_set *dst_cset;

2112 2113
		dst_cset = find_css_set(src_cset,
					dst_cgrp ?: src_cset->dfl_cgrp);
2114 2115 2116 2117
		if (!dst_cset)
			goto err;

		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131

		/*
		 * If src cset equals dst, it's noop.  Drop the src.
		 * cgroup_migrate() will skip the cset too.  Note that we
		 * can't handle src == dst as some nodes are used by both.
		 */
		if (src_cset == dst_cset) {
			src_cset->mg_src_cgrp = NULL;
			list_del_init(&src_cset->mg_preload_node);
			put_css_set(src_cset, false);
			put_css_set(dst_cset, false);
			continue;
		}

2132 2133 2134 2135 2136 2137 2138 2139
		src_cset->mg_dst_cset = dst_cset;

		if (list_empty(&dst_cset->mg_preload_node))
			list_add(&dst_cset->mg_preload_node, &csets);
		else
			put_css_set(dst_cset, false);
	}

2140
	list_splice_tail(&csets, preloaded_csets);
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166
	return 0;
err:
	cgroup_migrate_finish(&csets);
	return -ENOMEM;
}

/**
 * cgroup_migrate - migrate a process or task to a cgroup
 * @cgrp: the destination cgroup
 * @leader: the leader of the process or the task to migrate
 * @threadgroup: whether @leader points to the whole process or a single task
 *
 * Migrate a process or task denoted by @leader to @cgrp.  If migrating a
 * process, the caller must be holding threadgroup_lock of @leader.  The
 * caller is also responsible for invoking cgroup_migrate_add_src() and
 * cgroup_migrate_prepare_dst() on the targets before invoking this
 * function and following up with cgroup_migrate_finish().
 *
 * As long as a controller's ->can_attach() doesn't fail, this function is
 * guaranteed to succeed.  This means that, excluding ->can_attach()
 * failure, when migrating multiple targets, the success or failure can be
 * decided for all targets by invoking group_migrate_prepare_dst() before
 * actually starting migrating.
 */
static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader,
			  bool threadgroup)
B
Ben Blum 已提交
2167
{
2168 2169 2170 2171 2172
	struct cgroup_taskset tset = {
		.src_csets	= LIST_HEAD_INIT(tset.src_csets),
		.dst_csets	= LIST_HEAD_INIT(tset.dst_csets),
		.csets		= &tset.src_csets,
	};
T
Tejun Heo 已提交
2173
	struct cgroup_subsys_state *css, *failed_css = NULL;
2174 2175 2176
	struct css_set *cset, *tmp_cset;
	struct task_struct *task, *tmp_task;
	int i, ret;
B
Ben Blum 已提交
2177

2178 2179 2180 2181 2182
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
2183
	down_write(&css_set_rwsem);
2184
	rcu_read_lock();
2185
	task = leader;
B
Ben Blum 已提交
2186
	do {
2187 2188
		/* @task either already exited or can't exit until the end */
		if (task->flags & PF_EXITING)
2189
			goto next;
2190

2191 2192
		/* leave @task alone if post_fork() hasn't linked it yet */
		if (list_empty(&task->cg_list))
2193
			goto next;
2194

2195
		cset = task_css_set(task);
2196
		if (!cset->mg_src_cgrp)
2197
			goto next;
2198

2199
		/*
2200 2201
		 * cgroup_taskset_first() must always return the leader.
		 * Take care to avoid disturbing the ordering.
2202
		 */
2203 2204 2205 2206 2207 2208
		list_move_tail(&task->cg_list, &cset->mg_tasks);
		if (list_empty(&cset->mg_node))
			list_add_tail(&cset->mg_node, &tset.src_csets);
		if (list_empty(&cset->mg_dst_cset->mg_node))
			list_move_tail(&cset->mg_dst_cset->mg_node,
				       &tset.dst_csets);
2209
	next:
2210 2211
		if (!threadgroup)
			break;
2212
	} while_each_thread(leader, task);
2213
	rcu_read_unlock();
2214
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2215

2216
	/* methods shouldn't be called if no task is actually migrating */
2217 2218
	if (list_empty(&tset.src_csets))
		return 0;
2219

2220
	/* check that we can legitimately attach to the cgroup */
2221
	for_each_e_css(css, i, cgrp) {
T
Tejun Heo 已提交
2222
		if (css->ss->can_attach) {
2223 2224
			ret = css->ss->can_attach(css, &tset);
			if (ret) {
T
Tejun Heo 已提交
2225
				failed_css = css;
B
Ben Blum 已提交
2226 2227 2228 2229 2230 2231
				goto out_cancel_attach;
			}
		}
	}

	/*
2232 2233 2234
	 * Now that we're guaranteed success, proceed to move all tasks to
	 * the new cgroup.  There are no failure cases after here, so this
	 * is the commit point.
B
Ben Blum 已提交
2235
	 */
2236
	down_write(&css_set_rwsem);
2237 2238 2239 2240
	list_for_each_entry(cset, &tset.src_csets, mg_node) {
		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list)
			cgroup_task_migrate(cset->mg_src_cgrp, task,
					    cset->mg_dst_cset);
B
Ben Blum 已提交
2241
	}
2242
	up_write(&css_set_rwsem);
B
Ben Blum 已提交
2243 2244

	/*
2245 2246 2247
	 * Migration is committed, all target tasks are now on dst_csets.
	 * Nothing is sensitive to fork() after this point.  Notify
	 * controllers that migration is complete.
B
Ben Blum 已提交
2248
	 */
2249
	tset.csets = &tset.dst_csets;
B
Ben Blum 已提交
2250

2251
	for_each_e_css(css, i, cgrp)
T
Tejun Heo 已提交
2252 2253
		if (css->ss->attach)
			css->ss->attach(css, &tset);
B
Ben Blum 已提交
2254

2255
	ret = 0;
2256 2257
	goto out_release_tset;

B
Ben Blum 已提交
2258
out_cancel_attach:
2259
	for_each_e_css(css, i, cgrp) {
2260 2261 2262 2263
		if (css == failed_css)
			break;
		if (css->ss->cancel_attach)
			css->ss->cancel_attach(css, &tset);
B
Ben Blum 已提交
2264
	}
2265 2266 2267 2268
out_release_tset:
	down_write(&css_set_rwsem);
	list_splice_init(&tset.dst_csets, &tset.src_csets);
	list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) {
2269
		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2270 2271 2272
		list_del_init(&cset->mg_node);
	}
	up_write(&css_set_rwsem);
2273
	return ret;
B
Ben Blum 已提交
2274 2275
}

2276 2277 2278 2279 2280 2281
/**
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
 * @dst_cgrp: the cgroup to attach to
 * @leader: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
 *
2282
 * Call holding cgroup_mutex and threadgroup_lock of @leader.
2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
 */
static int cgroup_attach_task(struct cgroup *dst_cgrp,
			      struct task_struct *leader, bool threadgroup)
{
	LIST_HEAD(preloaded_csets);
	struct task_struct *task;
	int ret;

	/* look up all src csets */
	down_read(&css_set_rwsem);
	rcu_read_lock();
	task = leader;
	do {
		cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
				       &preloaded_csets);
		if (!threadgroup)
			break;
	} while_each_thread(leader, task);
	rcu_read_unlock();
	up_read(&css_set_rwsem);

	/* prepare dst csets and commit */
	ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
	if (!ret)
		ret = cgroup_migrate(dst_cgrp, leader, threadgroup);

	cgroup_migrate_finish(&preloaded_csets);
	return ret;
B
Ben Blum 已提交
2311 2312 2313 2314
}

/*
 * Find the task_struct of the task to attach by vpid and pass it along to the
2315
 * function to attach either it or all tasks in its threadgroup. Will lock
2316
 * cgroup_mutex and threadgroup.
2317
 */
2318 2319
static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
				    size_t nbytes, loff_t off, bool threadgroup)
2320 2321
{
	struct task_struct *tsk;
2322
	const struct cred *cred = current_cred(), *tcred;
2323
	struct cgroup *cgrp;
2324
	pid_t pid;
2325 2326
	int ret;

2327 2328 2329
	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
		return -EINVAL;

2330 2331
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
B
Ben Blum 已提交
2332 2333
		return -ENODEV;

2334 2335
retry_find_task:
	rcu_read_lock();
2336
	if (pid) {
2337
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2338 2339
		if (!tsk) {
			rcu_read_unlock();
S
SeongJae Park 已提交
2340
			ret = -ESRCH;
2341
			goto out_unlock_cgroup;
2342
		}
B
Ben Blum 已提交
2343 2344 2345 2346
		/*
		 * even if we're attaching all tasks in the thread group, we
		 * only need to check permissions on one of them.
		 */
2347
		tcred = __task_cred(tsk);
2348 2349 2350
		if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
		    !uid_eq(cred->euid, tcred->uid) &&
		    !uid_eq(cred->euid, tcred->suid)) {
2351
			rcu_read_unlock();
2352 2353
			ret = -EACCES;
			goto out_unlock_cgroup;
2354
		}
2355 2356
	} else
		tsk = current;
2357 2358

	if (threadgroup)
2359
		tsk = tsk->group_leader;
2360 2361

	/*
2362
	 * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2363 2364 2365
	 * trapped in a cpuset, or RT worker may be born in a cgroup
	 * with no rt_runtime allocated.  Just say no.
	 */
2366
	if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2367 2368 2369 2370 2371
		ret = -EINVAL;
		rcu_read_unlock();
		goto out_unlock_cgroup;
	}

2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
	get_task_struct(tsk);
	rcu_read_unlock();

	threadgroup_lock(tsk);
	if (threadgroup) {
		if (!thread_group_leader(tsk)) {
			/*
			 * a race with de_thread from another thread's exec()
			 * may strip us of our leadership, if this happens,
			 * there is no choice but to throw this task away and
			 * try again; this is
			 * "double-double-toil-and-trouble-check locking".
			 */
			threadgroup_unlock(tsk);
			put_task_struct(tsk);
			goto retry_find_task;
		}
2389 2390 2391 2392
	}

	ret = cgroup_attach_task(cgrp, tsk, threadgroup);

2393 2394
	threadgroup_unlock(tsk);

2395
	put_task_struct(tsk);
2396
out_unlock_cgroup:
2397
	cgroup_kn_unlock(of->kn);
2398
	return ret ?: nbytes;
2399 2400
}

2401 2402 2403 2404 2405 2406 2407
/**
 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
 * @from: attach to all cgroups of a given task
 * @tsk: the task to be attached
 */
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
{
2408
	struct cgroup_root *root;
2409 2410
	int retval = 0;

T
Tejun Heo 已提交
2411
	mutex_lock(&cgroup_mutex);
2412
	for_each_root(root) {
2413 2414
		struct cgroup *from_cgrp;

2415
		if (root == &cgrp_dfl_root)
2416 2417
			continue;

2418 2419 2420
		down_read(&css_set_rwsem);
		from_cgrp = task_cgroup_from_root(from, root);
		up_read(&css_set_rwsem);
2421

L
Li Zefan 已提交
2422
		retval = cgroup_attach_task(from_cgrp, tsk, false);
2423 2424 2425
		if (retval)
			break;
	}
T
Tejun Heo 已提交
2426
	mutex_unlock(&cgroup_mutex);
2427 2428 2429 2430 2431

	return retval;
}
EXPORT_SYMBOL_GPL(cgroup_attach_task_all);

2432 2433
static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
B
Ben Blum 已提交
2434
{
2435
	return __cgroup_procs_write(of, buf, nbytes, off, false);
B
Ben Blum 已提交
2436 2437
}

2438 2439
static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
2440
{
2441
	return __cgroup_procs_write(of, buf, nbytes, off, true);
2442 2443
}

2444 2445
static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
					  char *buf, size_t nbytes, loff_t off)
2446
{
2447
	struct cgroup *cgrp;
2448

2449
	BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
2450

2451 2452
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
2453
		return -ENODEV;
2454
	spin_lock(&release_agent_path_lock);
2455 2456
	strlcpy(cgrp->root->release_agent_path, strstrip(buf),
		sizeof(cgrp->root->release_agent_path));
2457
	spin_unlock(&release_agent_path_lock);
2458
	cgroup_kn_unlock(of->kn);
2459
	return nbytes;
2460 2461
}

2462
static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2463
{
2464
	struct cgroup *cgrp = seq_css(seq)->cgroup;
2465

2466
	spin_lock(&release_agent_path_lock);
2467
	seq_puts(seq, cgrp->root->release_agent_path);
2468
	spin_unlock(&release_agent_path_lock);
2469 2470 2471 2472
	seq_putc(seq, '\n');
	return 0;
}

2473
static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2474
{
2475
	seq_puts(seq, "0\n");
2476 2477 2478
	return 0;
}

2479
static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask)
2480
{
2481 2482 2483
	struct cgroup_subsys *ss;
	bool printed = false;
	int ssid;
2484

2485 2486 2487 2488 2489 2490 2491
	for_each_subsys(ss, ssid) {
		if (ss_mask & (1 << ssid)) {
			if (printed)
				seq_putc(seq, ' ');
			seq_printf(seq, "%s", ss->name);
			printed = true;
		}
2492
	}
2493 2494
	if (printed)
		seq_putc(seq, '\n');
2495 2496
}

2497 2498
/* show controllers which are currently attached to the default hierarchy */
static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
2499
{
2500 2501
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2502 2503
	cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
			     ~cgrp_dfl_root_inhibit_ss_mask);
2504
	return 0;
2505 2506
}

2507 2508
/* show controllers which are enabled from the parent */
static int cgroup_controllers_show(struct seq_file *seq, void *v)
2509
{
2510 2511
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2512
	cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
2513
	return 0;
2514 2515
}

2516 2517
/* show controllers which are enabled for a given cgroup's children */
static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2518
{
2519 2520
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2521
	cgroup_print_ss_mask(seq, cgrp->subtree_control);
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621
	return 0;
}

/**
 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
 * @cgrp: root of the subtree to update csses for
 *
 * @cgrp's child_subsys_mask has changed and its subtree's (self excluded)
 * css associations need to be updated accordingly.  This function looks up
 * all css_sets which are attached to the subtree, creates the matching
 * updated css_sets and migrates the tasks to the new ones.
 */
static int cgroup_update_dfl_csses(struct cgroup *cgrp)
{
	LIST_HEAD(preloaded_csets);
	struct cgroup_subsys_state *css;
	struct css_set *src_cset;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* look up all csses currently attached to @cgrp's subtree */
	down_read(&css_set_rwsem);
	css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
		struct cgrp_cset_link *link;

		/* self is not affected by child_subsys_mask change */
		if (css->cgroup == cgrp)
			continue;

		list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
			cgroup_migrate_add_src(link->cset, cgrp,
					       &preloaded_csets);
	}
	up_read(&css_set_rwsem);

	/* NULL dst indicates self on default hierarchy */
	ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
	if (ret)
		goto out_finish;

	list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
		struct task_struct *last_task = NULL, *task;

		/* src_csets precede dst_csets, break on the first dst_cset */
		if (!src_cset->mg_src_cgrp)
			break;

		/*
		 * All tasks in src_cset need to be migrated to the
		 * matching dst_cset.  Empty it process by process.  We
		 * walk tasks but migrate processes.  The leader might even
		 * belong to a different cset but such src_cset would also
		 * be among the target src_csets because the default
		 * hierarchy enforces per-process membership.
		 */
		while (true) {
			down_read(&css_set_rwsem);
			task = list_first_entry_or_null(&src_cset->tasks,
						struct task_struct, cg_list);
			if (task) {
				task = task->group_leader;
				WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp);
				get_task_struct(task);
			}
			up_read(&css_set_rwsem);

			if (!task)
				break;

			/* guard against possible infinite loop */
			if (WARN(last_task == task,
				 "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n"))
				goto out_finish;
			last_task = task;

			threadgroup_lock(task);
			/* raced against de_thread() from another thread? */
			if (!thread_group_leader(task)) {
				threadgroup_unlock(task);
				put_task_struct(task);
				continue;
			}

			ret = cgroup_migrate(src_cset->dfl_cgrp, task, true);

			threadgroup_unlock(task);
			put_task_struct(task);

			if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret))
				goto out_finish;
		}
	}

out_finish:
	cgroup_migrate_finish(&preloaded_csets);
	return ret;
}

/* change the enabled child controllers for a cgroup in the default hierarchy */
2622 2623 2624
static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
2625
{
2626
	unsigned int enable = 0, disable = 0;
2627
	unsigned int css_enable, css_disable, old_ctrl, new_ctrl;
2628
	struct cgroup *cgrp, *child;
2629
	struct cgroup_subsys *ss;
2630
	char *tok;
2631 2632 2633
	int ssid, ret;

	/*
2634 2635
	 * Parse input - space separated list of subsystem names prefixed
	 * with either + or -.
2636
	 */
2637 2638
	buf = strstrip(buf);
	while ((tok = strsep(&buf, " "))) {
2639 2640
		if (tok[0] == '\0')
			continue;
2641
		for_each_subsys(ss, ssid) {
2642 2643
			if (ss->disabled || strcmp(tok + 1, ss->name) ||
			    ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask))
2644 2645 2646
				continue;

			if (*tok == '+') {
2647 2648
				enable |= 1 << ssid;
				disable &= ~(1 << ssid);
2649
			} else if (*tok == '-') {
2650 2651
				disable |= 1 << ssid;
				enable &= ~(1 << ssid);
2652 2653 2654 2655 2656 2657 2658 2659 2660
			} else {
				return -EINVAL;
			}
			break;
		}
		if (ssid == CGROUP_SUBSYS_COUNT)
			return -EINVAL;
	}

2661 2662 2663
	cgrp = cgroup_kn_lock_live(of->kn);
	if (!cgrp)
		return -ENODEV;
2664 2665 2666

	for_each_subsys(ss, ssid) {
		if (enable & (1 << ssid)) {
2667
			if (cgrp->subtree_control & (1 << ssid)) {
2668 2669 2670 2671
				enable &= ~(1 << ssid);
				continue;
			}

2672 2673 2674
			/* unavailable or not enabled on the parent? */
			if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
			    (cgroup_parent(cgrp) &&
2675
			     !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
2676 2677 2678 2679
				ret = -ENOENT;
				goto out_unlock;
			}

2680 2681 2682 2683 2684 2685 2686
			/*
			 * @ss is already enabled through dependency and
			 * we'll just make it visible.  Skip draining.
			 */
			if (cgrp->child_subsys_mask & (1 << ssid))
				continue;

2687 2688 2689 2690 2691 2692 2693
			/*
			 * Because css offlining is asynchronous, userland
			 * might try to re-enable the same controller while
			 * the previous instance is still around.  In such
			 * cases, wait till it's gone using offline_waitq.
			 */
			cgroup_for_each_live_child(child, cgrp) {
2694
				DEFINE_WAIT(wait);
2695 2696 2697 2698

				if (!cgroup_css(child, ss))
					continue;

2699
				cgroup_get(child);
2700 2701
				prepare_to_wait(&child->offline_waitq, &wait,
						TASK_UNINTERRUPTIBLE);
2702
				cgroup_kn_unlock(of->kn);
2703 2704
				schedule();
				finish_wait(&child->offline_waitq, &wait);
2705
				cgroup_put(child);
2706

2707
				return restart_syscall();
2708 2709
			}
		} else if (disable & (1 << ssid)) {
2710
			if (!(cgrp->subtree_control & (1 << ssid))) {
2711 2712 2713 2714 2715 2716
				disable &= ~(1 << ssid);
				continue;
			}

			/* a child has it enabled? */
			cgroup_for_each_live_child(child, cgrp) {
2717
				if (child->subtree_control & (1 << ssid)) {
2718
					ret = -EBUSY;
2719
					goto out_unlock;
2720 2721 2722 2723 2724 2725 2726
				}
			}
		}
	}

	if (!enable && !disable) {
		ret = 0;
2727
		goto out_unlock;
2728 2729 2730
	}

	/*
2731
	 * Except for the root, subtree_control must be zero for a cgroup
2732 2733
	 * with tasks so that child cgroups don't compete against tasks.
	 */
T
Tejun Heo 已提交
2734
	if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
2735 2736 2737 2738
		ret = -EBUSY;
		goto out_unlock;
	}

2739 2740 2741 2742 2743
	/*
	 * Update subsys masks and calculate what needs to be done.  More
	 * subsystems than specified may need to be enabled or disabled
	 * depending on subsystem dependencies.
	 */
2744 2745
	cgrp->subtree_control |= enable;
	cgrp->subtree_control &= ~disable;
2746 2747

	old_ctrl = cgrp->child_subsys_mask;
2748
	cgroup_refresh_child_subsys_mask(cgrp);
2749 2750 2751 2752 2753 2754
	new_ctrl = cgrp->child_subsys_mask;

	css_enable = ~old_ctrl & new_ctrl;
	css_disable = old_ctrl & ~new_ctrl;
	enable |= css_enable;
	disable |= css_disable;
2755

2756 2757 2758 2759 2760 2761
	/*
	 * Create new csses or make the existing ones visible.  A css is
	 * created invisible if it's being implicitly enabled through
	 * dependency.  An invisible css is made visible when the userland
	 * explicitly enables it.
	 */
2762 2763 2764 2765 2766
	for_each_subsys(ss, ssid) {
		if (!(enable & (1 << ssid)))
			continue;

		cgroup_for_each_live_child(child, cgrp) {
2767 2768 2769 2770 2771
			if (css_enable & (1 << ssid))
				ret = create_css(child, ss,
					cgrp->subtree_control & (1 << ssid));
			else
				ret = cgroup_populate_dir(child, 1 << ssid);
2772 2773 2774 2775 2776
			if (ret)
				goto err_undo_css;
		}
	}

2777 2778 2779 2780 2781
	/*
	 * At this point, cgroup_e_css() results reflect the new csses
	 * making the following cgroup_update_dfl_csses() properly update
	 * css associations of all tasks in the subtree.
	 */
2782 2783 2784 2785
	ret = cgroup_update_dfl_csses(cgrp);
	if (ret)
		goto err_undo_css;

2786 2787 2788
	/*
	 * All tasks are migrated out of disabled csses.  Kill or hide
	 * them.  A css is hidden when the userland requests it to be
2789 2790 2791 2792
	 * disabled while other subsystems are still depending on it.  The
	 * css must not actively control resources and be in the vanilla
	 * state if it's made visible again later.  Controllers which may
	 * be depended upon should provide ->css_reset() for this purpose.
2793
	 */
2794 2795 2796 2797
	for_each_subsys(ss, ssid) {
		if (!(disable & (1 << ssid)))
			continue;

2798
		cgroup_for_each_live_child(child, cgrp) {
2799 2800 2801 2802 2803
			struct cgroup_subsys_state *css = cgroup_css(child, ss);

			if (css_disable & (1 << ssid)) {
				kill_css(css);
			} else {
2804
				cgroup_clear_dir(child, 1 << ssid);
2805 2806 2807
				if (ss->css_reset)
					ss->css_reset(css);
			}
2808
		}
2809 2810 2811 2812 2813
	}

	kernfs_activate(cgrp->kn);
	ret = 0;
out_unlock:
2814
	cgroup_kn_unlock(of->kn);
2815
	return ret ?: nbytes;
2816 2817

err_undo_css:
2818 2819 2820
	cgrp->subtree_control &= ~enable;
	cgrp->subtree_control |= disable;
	cgroup_refresh_child_subsys_mask(cgrp);
2821 2822 2823 2824 2825 2826 2827

	for_each_subsys(ss, ssid) {
		if (!(enable & (1 << ssid)))
			continue;

		cgroup_for_each_live_child(child, cgrp) {
			struct cgroup_subsys_state *css = cgroup_css(child, ss);
2828 2829 2830 2831 2832

			if (!css)
				continue;

			if (css_enable & (1 << ssid))
2833
				kill_css(css);
2834 2835
			else
				cgroup_clear_dir(child, 1 << ssid);
2836 2837 2838 2839 2840
		}
	}
	goto out_unlock;
}

2841 2842 2843 2844 2845 2846
static int cgroup_populated_show(struct seq_file *seq, void *v)
{
	seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt);
	return 0;
}

T
Tejun Heo 已提交
2847 2848
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
2849
{
T
Tejun Heo 已提交
2850 2851 2852
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = of->kn->priv;
	struct cgroup_subsys_state *css;
2853
	int ret;
2854

T
Tejun Heo 已提交
2855 2856 2857
	if (cft->write)
		return cft->write(of, buf, nbytes, off);

T
Tejun Heo 已提交
2858 2859 2860 2861 2862 2863 2864 2865 2866
	/*
	 * kernfs guarantees that a file isn't deleted with operations in
	 * flight, which means that the matching css is and stays alive and
	 * doesn't need to be pinned.  The RCU locking is not necessary
	 * either.  It's just for the convenience of using cgroup_css().
	 */
	rcu_read_lock();
	css = cgroup_css(cgrp, cft->ss);
	rcu_read_unlock();
2867

2868
	if (cft->write_u64) {
2869 2870 2871 2872 2873 2874 2875 2876 2877
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
2878
	} else {
2879
		ret = -EINVAL;
2880
	}
T
Tejun Heo 已提交
2881

2882
	return ret ?: nbytes;
2883 2884
}

2885
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
2886
{
T
Tejun Heo 已提交
2887
	return seq_cft(seq)->seq_start(seq, ppos);
2888 2889
}

2890
static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
2891
{
T
Tejun Heo 已提交
2892
	return seq_cft(seq)->seq_next(seq, v, ppos);
2893 2894
}

2895
static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
2896
{
T
Tejun Heo 已提交
2897
	seq_cft(seq)->seq_stop(seq, v);
2898 2899
}

2900
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
2901
{
2902 2903
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
2904

2905 2906
	if (cft->seq_show)
		return cft->seq_show(m, arg);
2907

2908
	if (cft->read_u64)
2909 2910 2911 2912 2913 2914
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
2915 2916
}

T
Tejun Heo 已提交
2917 2918 2919 2920
static struct kernfs_ops cgroup_kf_single_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_show		= cgroup_seqfile_show,
2921 2922
};

T
Tejun Heo 已提交
2923 2924 2925 2926 2927 2928 2929 2930
static struct kernfs_ops cgroup_kf_ops = {
	.atomic_write_len	= PAGE_SIZE,
	.write			= cgroup_file_write,
	.seq_start		= cgroup_seqfile_start,
	.seq_next		= cgroup_seqfile_next,
	.seq_stop		= cgroup_seqfile_stop,
	.seq_show		= cgroup_seqfile_show,
};
2931 2932 2933 2934

/*
 * cgroup_rename - Only allow simple rename of directories in place.
 */
T
Tejun Heo 已提交
2935 2936
static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
			 const char *new_name_str)
2937
{
T
Tejun Heo 已提交
2938
	struct cgroup *cgrp = kn->priv;
2939 2940
	int ret;

T
Tejun Heo 已提交
2941
	if (kernfs_type(kn) != KERNFS_DIR)
2942
		return -ENOTDIR;
T
Tejun Heo 已提交
2943
	if (kn->parent != new_parent)
2944
		return -EIO;
2945

2946 2947
	/*
	 * This isn't a proper migration and its usefulness is very
2948
	 * limited.  Disallow on the default hierarchy.
2949
	 */
2950
	if (cgroup_on_dfl(cgrp))
2951
		return -EPERM;
L
Li Zefan 已提交
2952

2953
	/*
T
Tejun Heo 已提交
2954
	 * We're gonna grab cgroup_mutex which nests outside kernfs
2955
	 * active_ref.  kernfs_rename() doesn't require active_ref
T
Tejun Heo 已提交
2956
	 * protection.  Break them before grabbing cgroup_mutex.
2957 2958 2959
	 */
	kernfs_break_active_protection(new_parent);
	kernfs_break_active_protection(kn);
L
Li Zefan 已提交
2960

T
Tejun Heo 已提交
2961
	mutex_lock(&cgroup_mutex);
L
Li Zefan 已提交
2962

T
Tejun Heo 已提交
2963
	ret = kernfs_rename(kn, new_parent, new_name_str);
L
Li Zefan 已提交
2964

T
Tejun Heo 已提交
2965
	mutex_unlock(&cgroup_mutex);
2966 2967 2968

	kernfs_unbreak_active_protection(kn);
	kernfs_unbreak_active_protection(new_parent);
T
Tejun Heo 已提交
2969
	return ret;
L
Li Zefan 已提交
2970 2971
}

2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985
/* set uid and gid of cgroup dirs and files to that of the creator */
static int cgroup_kn_set_ugid(struct kernfs_node *kn)
{
	struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
			       .ia_uid = current_fsuid(),
			       .ia_gid = current_fsgid(), };

	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
	    gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
		return 0;

	return kernfs_setattr(kn, &iattr);
}

2986
static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2987
{
T
Tejun Heo 已提交
2988
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
2989 2990
	struct kernfs_node *kn;
	struct lock_class_key *key = NULL;
2991
	int ret;
T
Tejun Heo 已提交
2992

T
Tejun Heo 已提交
2993 2994 2995 2996 2997 2998
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	key = &cft->lockdep_key;
#endif
	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
				  cgroup_file_mode(cft), 0, cft->kf_ops, cft,
				  NULL, false, key);
2999 3000 3001 3002
	if (IS_ERR(kn))
		return PTR_ERR(kn);

	ret = cgroup_kn_set_ugid(kn);
3003
	if (ret) {
3004
		kernfs_remove(kn);
3005 3006 3007
		return ret;
	}

T
Tejun Heo 已提交
3008
	if (cft->seq_show == cgroup_populated_show)
3009
		cgrp->populated_kn = kn;
3010
	return 0;
3011 3012
}

3013 3014 3015 3016 3017 3018 3019
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
 * @cgrp: the target cgroup
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3020 3021 3022
 * For removals, this function never fails.  If addition fails, this
 * function doesn't remove files already added.  The caller is responsible
 * for cleaning up.
3023
 */
3024 3025
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add)
3026
{
A
Aristeu Rozanski 已提交
3027
	struct cftype *cft;
3028 3029
	int ret;

3030
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
3031 3032

	for (cft = cfts; cft->name[0] != '\0'; cft++) {
3033
		/* does cft->flags tell us to skip this file on @cgrp? */
T
Tejun Heo 已提交
3034 3035
		if ((cft->flags & CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
			continue;
3036
		if ((cft->flags & CFTYPE_INSANE) && cgroup_on_dfl(cgrp))
3037
			continue;
T
Tejun Heo 已提交
3038
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3039
			continue;
T
Tejun Heo 已提交
3040
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3041 3042
			continue;

3043
		if (is_add) {
3044
			ret = cgroup_add_file(cgrp, cft);
3045
			if (ret) {
3046 3047
				pr_warn("%s: failed to add %s, err=%d\n",
					__func__, cft->name, ret);
3048 3049
				return ret;
			}
3050 3051
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
3052
		}
3053
	}
3054
	return 0;
3055 3056
}

3057
static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3058 3059
{
	LIST_HEAD(pending);
3060
	struct cgroup_subsys *ss = cfts[0].ss;
3061
	struct cgroup *root = &ss->root->cgrp;
3062
	struct cgroup_subsys_state *css;
3063
	int ret = 0;
3064

3065
	lockdep_assert_held(&cgroup_mutex);
3066 3067

	/* add/rm files for all cgroups created before */
3068
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3069 3070
		struct cgroup *cgrp = css->cgroup;

3071 3072 3073
		if (cgroup_is_dead(cgrp))
			continue;

3074
		ret = cgroup_addrm_files(cgrp, cfts, is_add);
3075 3076
		if (ret)
			break;
3077
	}
3078 3079 3080

	if (is_add && !ret)
		kernfs_activate(root->kn);
3081
	return ret;
3082 3083
}

3084
static void cgroup_exit_cftypes(struct cftype *cfts)
3085
{
3086
	struct cftype *cft;
3087

T
Tejun Heo 已提交
3088 3089 3090 3091 3092
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		/* free copy for custom atomic_write_len, see init_cftypes() */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
			kfree(cft->kf_ops);
		cft->kf_ops = NULL;
3093
		cft->ss = NULL;
3094 3095 3096

		/* revert flags set by cgroup core while adding @cfts */
		cft->flags &= ~(CFTYPE_ONLY_ON_DFL | CFTYPE_INSANE);
T
Tejun Heo 已提交
3097
	}
3098 3099
}

T
Tejun Heo 已提交
3100
static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3101 3102 3103
{
	struct cftype *cft;

T
Tejun Heo 已提交
3104 3105 3106
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		struct kernfs_ops *kf_ops;

T
Tejun Heo 已提交
3107 3108
		WARN_ON(cft->ss || cft->kf_ops);

T
Tejun Heo 已提交
3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125
		if (cft->seq_start)
			kf_ops = &cgroup_kf_ops;
		else
			kf_ops = &cgroup_kf_single_ops;

		/*
		 * Ugh... if @cft wants a custom max_write_len, we need to
		 * make a copy of kf_ops to set its atomic_write_len.
		 */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
			kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
			if (!kf_ops) {
				cgroup_exit_cftypes(cfts);
				return -ENOMEM;
			}
			kf_ops->atomic_write_len = cft->max_write_len;
		}
3126

T
Tejun Heo 已提交
3127
		cft->kf_ops = kf_ops;
3128
		cft->ss = ss;
T
Tejun Heo 已提交
3129
	}
3130

T
Tejun Heo 已提交
3131
	return 0;
3132 3133
}

3134 3135
static int cgroup_rm_cftypes_locked(struct cftype *cfts)
{
3136
	lockdep_assert_held(&cgroup_mutex);
3137 3138 3139 3140 3141 3142 3143 3144

	if (!cfts || !cfts[0].ss)
		return -ENOENT;

	list_del(&cfts->node);
	cgroup_apply_cftypes(cfts, false);
	cgroup_exit_cftypes(cfts);
	return 0;
3145 3146
}

3147 3148 3149 3150
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
3151 3152 3153
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
3154 3155
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3156
 * registered.
3157
 */
3158
int cgroup_rm_cftypes(struct cftype *cfts)
3159
{
3160
	int ret;
3161

3162
	mutex_lock(&cgroup_mutex);
3163
	ret = cgroup_rm_cftypes_locked(cfts);
3164
	mutex_unlock(&cgroup_mutex);
3165
	return ret;
T
Tejun Heo 已提交
3166 3167
}

3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181
/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
3182
static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3183
{
3184
	int ret;
3185

3186 3187 3188
	if (ss->disabled)
		return 0;

3189 3190
	if (!cfts || cfts[0].name[0] == '\0')
		return 0;
3191

T
Tejun Heo 已提交
3192 3193 3194
	ret = cgroup_init_cftypes(ss, cfts);
	if (ret)
		return ret;
3195

3196
	mutex_lock(&cgroup_mutex);
3197

T
Tejun Heo 已提交
3198
	list_add_tail(&cfts->node, &ss->cfts);
3199
	ret = cgroup_apply_cftypes(cfts, true);
3200
	if (ret)
3201
		cgroup_rm_cftypes_locked(cfts);
3202

3203
	mutex_unlock(&cgroup_mutex);
3204
	return ret;
3205 3206
}

3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231
/**
 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the default hierarchy.
 */
int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
	struct cftype *cft;

	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
		cft->flags |= CFTYPE_ONLY_ON_DFL;
	return cgroup_add_cftypes(ss, cfts);
}

/**
 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the legacy hierarchies.
 */
3232 3233
int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
3234 3235 3236 3237
	struct cftype *cft;

	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
		cft->flags |= CFTYPE_INSANE;
3238 3239 3240
	return cgroup_add_cftypes(ss, cfts);
}

L
Li Zefan 已提交
3241 3242 3243 3244 3245 3246
/**
 * cgroup_task_count - count the number of tasks in a cgroup.
 * @cgrp: the cgroup in question
 *
 * Return the number of tasks in the cgroup.
 */
3247
static int cgroup_task_count(const struct cgroup *cgrp)
3248 3249
{
	int count = 0;
3250
	struct cgrp_cset_link *link;
3251

3252
	down_read(&css_set_rwsem);
3253 3254
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		count += atomic_read(&link->cset->refcount);
3255
	up_read(&css_set_rwsem);
3256 3257 3258
	return count;
}

3259
/**
3260
 * css_next_child - find the next child of a given css
3261 3262
 * @pos: the current position (%NULL to initiate traversal)
 * @parent: css whose children to walk
3263
 *
3264
 * This function returns the next child of @parent and should be called
3265
 * under either cgroup_mutex or RCU read lock.  The only requirement is
3266 3267 3268 3269 3270 3271 3272 3273 3274
 * that @parent and @pos are accessible.  The next sibling is guaranteed to
 * be returned regardless of their states.
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3275
 */
3276 3277
struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
					   struct cgroup_subsys_state *parent)
3278
{
3279
	struct cgroup_subsys_state *next;
3280

T
Tejun Heo 已提交
3281
	cgroup_assert_mutex_or_rcu_locked();
3282 3283

	/*
3284 3285 3286 3287 3288 3289 3290 3291 3292 3293
	 * @pos could already have been unlinked from the sibling list.
	 * Once a cgroup is removed, its ->sibling.next is no longer
	 * updated when its next sibling changes.  CSS_RELEASED is set when
	 * @pos is taken off list, at which time its next pointer is valid,
	 * and, as releases are serialized, the one pointed to by the next
	 * pointer is guaranteed to not have started release yet.  This
	 * implies that if we observe !CSS_RELEASED on @pos in this RCU
	 * critical section, the one pointed to by its next pointer is
	 * guaranteed to not have finished its RCU grace period even if we
	 * have dropped rcu_read_lock() inbetween iterations.
3294
	 *
3295 3296 3297 3298 3299 3300 3301
	 * If @pos has CSS_RELEASED set, its next pointer can't be
	 * dereferenced; however, as each css is given a monotonically
	 * increasing unique serial number and always appended to the
	 * sibling list, the next one can be found by walking the parent's
	 * children until the first css with higher serial number than
	 * @pos's.  While this path can be slower, it happens iff iteration
	 * races against release and the race window is very small.
3302
	 */
3303
	if (!pos) {
3304 3305 3306
		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
	} else if (likely(!(pos->flags & CSS_RELEASED))) {
		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3307
	} else {
3308
		list_for_each_entry_rcu(next, &parent->children, sibling)
3309 3310
			if (next->serial_nr > pos->serial_nr)
				break;
3311 3312
	}

3313 3314
	/*
	 * @next, if not pointing to the head, can be dereferenced and is
3315
	 * the next sibling.
3316
	 */
3317 3318
	if (&next->sibling != &parent->children)
		return next;
3319
	return NULL;
3320 3321
}

3322
/**
3323
 * css_next_descendant_pre - find the next descendant for pre-order walk
3324
 * @pos: the current position (%NULL to initiate traversal)
3325
 * @root: css whose descendants to walk
3326
 *
3327
 * To be used by css_for_each_descendant_pre().  Find the next descendant
3328 3329
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
3330
 *
3331 3332 3333 3334
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
3335 3336 3337 3338 3339 3340 3341
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3342
 */
3343 3344 3345
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
3346
{
3347
	struct cgroup_subsys_state *next;
3348

T
Tejun Heo 已提交
3349
	cgroup_assert_mutex_or_rcu_locked();
3350

3351
	/* if first iteration, visit @root */
3352
	if (!pos)
3353
		return root;
3354 3355

	/* visit the first child if exists */
3356
	next = css_next_child(NULL, pos);
3357 3358 3359 3360
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
3361
	while (pos != root) {
T
Tejun Heo 已提交
3362
		next = css_next_child(pos, pos->parent);
3363
		if (next)
3364
			return next;
T
Tejun Heo 已提交
3365
		pos = pos->parent;
3366
	}
3367 3368 3369 3370

	return NULL;
}

3371
/**
3372 3373
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
3374
 *
3375 3376
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
3377
 * subtree of @pos.
3378
 *
3379 3380 3381 3382
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
3383
 */
3384 3385
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
3386
{
3387
	struct cgroup_subsys_state *last, *tmp;
3388

T
Tejun Heo 已提交
3389
	cgroup_assert_mutex_or_rcu_locked();
3390 3391 3392 3393 3394

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
3395
		css_for_each_child(tmp, last)
3396 3397 3398 3399 3400 3401
			pos = tmp;
	} while (pos);

	return last;
}

3402 3403
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
3404
{
3405
	struct cgroup_subsys_state *last;
3406 3407 3408

	do {
		last = pos;
3409
		pos = css_next_child(NULL, pos);
3410 3411 3412 3413 3414 3415
	} while (pos);

	return last;
}

/**
3416
 * css_next_descendant_post - find the next descendant for post-order walk
3417
 * @pos: the current position (%NULL to initiate traversal)
3418
 * @root: css whose descendants to walk
3419
 *
3420
 * To be used by css_for_each_descendant_post().  Find the next descendant
3421 3422
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
3423
 *
3424 3425 3426 3427 3428
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
3429 3430 3431 3432 3433 3434 3435
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3436
 */
3437 3438 3439
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
3440
{
3441
	struct cgroup_subsys_state *next;
3442

T
Tejun Heo 已提交
3443
	cgroup_assert_mutex_or_rcu_locked();
3444

3445 3446 3447
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
3448

3449 3450 3451 3452
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

3453
	/* if there's an unvisited sibling, visit its leftmost descendant */
T
Tejun Heo 已提交
3454
	next = css_next_child(pos, pos->parent);
3455
	if (next)
3456
		return css_leftmost_descendant(next);
3457 3458

	/* no sibling left, visit parent */
T
Tejun Heo 已提交
3459
	return pos->parent;
3460 3461
}

3462 3463 3464 3465 3466 3467 3468 3469 3470
/**
 * css_has_online_children - does a css have online children
 * @css: the target css
 *
 * Returns %true if @css has any online children; otherwise, %false.  This
 * function can be called from any context but the caller is responsible
 * for synchronizing against on/offlining as necessary.
 */
bool css_has_online_children(struct cgroup_subsys_state *css)
3471
{
3472 3473
	struct cgroup_subsys_state *child;
	bool ret = false;
3474 3475

	rcu_read_lock();
3476 3477 3478 3479
	css_for_each_child(child, css) {
		if (css->flags & CSS_ONLINE) {
			ret = true;
			break;
3480 3481 3482
		}
	}
	rcu_read_unlock();
3483
	return ret;
3484 3485
}

3486
/**
3487
 * css_advance_task_iter - advance a task itererator to the next css_set
3488 3489 3490
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
3491
 */
3492
static void css_advance_task_iter(struct css_task_iter *it)
3493
{
T
Tejun Heo 已提交
3494
	struct list_head *l = it->cset_pos;
3495 3496 3497 3498 3499 3500
	struct cgrp_cset_link *link;
	struct css_set *cset;

	/* Advance to the next non-empty css_set */
	do {
		l = l->next;
T
Tejun Heo 已提交
3501 3502
		if (l == it->cset_head) {
			it->cset_pos = NULL;
3503 3504
			return;
		}
3505 3506 3507 3508 3509 3510 3511 3512

		if (it->ss) {
			cset = container_of(l, struct css_set,
					    e_cset_node[it->ss->id]);
		} else {
			link = list_entry(l, struct cgrp_cset_link, cset_link);
			cset = link->cset;
		}
T
Tejun Heo 已提交
3513 3514
	} while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks));

T
Tejun Heo 已提交
3515
	it->cset_pos = l;
T
Tejun Heo 已提交
3516 3517

	if (!list_empty(&cset->tasks))
T
Tejun Heo 已提交
3518
		it->task_pos = cset->tasks.next;
T
Tejun Heo 已提交
3519
	else
T
Tejun Heo 已提交
3520 3521 3522 3523
		it->task_pos = cset->mg_tasks.next;

	it->tasks_head = &cset->tasks;
	it->mg_tasks_head = &cset->mg_tasks;
3524 3525
}

3526
/**
3527 3528
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
3529 3530
 * @it: the task iterator to use
 *
3531 3532 3533 3534
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
3535 3536 3537 3538 3539
 *
 * Note that this function acquires a lock which is released when the
 * iteration finishes.  The caller can't sleep while iteration is in
 * progress.
 */
3540 3541
void css_task_iter_start(struct cgroup_subsys_state *css,
			 struct css_task_iter *it)
3542
	__acquires(css_set_rwsem)
3543
{
3544 3545
	/* no one should try to iterate before mounting cgroups */
	WARN_ON_ONCE(!use_task_css_set_links);
3546

3547
	down_read(&css_set_rwsem);
3548

3549 3550 3551 3552 3553 3554 3555
	it->ss = css->ss;

	if (it->ss)
		it->cset_pos = &css->cgroup->e_csets[css->ss->id];
	else
		it->cset_pos = &css->cgroup->cset_links;

T
Tejun Heo 已提交
3556
	it->cset_head = it->cset_pos;
3557

3558
	css_advance_task_iter(it);
3559 3560
}

3561
/**
3562
 * css_task_iter_next - return the next task for the iterator
3563 3564 3565
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
3566 3567
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
3568
 */
3569
struct task_struct *css_task_iter_next(struct css_task_iter *it)
3570 3571
{
	struct task_struct *res;
T
Tejun Heo 已提交
3572
	struct list_head *l = it->task_pos;
3573 3574

	/* If the iterator cg is NULL, we have no tasks */
T
Tejun Heo 已提交
3575
	if (!it->cset_pos)
3576 3577
		return NULL;
	res = list_entry(l, struct task_struct, cg_list);
T
Tejun Heo 已提交
3578 3579 3580 3581 3582 3583

	/*
	 * Advance iterator to find next entry.  cset->tasks is consumed
	 * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
	 * next cset.
	 */
3584
	l = l->next;
T
Tejun Heo 已提交
3585

T
Tejun Heo 已提交
3586 3587
	if (l == it->tasks_head)
		l = it->mg_tasks_head->next;
T
Tejun Heo 已提交
3588

T
Tejun Heo 已提交
3589
	if (l == it->mg_tasks_head)
3590
		css_advance_task_iter(it);
T
Tejun Heo 已提交
3591
	else
T
Tejun Heo 已提交
3592
		it->task_pos = l;
T
Tejun Heo 已提交
3593

3594 3595 3596
	return res;
}

3597
/**
3598
 * css_task_iter_end - finish task iteration
3599 3600
 * @it: the task iterator to finish
 *
3601
 * Finish task iteration started by css_task_iter_start().
3602
 */
3603
void css_task_iter_end(struct css_task_iter *it)
3604
	__releases(css_set_rwsem)
3605
{
3606
	up_read(&css_set_rwsem);
3607 3608 3609
}

/**
3610 3611 3612
 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
 * @to: cgroup to which the tasks will be moved
 * @from: cgroup in which the tasks currently reside
3613
 *
3614 3615 3616 3617 3618
 * Locking rules between cgroup_post_fork() and the migration path
 * guarantee that, if a task is forking while being migrated, the new child
 * is guaranteed to be either visible in the source cgroup after the
 * parent's migration is complete or put into the target cgroup.  No task
 * can slip out of migration through forking.
3619
 */
3620
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
3621
{
3622 3623
	LIST_HEAD(preloaded_csets);
	struct cgrp_cset_link *link;
3624
	struct css_task_iter it;
3625
	struct task_struct *task;
3626
	int ret;
3627

3628
	mutex_lock(&cgroup_mutex);
3629

3630 3631 3632 3633 3634
	/* all tasks in @from are being moved, all csets are source */
	down_read(&css_set_rwsem);
	list_for_each_entry(link, &from->cset_links, cset_link)
		cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
	up_read(&css_set_rwsem);
3635

3636 3637 3638
	ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
	if (ret)
		goto out_err;
3639

3640 3641 3642 3643
	/*
	 * Migrate tasks one-by-one until @form is empty.  This fails iff
	 * ->can_attach() fails.
	 */
3644
	do {
3645
		css_task_iter_start(&from->self, &it);
3646 3647 3648 3649 3650 3651
		task = css_task_iter_next(&it);
		if (task)
			get_task_struct(task);
		css_task_iter_end(&it);

		if (task) {
3652
			ret = cgroup_migrate(to, task, false);
3653 3654 3655
			put_task_struct(task);
		}
	} while (task && !ret);
3656 3657
out_err:
	cgroup_migrate_finish(&preloaded_csets);
T
Tejun Heo 已提交
3658
	mutex_unlock(&cgroup_mutex);
3659
	return ret;
3660 3661
}

3662
/*
3663
 * Stuff for reading the 'tasks'/'procs' files.
3664 3665 3666 3667 3668 3669 3670 3671
 *
 * Reading this file can return large amounts of data if a cgroup has
 * *lots* of attached tasks. So it may need several calls to read(),
 * but we cannot guarantee that the information we produce is correct
 * unless we produce it entirely atomically.
 *
 */

3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697
/* which pidlist file are we talking about? */
enum cgroup_filetype {
	CGROUP_FILE_PROCS,
	CGROUP_FILE_TASKS,
};

/*
 * A pidlist is a list of pids that virtually represents the contents of one
 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
 * a pair (one each for procs, tasks) for each pid namespace that's relevant
 * to the cgroup.
 */
struct cgroup_pidlist {
	/*
	 * used to find which pidlist is wanted. doesn't change as long as
	 * this particular list stays in the list.
	*/
	struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
	/* array of xids */
	pid_t *list;
	/* how many elements the above list has */
	int length;
	/* each of these stored in a list by its cgroup */
	struct list_head links;
	/* pointer to the cgroup we belong to, for list removal purposes */
	struct cgroup *owner;
3698 3699
	/* for delayed destruction */
	struct delayed_work destroy_dwork;
3700 3701
};

3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
/*
 * The following two functions "fix" the issue where there are more pids
 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
 * TODO: replace with a kernel-wide solution to this problem
 */
#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
static void *pidlist_allocate(int count)
{
	if (PIDLIST_TOO_LARGE(count))
		return vmalloc(count * sizeof(pid_t));
	else
		return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
}
3715

3716 3717 3718 3719 3720 3721 3722 3723
static void pidlist_free(void *p)
{
	if (is_vmalloc_addr(p))
		vfree(p);
	else
		kfree(p);
}

3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750
/*
 * Used to destroy all pidlists lingering waiting for destroy timer.  None
 * should be left afterwards.
 */
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
{
	struct cgroup_pidlist *l, *tmp_l;

	mutex_lock(&cgrp->pidlist_mutex);
	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
	mutex_unlock(&cgrp->pidlist_mutex);

	flush_workqueue(cgroup_pidlist_destroy_wq);
	BUG_ON(!list_empty(&cgrp->pidlists));
}

static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
						destroy_dwork);
	struct cgroup_pidlist *tofree = NULL;

	mutex_lock(&l->owner->pidlist_mutex);

	/*
3751 3752
	 * Destroy iff we didn't get queued again.  The state won't change
	 * as destroy_dwork can only be queued while locked.
3753
	 */
3754
	if (!delayed_work_pending(dwork)) {
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764
		list_del(&l->links);
		pidlist_free(l->list);
		put_pid_ns(l->key.ns);
		tofree = l;
	}

	mutex_unlock(&l->owner->pidlist_mutex);
	kfree(tofree);
}

3765
/*
3766
 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3767
 * Returns the number of unique elements.
3768
 */
3769
static int pidlist_uniq(pid_t *list, int length)
3770
{
3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794
	int src, dest = 1;

	/*
	 * we presume the 0th element is unique, so i starts at 1. trivial
	 * edge cases first; no work needs to be done for either
	 */
	if (length == 0 || length == 1)
		return length;
	/* src and dest walk down the list; dest counts unique elements */
	for (src = 1; src < length; src++) {
		/* find next unique element */
		while (list[src] == list[src-1]) {
			src++;
			if (src == length)
				goto after;
		}
		/* dest always points to where the next unique element goes */
		list[dest] = list[src];
		dest++;
	}
after:
	return dest;
}

3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
/*
 * The two pid files - task and cgroup.procs - guaranteed that the result
 * is sorted, which forced this whole pidlist fiasco.  As pid order is
 * different per namespace, each namespace needs differently sorted list,
 * making it impossible to use, for example, single rbtree of member tasks
 * sorted by task pointer.  As pidlists can be fairly large, allocating one
 * per open file is dangerous, so cgroup had to implement shared pool of
 * pidlists keyed by cgroup and namespace.
 *
 * All this extra complexity was caused by the original implementation
 * committing to an entirely unnecessary property.  In the long term, we
3806 3807 3808
 * want to do away with it.  Explicitly scramble sort order if on the
 * default hierarchy so that no such expectation exists in the new
 * interface.
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
 *
 * Scrambling is done by swapping every two consecutive bits, which is
 * non-identity one-to-one mapping which disturbs sort order sufficiently.
 */
static pid_t pid_fry(pid_t pid)
{
	unsigned a = pid & 0x55555555;
	unsigned b = pid & 0xAAAAAAAA;

	return (a << 1) | (b >> 1);
}

static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
{
3823
	if (cgroup_on_dfl(cgrp))
3824 3825 3826 3827 3828
		return pid_fry(pid);
	else
		return pid;
}

3829 3830 3831 3832 3833
static int cmppid(const void *a, const void *b)
{
	return *(pid_t *)a - *(pid_t *)b;
}

3834 3835 3836 3837 3838
static int fried_cmppid(const void *a, const void *b)
{
	return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
}

T
Tejun Heo 已提交
3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853
static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
						  enum cgroup_filetype type)
{
	struct cgroup_pidlist *l;
	/* don't need task_nsproxy() if we're looking at ourself */
	struct pid_namespace *ns = task_active_pid_ns(current);

	lockdep_assert_held(&cgrp->pidlist_mutex);

	list_for_each_entry(l, &cgrp->pidlists, links)
		if (l->key.type == type && l->key.ns == ns)
			return l;
	return NULL;
}

3854 3855 3856 3857 3858 3859
/*
 * find the appropriate pidlist for our purpose (given procs vs tasks)
 * returns with the lock on that pidlist already held, and takes care
 * of the use count, or returns NULL with no locks held if we're out of
 * memory.
 */
T
Tejun Heo 已提交
3860 3861
static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
						enum cgroup_filetype type)
3862 3863
{
	struct cgroup_pidlist *l;
3864

T
Tejun Heo 已提交
3865 3866 3867 3868 3869 3870
	lockdep_assert_held(&cgrp->pidlist_mutex);

	l = cgroup_pidlist_find(cgrp, type);
	if (l)
		return l;

3871
	/* entry not found; create a new one */
3872
	l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
T
Tejun Heo 已提交
3873
	if (!l)
3874
		return l;
T
Tejun Heo 已提交
3875

3876
	INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3877
	l->key.type = type;
T
Tejun Heo 已提交
3878 3879
	/* don't need task_nsproxy() if we're looking at ourself */
	l->key.ns = get_pid_ns(task_active_pid_ns(current));
3880 3881 3882 3883 3884
	l->owner = cgrp;
	list_add(&l->links, &cgrp->pidlists);
	return l;
}

3885 3886 3887
/*
 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
 */
3888 3889
static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
			      struct cgroup_pidlist **lp)
3890 3891 3892 3893
{
	pid_t *array;
	int length;
	int pid, n = 0; /* used for populating the array */
3894
	struct css_task_iter it;
3895
	struct task_struct *tsk;
3896 3897
	struct cgroup_pidlist *l;

3898 3899
	lockdep_assert_held(&cgrp->pidlist_mutex);

3900 3901 3902 3903 3904 3905 3906
	/*
	 * If cgroup gets more users after we read count, we won't have
	 * enough space - tough.  This race is indistinguishable to the
	 * caller from the case that the additional cgroup users didn't
	 * show up until sometime later on.
	 */
	length = cgroup_task_count(cgrp);
3907
	array = pidlist_allocate(length);
3908 3909 3910
	if (!array)
		return -ENOMEM;
	/* now, populate the array */
3911
	css_task_iter_start(&cgrp->self, &it);
3912
	while ((tsk = css_task_iter_next(&it))) {
3913
		if (unlikely(n == length))
3914
			break;
3915
		/* get tgid or pid for procs or tasks file respectively */
3916 3917 3918 3919
		if (type == CGROUP_FILE_PROCS)
			pid = task_tgid_vnr(tsk);
		else
			pid = task_pid_vnr(tsk);
3920 3921
		if (pid > 0) /* make sure to only use valid results */
			array[n++] = pid;
3922
	}
3923
	css_task_iter_end(&it);
3924 3925
	length = n;
	/* now sort & (if procs) strip out duplicates */
3926
	if (cgroup_on_dfl(cgrp))
3927 3928 3929
		sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
	else
		sort(array, length, sizeof(pid_t), cmppid, NULL);
3930
	if (type == CGROUP_FILE_PROCS)
3931
		length = pidlist_uniq(array, length);
T
Tejun Heo 已提交
3932 3933

	l = cgroup_pidlist_find_create(cgrp, type);
3934
	if (!l) {
T
Tejun Heo 已提交
3935
		mutex_unlock(&cgrp->pidlist_mutex);
3936
		pidlist_free(array);
3937
		return -ENOMEM;
3938
	}
T
Tejun Heo 已提交
3939 3940

	/* store array, freeing old if necessary */
3941
	pidlist_free(l->list);
3942 3943
	l->list = array;
	l->length = length;
3944
	*lp = l;
3945
	return 0;
3946 3947
}

B
Balbir Singh 已提交
3948
/**
L
Li Zefan 已提交
3949
 * cgroupstats_build - build and fill cgroupstats
B
Balbir Singh 已提交
3950 3951 3952
 * @stats: cgroupstats to fill information into
 * @dentry: A dentry entry belonging to the cgroup for which stats have
 * been requested.
L
Li Zefan 已提交
3953 3954 3955
 *
 * Build and fill cgroupstats so that taskstats can export it to user
 * space.
B
Balbir Singh 已提交
3956 3957 3958
 */
int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
{
T
Tejun Heo 已提交
3959
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
3960
	struct cgroup *cgrp;
3961
	struct css_task_iter it;
B
Balbir Singh 已提交
3962
	struct task_struct *tsk;
3963

T
Tejun Heo 已提交
3964 3965 3966 3967 3968
	/* it should be kernfs_node belonging to cgroupfs and is a directory */
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
		return -EINVAL;

3969 3970
	mutex_lock(&cgroup_mutex);

B
Balbir Singh 已提交
3971
	/*
T
Tejun Heo 已提交
3972
	 * We aren't being called from kernfs and there's no guarantee on
3973
	 * @kn->priv's validity.  For this and css_tryget_online_from_dir(),
T
Tejun Heo 已提交
3974
	 * @kn->priv is RCU safe.  Let's do the RCU dancing.
B
Balbir Singh 已提交
3975
	 */
T
Tejun Heo 已提交
3976 3977
	rcu_read_lock();
	cgrp = rcu_dereference(kn->priv);
3978
	if (!cgrp || cgroup_is_dead(cgrp)) {
T
Tejun Heo 已提交
3979
		rcu_read_unlock();
3980
		mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
3981 3982
		return -ENOENT;
	}
3983
	rcu_read_unlock();
B
Balbir Singh 已提交
3984

3985
	css_task_iter_start(&cgrp->self, &it);
3986
	while ((tsk = css_task_iter_next(&it))) {
B
Balbir Singh 已提交
3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005
		switch (tsk->state) {
		case TASK_RUNNING:
			stats->nr_running++;
			break;
		case TASK_INTERRUPTIBLE:
			stats->nr_sleeping++;
			break;
		case TASK_UNINTERRUPTIBLE:
			stats->nr_uninterruptible++;
			break;
		case TASK_STOPPED:
			stats->nr_stopped++;
			break;
		default:
			if (delayacct_is_task_waiting_on_io(tsk))
				stats->nr_io_wait++;
			break;
		}
	}
4006
	css_task_iter_end(&it);
B
Balbir Singh 已提交
4007

4008
	mutex_unlock(&cgroup_mutex);
T
Tejun Heo 已提交
4009
	return 0;
B
Balbir Singh 已提交
4010 4011
}

4012

4013
/*
4014
 * seq_file methods for the tasks/procs files. The seq_file position is the
4015
 * next pid to display; the seq_file iterator is a pointer to the pid
4016
 * in the cgroup->l->list array.
4017
 */
4018

4019
static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
4020
{
4021 4022 4023 4024 4025 4026
	/*
	 * Initially we receive a position value that corresponds to
	 * one more than the last pid shown (or 0 on the first call or
	 * after a seek to the start). Use a binary-search to find the
	 * next pid to display, if any
	 */
T
Tejun Heo 已提交
4027
	struct kernfs_open_file *of = s->private;
4028
	struct cgroup *cgrp = seq_css(s)->cgroup;
4029
	struct cgroup_pidlist *l;
4030
	enum cgroup_filetype type = seq_cft(s)->private;
4031
	int index = 0, pid = *pos;
4032 4033 4034 4035 4036
	int *iter, ret;

	mutex_lock(&cgrp->pidlist_mutex);

	/*
4037
	 * !NULL @of->priv indicates that this isn't the first start()
4038
	 * after open.  If the matching pidlist is around, we can use that.
4039
	 * Look for it.  Note that @of->priv can't be used directly.  It
4040 4041
	 * could already have been destroyed.
	 */
4042 4043
	if (of->priv)
		of->priv = cgroup_pidlist_find(cgrp, type);
4044 4045 4046 4047 4048

	/*
	 * Either this is the first start() after open or the matching
	 * pidlist has been destroyed inbetween.  Create a new one.
	 */
4049 4050 4051
	if (!of->priv) {
		ret = pidlist_array_load(cgrp, type,
					 (struct cgroup_pidlist **)&of->priv);
4052 4053 4054
		if (ret)
			return ERR_PTR(ret);
	}
4055
	l = of->priv;
4056 4057

	if (pid) {
4058
		int end = l->length;
S
Stephen Rothwell 已提交
4059

4060 4061
		while (index < end) {
			int mid = (index + end) / 2;
4062
			if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
4063 4064
				index = mid;
				break;
4065
			} else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
4066 4067 4068 4069 4070 4071
				index = mid + 1;
			else
				end = mid;
		}
	}
	/* If we're off the end of the array, we're done */
4072
	if (index >= l->length)
4073 4074
		return NULL;
	/* Update the abstract position to be the actual pid that we found */
4075
	iter = l->list + index;
4076
	*pos = cgroup_pid_fry(cgrp, *iter);
4077 4078 4079
	return iter;
}

4080
static void cgroup_pidlist_stop(struct seq_file *s, void *v)
4081
{
T
Tejun Heo 已提交
4082
	struct kernfs_open_file *of = s->private;
4083
	struct cgroup_pidlist *l = of->priv;
4084

4085 4086
	if (l)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
4087
				 CGROUP_PIDLIST_DESTROY_DELAY);
4088
	mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
4089 4090
}

4091
static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
4092
{
T
Tejun Heo 已提交
4093
	struct kernfs_open_file *of = s->private;
4094
	struct cgroup_pidlist *l = of->priv;
4095 4096
	pid_t *p = v;
	pid_t *end = l->list + l->length;
4097 4098 4099 4100 4101 4102 4103 4104
	/*
	 * Advance to the next pid in the array. If this goes off the
	 * end, we're done
	 */
	p++;
	if (p >= end) {
		return NULL;
	} else {
4105
		*pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
4106 4107 4108 4109
		return p;
	}
}

4110
static int cgroup_pidlist_show(struct seq_file *s, void *v)
4111 4112 4113
{
	return seq_printf(s, "%d\n", *(int *)v);
}
4114

4115 4116
static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
					 struct cftype *cft)
4117
{
4118
	return notify_on_release(css->cgroup);
4119 4120
}

4121 4122
static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
					  struct cftype *cft, u64 val)
4123
{
4124
	clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
4125
	if (val)
4126
		set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4127
	else
4128
		clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
4129 4130 4131
	return 0;
}

4132 4133
static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
4134
{
4135
	return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4136 4137
}

4138 4139
static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
4140 4141
{
	if (val)
4142
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4143
	else
4144
		clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
4145 4146 4147
	return 0;
}

4148 4149
/* cgroup core interface files for the default hierarchy */
static struct cftype cgroup_dfl_base_files[] = {
4150
	{
4151
		.name = "cgroup.procs",
4152 4153 4154 4155
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
4156
		.private = CGROUP_FILE_PROCS,
4157
		.write = cgroup_procs_write,
B
Ben Blum 已提交
4158
		.mode = S_IRUGO | S_IWUSR,
4159
	},
4160 4161
	{
		.name = "cgroup.controllers",
4162
		.flags = CFTYPE_ONLY_ON_ROOT,
4163 4164 4165 4166
		.seq_show = cgroup_root_controllers_show,
	},
	{
		.name = "cgroup.controllers",
4167
		.flags = CFTYPE_NOT_ON_ROOT,
4168 4169 4170 4171 4172
		.seq_show = cgroup_controllers_show,
	},
	{
		.name = "cgroup.subtree_control",
		.seq_show = cgroup_subtree_control_show,
4173
		.write = cgroup_subtree_control_write,
4174
	},
4175 4176
	{
		.name = "cgroup.populated",
4177
		.flags = CFTYPE_NOT_ON_ROOT,
4178 4179
		.seq_show = cgroup_populated_show,
	},
4180 4181
	{ }	/* terminate */
};
4182

4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204
/* cgroup core interface files for the legacy hierarchies */
static struct cftype cgroup_legacy_base_files[] = {
	{
		.name = "cgroup.procs",
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
		.private = CGROUP_FILE_PROCS,
		.write = cgroup_procs_write,
		.mode = S_IRUGO | S_IWUSR,
	},
	{
		.name = "cgroup.clone_children",
		.read_u64 = cgroup_clone_children_read,
		.write_u64 = cgroup_clone_children_write,
	},
	{
		.name = "cgroup.sane_behavior",
		.flags = CFTYPE_ONLY_ON_ROOT,
		.seq_show = cgroup_sane_behavior_show,
	},
4205 4206
	{
		.name = "tasks",
4207 4208 4209 4210
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
4211
		.private = CGROUP_FILE_TASKS,
4212
		.write = cgroup_tasks_write,
4213 4214 4215 4216 4217 4218 4219
		.mode = S_IRUGO | S_IWUSR,
	},
	{
		.name = "notify_on_release",
		.read_u64 = cgroup_read_notify_on_release,
		.write_u64 = cgroup_write_notify_on_release,
	},
4220 4221
	{
		.name = "release_agent",
4222
		.flags = CFTYPE_ONLY_ON_ROOT,
4223
		.seq_show = cgroup_release_agent_show,
4224
		.write = cgroup_release_agent_write,
4225
		.max_write_len = PATH_MAX - 1,
4226
	},
T
Tejun Heo 已提交
4227
	{ }	/* terminate */
4228 4229
};

4230
/**
4231
 * cgroup_populate_dir - create subsys files in a cgroup directory
4232 4233
 * @cgrp: target cgroup
 * @subsys_mask: mask of the subsystem ids whose files should be added
4234 4235
 *
 * On failure, no file is added.
4236
 */
4237
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask)
4238 4239
{
	struct cgroup_subsys *ss;
4240
	int i, ret = 0;
4241

4242
	/* process cftsets of each subsystem */
4243
	for_each_subsys(ss, i) {
T
Tejun Heo 已提交
4244
		struct cftype *cfts;
4245

4246
		if (!(subsys_mask & (1 << i)))
4247
			continue;
4248

T
Tejun Heo 已提交
4249 4250
		list_for_each_entry(cfts, &ss->cfts, node) {
			ret = cgroup_addrm_files(cgrp, cfts, true);
4251 4252 4253
			if (ret < 0)
				goto err;
		}
4254 4255
	}
	return 0;
4256 4257 4258
err:
	cgroup_clear_dir(cgrp, subsys_mask);
	return ret;
4259 4260
}

4261 4262 4263 4264 4265 4266 4267
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4268 4269 4270
 *    and thus css_tryget_online() is guaranteed to fail, the css can be
 *    offlined by invoking offline_css().  After offlining, the base ref is
 *    put.  Implemented in css_killed_work_fn().
4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
4283
static void css_free_work_fn(struct work_struct *work)
4284 4285
{
	struct cgroup_subsys_state *css =
4286
		container_of(work, struct cgroup_subsys_state, destroy_work);
4287
	struct cgroup *cgrp = css->cgroup;
4288

4289 4290 4291 4292
	if (css->ss) {
		/* css free path */
		if (css->parent)
			css_put(css->parent);
4293

4294 4295 4296 4297 4298 4299 4300
		css->ss->css_free(css);
		cgroup_put(cgrp);
	} else {
		/* cgroup free path */
		atomic_dec(&cgrp->root->nr_cgrps);
		cgroup_pidlist_destroy_all(cgrp);

T
Tejun Heo 已提交
4301
		if (cgroup_parent(cgrp)) {
4302 4303 4304 4305 4306 4307
			/*
			 * We get a ref to the parent, and put the ref when
			 * this cgroup is being freed, so it's guaranteed
			 * that the parent won't be destroyed before its
			 * children.
			 */
T
Tejun Heo 已提交
4308
			cgroup_put(cgroup_parent(cgrp));
4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319
			kernfs_put(cgrp->kn);
			kfree(cgrp);
		} else {
			/*
			 * This is root cgroup's refcnt reaching zero,
			 * which indicates that the root should be
			 * released.
			 */
			cgroup_destroy_root(cgrp->root);
		}
	}
4320 4321
}

4322
static void css_free_rcu_fn(struct rcu_head *rcu_head)
4323 4324
{
	struct cgroup_subsys_state *css =
4325
		container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4326

4327
	INIT_WORK(&css->destroy_work, css_free_work_fn);
4328
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4329 4330
}

4331
static void css_release_work_fn(struct work_struct *work)
4332 4333
{
	struct cgroup_subsys_state *css =
4334
		container_of(work, struct cgroup_subsys_state, destroy_work);
4335
	struct cgroup_subsys *ss = css->ss;
4336
	struct cgroup *cgrp = css->cgroup;
4337

4338 4339
	mutex_lock(&cgroup_mutex);

4340
	css->flags |= CSS_RELEASED;
4341 4342
	list_del_rcu(&css->sibling);

4343 4344 4345 4346 4347 4348 4349 4350
	if (ss) {
		/* css release path */
		cgroup_idr_remove(&ss->css_idr, css->id);
	} else {
		/* cgroup release path */
		cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
		cgrp->id = -1;
	}
4351

4352 4353
	mutex_unlock(&cgroup_mutex);

4354
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4355 4356 4357 4358 4359 4360 4361
}

static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4362 4363
	INIT_WORK(&css->destroy_work, css_release_work_fn);
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4364 4365
}

4366 4367
static void init_and_link_css(struct cgroup_subsys_state *css,
			      struct cgroup_subsys *ss, struct cgroup *cgrp)
4368
{
4369 4370
	lockdep_assert_held(&cgroup_mutex);

4371 4372
	cgroup_get(cgrp);

4373
	memset(css, 0, sizeof(*css));
4374
	css->cgroup = cgrp;
4375
	css->ss = ss;
4376 4377
	INIT_LIST_HEAD(&css->sibling);
	INIT_LIST_HEAD(&css->children);
4378
	css->serial_nr = css_serial_nr_next++;
4379

T
Tejun Heo 已提交
4380 4381
	if (cgroup_parent(cgrp)) {
		css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4382 4383
		css_get(css->parent);
	}
4384

4385
	BUG_ON(cgroup_css(cgrp, ss));
4386 4387
}

4388
/* invoke ->css_online() on a new CSS and mark it online if successful */
4389
static int online_css(struct cgroup_subsys_state *css)
4390
{
4391
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
4392 4393
	int ret = 0;

4394 4395
	lockdep_assert_held(&cgroup_mutex);

4396
	if (ss->css_online)
4397
		ret = ss->css_online(css);
4398
	if (!ret) {
4399
		css->flags |= CSS_ONLINE;
4400
		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4401
	}
T
Tejun Heo 已提交
4402
	return ret;
4403 4404
}

4405
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4406
static void offline_css(struct cgroup_subsys_state *css)
4407
{
4408
	struct cgroup_subsys *ss = css->ss;
4409 4410 4411 4412 4413 4414

	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

4415
	if (ss->css_offline)
4416
		ss->css_offline(css);
4417

4418
	css->flags &= ~CSS_ONLINE;
4419
	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4420 4421

	wake_up_all(&css->cgroup->offline_waitq);
4422 4423
}

4424 4425 4426 4427
/**
 * create_css - create a cgroup_subsys_state
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
4428
 * @visible: whether to create control knobs for the new css or not
4429 4430
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
4431 4432
 * css is online and installed in @cgrp with all interface files created if
 * @visible.  Returns 0 on success, -errno on failure.
4433
 */
4434 4435
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss,
		      bool visible)
4436
{
T
Tejun Heo 已提交
4437
	struct cgroup *parent = cgroup_parent(cgrp);
4438
	struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4439 4440 4441 4442 4443
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgroup_mutex);

4444
	css = ss->css_alloc(parent_css);
4445 4446 4447
	if (IS_ERR(css))
		return PTR_ERR(css);

4448
	init_and_link_css(css, ss, cgrp);
4449

4450 4451
	err = percpu_ref_init(&css->refcnt, css_release);
	if (err)
4452
		goto err_free_css;
4453

4454 4455 4456 4457
	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT);
	if (err < 0)
		goto err_free_percpu_ref;
	css->id = err;
4458

4459 4460 4461 4462 4463
	if (visible) {
		err = cgroup_populate_dir(cgrp, 1 << ss->id);
		if (err)
			goto err_free_id;
	}
4464 4465

	/* @css is ready to be brought online now, make it visible */
4466
	list_add_tail_rcu(&css->sibling, &parent_css->children);
4467
	cgroup_idr_replace(&ss->css_idr, css, css->id);
4468 4469 4470

	err = online_css(css);
	if (err)
4471
		goto err_list_del;
4472

4473
	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
T
Tejun Heo 已提交
4474
	    cgroup_parent(parent)) {
4475
		pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4476
			current->comm, current->pid, ss->name);
4477
		if (!strcmp(ss->name, "memory"))
4478
			pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4479 4480 4481 4482 4483
		ss->warned_broken_hierarchy = true;
	}

	return 0;

4484 4485
err_list_del:
	list_del_rcu(&css->sibling);
4486
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4487 4488
err_free_id:
	cgroup_idr_remove(&ss->css_idr, css->id);
4489
err_free_percpu_ref:
4490
	percpu_ref_cancel_init(&css->refcnt);
4491
err_free_css:
4492
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4493 4494 4495
	return err;
}

4496 4497
static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
			umode_t mode)
4498
{
4499 4500
	struct cgroup *parent, *cgrp;
	struct cgroup_root *root;
4501
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
4502
	struct kernfs_node *kn;
4503
	struct cftype *base_files;
4504
	int ssid, ret;
4505

4506 4507 4508 4509
	parent = cgroup_kn_lock_live(parent_kn);
	if (!parent)
		return -ENODEV;
	root = parent->root;
4510

T
Tejun Heo 已提交
4511
	/* allocate the cgroup and its ID, 0 is reserved for the root */
4512
	cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
T
Tejun Heo 已提交
4513 4514 4515
	if (!cgrp) {
		ret = -ENOMEM;
		goto out_unlock;
4516 4517
	}

4518 4519 4520 4521
	ret = percpu_ref_init(&cgrp->self.refcnt, css_release);
	if (ret)
		goto out_free_cgrp;

4522 4523 4524 4525
	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
4526
	cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT);
4527
	if (cgrp->id < 0) {
T
Tejun Heo 已提交
4528
		ret = -ENOMEM;
4529
		goto out_cancel_ref;
4530 4531
	}

4532
	init_cgroup_housekeeping(cgrp);
4533

4534
	cgrp->self.parent = &parent->self;
T
Tejun Heo 已提交
4535
	cgrp->root = root;
4536

4537 4538 4539
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

4540 4541
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4542

T
Tejun Heo 已提交
4543
	/* create the directory */
T
Tejun Heo 已提交
4544
	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
T
Tejun Heo 已提交
4545
	if (IS_ERR(kn)) {
T
Tejun Heo 已提交
4546 4547
		ret = PTR_ERR(kn);
		goto out_free_id;
T
Tejun Heo 已提交
4548 4549
	}
	cgrp->kn = kn;
4550

4551
	/*
4552 4553
	 * This extra ref will be put in cgroup_free_fn() and guarantees
	 * that @cgrp->kn is always accessible.
4554
	 */
4555
	kernfs_get(kn);
4556

4557
	cgrp->self.serial_nr = css_serial_nr_next++;
4558

4559
	/* allocation complete, commit to creation */
4560
	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4561
	atomic_inc(&root->nr_cgrps);
4562
	cgroup_get(parent);
4563

4564 4565 4566 4567
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
4568
	cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4569

T
Tejun Heo 已提交
4570 4571 4572
	ret = cgroup_kn_set_ugid(kn);
	if (ret)
		goto out_destroy;
4573

4574 4575 4576 4577 4578 4579
	if (cgroup_on_dfl(cgrp))
		base_files = cgroup_dfl_base_files;
	else
		base_files = cgroup_legacy_base_files;

	ret = cgroup_addrm_files(cgrp, base_files, true);
T
Tejun Heo 已提交
4580 4581
	if (ret)
		goto out_destroy;
4582

4583
	/* let's create and online css's */
T
Tejun Heo 已提交
4584
	for_each_subsys(ss, ssid) {
4585
		if (parent->child_subsys_mask & (1 << ssid)) {
4586 4587
			ret = create_css(cgrp, ss,
					 parent->subtree_control & (1 << ssid));
T
Tejun Heo 已提交
4588 4589
			if (ret)
				goto out_destroy;
T
Tejun Heo 已提交
4590
		}
4591
	}
4592

4593 4594
	/*
	 * On the default hierarchy, a child doesn't automatically inherit
4595
	 * subtree_control from the parent.  Each is configured manually.
4596
	 */
4597 4598 4599 4600
	if (!cgroup_on_dfl(cgrp)) {
		cgrp->subtree_control = parent->subtree_control;
		cgroup_refresh_child_subsys_mask(cgrp);
	}
T
Tejun Heo 已提交
4601 4602

	kernfs_activate(kn);
4603

T
Tejun Heo 已提交
4604 4605
	ret = 0;
	goto out_unlock;
4606

T
Tejun Heo 已提交
4607
out_free_id:
4608
	cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
4609 4610
out_cancel_ref:
	percpu_ref_cancel_init(&cgrp->self.refcnt);
T
Tejun Heo 已提交
4611
out_free_cgrp:
4612
	kfree(cgrp);
T
Tejun Heo 已提交
4613
out_unlock:
4614
	cgroup_kn_unlock(parent_kn);
T
Tejun Heo 已提交
4615
	return ret;
4616

T
Tejun Heo 已提交
4617
out_destroy:
4618
	cgroup_destroy_locked(cgrp);
T
Tejun Heo 已提交
4619
	goto out_unlock;
4620 4621
}

4622 4623
/*
 * This is called when the refcnt of a css is confirmed to be killed.
4624 4625
 * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
 * initate destruction and put the css ref from kill_css().
4626 4627
 */
static void css_killed_work_fn(struct work_struct *work)
4628
{
4629 4630
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
4631

4632
	mutex_lock(&cgroup_mutex);
4633
	offline_css(css);
4634
	mutex_unlock(&cgroup_mutex);
4635 4636

	css_put(css);
4637 4638
}

4639 4640
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
4641 4642 4643 4644
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4645
	INIT_WORK(&css->destroy_work, css_killed_work_fn);
4646
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4647 4648
}

4649 4650 4651 4652 4653 4654
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
4655 4656
 * asynchronously once css_tryget_online() is guaranteed to fail and when
 * the reference count reaches zero, @css will be released.
4657 4658
 */
static void kill_css(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
4659
{
4660
	lockdep_assert_held(&cgroup_mutex);
4661

T
Tejun Heo 已提交
4662 4663 4664 4665
	/*
	 * This must happen before css is disassociated with its cgroup.
	 * See seq_css() for details.
	 */
4666
	cgroup_clear_dir(css->cgroup, 1 << css->ss->id);
4667

T
Tejun Heo 已提交
4668 4669 4670 4671 4672 4673 4674 4675 4676
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
4677
	 * css_tryget_online().  We can't simply call percpu_ref_kill() and
T
Tejun Heo 已提交
4678 4679 4680 4681 4682 4683 4684
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4685 4686 4687 4688 4689 4690 4691 4692
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
4693 4694 4695
 * guarantee that css_tryget_online() won't succeed by the time
 * ->css_offline() is invoked.  To satisfy all the requirements,
 * destruction is implemented in the following two steps.
4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
4711 4712
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4713
{
T
Tejun Heo 已提交
4714
	struct cgroup_subsys_state *css;
4715
	bool empty;
T
Tejun Heo 已提交
4716
	int ssid;
4717

4718 4719
	lockdep_assert_held(&cgroup_mutex);

4720
	/*
4721
	 * css_set_rwsem synchronizes access to ->cset_links and prevents
4722
	 * @cgrp from being removed while put_css_set() is in progress.
4723
	 */
4724
	down_read(&css_set_rwsem);
4725
	empty = list_empty(&cgrp->cset_links);
4726
	up_read(&css_set_rwsem);
4727
	if (!empty)
4728
		return -EBUSY;
L
Li Zefan 已提交
4729

4730
	/*
4731 4732 4733
	 * Make sure there's no live children.  We can't test emptiness of
	 * ->self.children as dead children linger on it while being
	 * drained; otherwise, "rmdir parent/child parent" may fail.
4734
	 */
4735
	if (css_has_online_children(&cgrp->self))
4736 4737
		return -EBUSY;

4738 4739
	/*
	 * Mark @cgrp dead.  This prevents further task migration and child
4740
	 * creation by disabling cgroup_lock_live_group().
4741
	 */
4742
	cgrp->self.flags &= ~CSS_ONLINE;
4743

4744
	/* initiate massacre of all css's */
T
Tejun Heo 已提交
4745 4746
	for_each_css(css, ssid, cgrp)
		kill_css(css);
4747

4748
	/* CSS_ONLINE is clear, remove from ->release_list for the last time */
4749 4750 4751 4752 4753 4754
	raw_spin_lock(&release_list_lock);
	if (!list_empty(&cgrp->release_list))
		list_del_init(&cgrp->release_list);
	raw_spin_unlock(&release_list_lock);

	/*
4755 4756
	 * Remove @cgrp directory along with the base files.  @cgrp has an
	 * extra ref on its kn.
4757
	 */
4758
	kernfs_remove(cgrp->kn);
4759

T
Tejun Heo 已提交
4760 4761
	set_bit(CGRP_RELEASABLE, &cgroup_parent(cgrp)->flags);
	check_for_release(cgroup_parent(cgrp));
T
Tejun Heo 已提交
4762

4763
	/* put the base reference */
4764
	percpu_ref_kill(&cgrp->self.refcnt);
4765

4766 4767 4768
	return 0;
};

T
Tejun Heo 已提交
4769
static int cgroup_rmdir(struct kernfs_node *kn)
4770
{
4771
	struct cgroup *cgrp;
T
Tejun Heo 已提交
4772
	int ret = 0;
4773

4774 4775 4776 4777
	cgrp = cgroup_kn_lock_live(kn);
	if (!cgrp)
		return 0;
	cgroup_get(cgrp);	/* for @kn->priv clearing */
4778

4779
	ret = cgroup_destroy_locked(cgrp);
4780

4781
	cgroup_kn_unlock(kn);
4782 4783

	/*
4784 4785 4786 4787 4788
	 * There are two control paths which try to determine cgroup from
	 * dentry without going through kernfs - cgroupstats_build() and
	 * css_tryget_online_from_dir().  Those are supported by RCU
	 * protecting clearing of cgrp->kn->priv backpointer, which should
	 * happen after all files under it have been removed.
4789
	 */
4790 4791
	if (!ret)
		RCU_INIT_POINTER(*(void __rcu __force **)&kn->priv, NULL);
4792

T
Tejun Heo 已提交
4793
	cgroup_put(cgrp);
4794
	return ret;
4795 4796
}

T
Tejun Heo 已提交
4797 4798 4799 4800 4801 4802 4803 4804
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
	.remount_fs		= cgroup_remount,
	.show_options		= cgroup_show_options,
	.mkdir			= cgroup_mkdir,
	.rmdir			= cgroup_rmdir,
	.rename			= cgroup_rename,
};

4805
static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
4806 4807
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
4808 4809

	printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4810

4811 4812
	mutex_lock(&cgroup_mutex);

4813
	idr_init(&ss->css_idr);
T
Tejun Heo 已提交
4814
	INIT_LIST_HEAD(&ss->cfts);
4815

4816 4817 4818
	/* Create the root cgroup state for this subsystem */
	ss->root = &cgrp_dfl_root;
	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
4819 4820
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
4821
	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
4822 4823 4824 4825 4826 4827 4828

	/*
	 * Root csses are never destroyed and we can't initialize
	 * percpu_ref during early init.  Disable refcnting.
	 */
	css->flags |= CSS_NO_REF;

4829
	if (early) {
4830
		/* allocation can't be done safely during early init */
4831 4832 4833 4834 4835
		css->id = 1;
	} else {
		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
		BUG_ON(css->id < 0);
	}
4836

L
Li Zefan 已提交
4837
	/* Update the init_css_set to contain a subsys
4838
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
4839
	 * newly registered, all tasks and hence the
4840
	 * init_css_set is in the subsystem's root cgroup. */
4841
	init_css_set.subsys[ss->id] = css;
4842 4843 4844

	need_forkexit_callback |= ss->fork || ss->exit;

L
Li Zefan 已提交
4845 4846 4847 4848 4849
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

4850
	BUG_ON(online_css(css));
4851

B
Ben Blum 已提交
4852 4853 4854
	mutex_unlock(&cgroup_mutex);
}

4855
/**
L
Li Zefan 已提交
4856 4857 4858 4859
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
4860 4861 4862
 */
int __init cgroup_init_early(void)
{
4863
	static struct cgroup_sb_opts __initdata opts;
4864
	struct cgroup_subsys *ss;
4865
	int i;
4866

4867
	init_cgroup_root(&cgrp_dfl_root, &opts);
4868 4869
	cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;

4870
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4871

T
Tejun Heo 已提交
4872
	for_each_subsys(ss, i) {
4873
		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
4874 4875
		     "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n",
		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
4876
		     ss->id, ss->name);
4877 4878 4879
		WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
		     "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);

4880
		ss->id = i;
4881
		ss->name = cgroup_subsys_name[i];
4882 4883

		if (ss->early_init)
4884
			cgroup_init_subsys(ss, true);
4885 4886 4887 4888 4889
	}
	return 0;
}

/**
L
Li Zefan 已提交
4890 4891 4892 4893
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
4894 4895 4896
 */
int __init cgroup_init(void)
{
4897
	struct cgroup_subsys *ss;
4898
	unsigned long key;
4899
	int ssid, err;
4900

4901 4902
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
4903

T
Tejun Heo 已提交
4904 4905
	mutex_lock(&cgroup_mutex);

4906 4907 4908 4909
	/* Add init_css_set to the hash table */
	key = css_set_hash(init_css_set.subsys);
	hash_add(css_set_table, &init_css_set.hlist, key);

4910
	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4911

T
Tejun Heo 已提交
4912 4913
	mutex_unlock(&cgroup_mutex);

4914
	for_each_subsys(ss, ssid) {
4915 4916 4917 4918 4919 4920 4921 4922 4923 4924
		if (ss->early_init) {
			struct cgroup_subsys_state *css =
				init_css_set.subsys[ss->id];

			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
						   GFP_KERNEL);
			BUG_ON(css->id < 0);
		} else {
			cgroup_init_subsys(ss, false);
		}
4925

T
Tejun Heo 已提交
4926 4927
		list_add_tail(&init_css_set.e_cset_node[ssid],
			      &cgrp_dfl_root.cgrp.e_csets[ssid]);
4928 4929

		/*
4930 4931 4932
		 * Setting dfl_root subsys_mask needs to consider the
		 * disabled flag and cftype registration needs kmalloc,
		 * both of which aren't available during early_init.
4933
		 */
4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946
		if (ss->disabled)
			continue;

		cgrp_dfl_root.subsys_mask |= 1 << ss->id;

		if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes)
			ss->dfl_cftypes = ss->legacy_cftypes;

		if (ss->dfl_cftypes == ss->legacy_cftypes) {
			WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
		} else {
			WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
			WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
4947
		}
4948 4949 4950
	}

	cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
T
Tejun Heo 已提交
4951 4952
	if (!cgroup_kobj)
		return -ENOMEM;
4953

4954
	err = register_filesystem(&cgroup_fs_type);
4955 4956
	if (err < 0) {
		kobject_put(cgroup_kobj);
T
Tejun Heo 已提交
4957
		return err;
4958
	}
4959

L
Li Zefan 已提交
4960
	proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
T
Tejun Heo 已提交
4961
	return 0;
4962
}
4963

4964 4965 4966 4967 4968
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
4969
	 * Use 1 for @max_active.
4970 4971 4972 4973
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
4974
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
4975
	BUG_ON(!cgroup_destroy_wq);
4976 4977 4978 4979 4980 4981 4982 4983 4984

	/*
	 * Used to destroy pidlists and separate to serve as flush domain.
	 * Cap @max_active to 1 too.
	 */
	cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
						    0, 1);
	BUG_ON(!cgroup_pidlist_destroy_wq);

4985 4986 4987 4988
	return 0;
}
core_initcall(cgroup_wq_init);

4989 4990 4991 4992 4993 4994 4995
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 */

/* TODO: Use a proper seq_file iterator */
4996
int proc_cgroup_show(struct seq_file *m, void *v)
4997 4998 4999
{
	struct pid *pid;
	struct task_struct *tsk;
T
Tejun Heo 已提交
5000
	char *buf, *path;
5001
	int retval;
5002
	struct cgroup_root *root;
5003 5004

	retval = -ENOMEM;
T
Tejun Heo 已提交
5005
	buf = kmalloc(PATH_MAX, GFP_KERNEL);
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017
	if (!buf)
		goto out;

	retval = -ESRCH;
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
	if (!tsk)
		goto out_free;

	retval = 0;

	mutex_lock(&cgroup_mutex);
5018
	down_read(&css_set_rwsem);
5019

5020
	for_each_root(root) {
5021
		struct cgroup_subsys *ss;
5022
		struct cgroup *cgrp;
T
Tejun Heo 已提交
5023
		int ssid, count = 0;
5024

T
Tejun Heo 已提交
5025
		if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible)
5026 5027
			continue;

5028
		seq_printf(m, "%d:", root->hierarchy_id);
T
Tejun Heo 已提交
5029
		for_each_subsys(ss, ssid)
5030
			if (root->subsys_mask & (1 << ssid))
T
Tejun Heo 已提交
5031
				seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
5032 5033 5034
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
5035
		seq_putc(m, ':');
5036
		cgrp = task_cgroup_from_root(tsk, root);
T
Tejun Heo 已提交
5037 5038 5039
		path = cgroup_path(cgrp, buf, PATH_MAX);
		if (!path) {
			retval = -ENAMETOOLONG;
5040
			goto out_unlock;
T
Tejun Heo 已提交
5041 5042
		}
		seq_puts(m, path);
5043 5044 5045 5046
		seq_putc(m, '\n');
	}

out_unlock:
5047
	up_read(&css_set_rwsem);
5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058
	mutex_unlock(&cgroup_mutex);
	put_task_struct(tsk);
out_free:
	kfree(buf);
out:
	return retval;
}

/* Display information about each subsystem and each hierarchy */
static int proc_cgroupstats_show(struct seq_file *m, void *v)
{
5059
	struct cgroup_subsys *ss;
5060 5061
	int i;

5062
	seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
B
Ben Blum 已提交
5063 5064 5065 5066 5067
	/*
	 * ideally we don't want subsystems moving around while we do this.
	 * cgroup_mutex is also necessary to guarantee an atomic snapshot of
	 * subsys/hierarchy state.
	 */
5068
	mutex_lock(&cgroup_mutex);
5069 5070

	for_each_subsys(ss, i)
5071 5072
		seq_printf(m, "%s\t%d\t%d\t%d\n",
			   ss->name, ss->root->hierarchy_id,
5073
			   atomic_read(&ss->root->nr_cgrps), !ss->disabled);
5074

5075 5076 5077 5078 5079 5080
	mutex_unlock(&cgroup_mutex);
	return 0;
}

static int cgroupstats_open(struct inode *inode, struct file *file)
{
A
Al Viro 已提交
5081
	return single_open(file, proc_cgroupstats_show, NULL);
5082 5083
}

5084
static const struct file_operations proc_cgroupstats_operations = {
5085 5086 5087 5088 5089 5090
	.open = cgroupstats_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

5091
/**
5092
 * cgroup_fork - initialize cgroup related fields during copy_process()
L
Li Zefan 已提交
5093
 * @child: pointer to task_struct of forking parent process.
5094
 *
5095 5096 5097
 * A task is associated with the init_css_set until cgroup_post_fork()
 * attaches it to the parent's css_set.  Empty cg_list indicates that
 * @child isn't holding reference to its css_set.
5098 5099 5100
 */
void cgroup_fork(struct task_struct *child)
{
5101
	RCU_INIT_POINTER(child->cgroups, &init_css_set);
5102
	INIT_LIST_HEAD(&child->cg_list);
5103 5104
}

5105
/**
L
Li Zefan 已提交
5106 5107 5108
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
5109 5110 5111
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
5112
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5113
 * list.
L
Li Zefan 已提交
5114
 */
5115 5116
void cgroup_post_fork(struct task_struct *child)
{
5117
	struct cgroup_subsys *ss;
5118 5119
	int i;

5120
	/*
5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139
	 * This may race against cgroup_enable_task_cg_links().  As that
	 * function sets use_task_css_set_links before grabbing
	 * tasklist_lock and we just went through tasklist_lock to add
	 * @child, it's guaranteed that either we see the set
	 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
	 * @child during its iteration.
	 *
	 * If we won the race, @child is associated with %current's
	 * css_set.  Grabbing css_set_rwsem guarantees both that the
	 * association is stable, and, on completion of the parent's
	 * migration, @child is visible in the source of migration or
	 * already in the destination cgroup.  This guarantee is necessary
	 * when implementing operations which need to migrate all tasks of
	 * a cgroup to another.
	 *
	 * Note that if we lose to cgroup_enable_task_cg_links(), @child
	 * will remain in init_css_set.  This is safe because all tasks are
	 * in the init_css_set before cg_links is enabled and there's no
	 * operation which transfers all tasks out of init_css_set.
5140
	 */
5141
	if (use_task_css_set_links) {
5142 5143
		struct css_set *cset;

5144
		down_write(&css_set_rwsem);
5145
		cset = task_css_set(current);
5146 5147 5148 5149 5150
		if (list_empty(&child->cg_list)) {
			rcu_assign_pointer(child->cgroups, cset);
			list_add(&child->cg_list, &cset->tasks);
			get_css_set(cset);
		}
5151
		up_write(&css_set_rwsem);
5152
	}
5153 5154 5155 5156 5157 5158 5159

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
5160
		for_each_subsys(ss, i)
5161 5162 5163
			if (ss->fork)
				ss->fork(child);
	}
5164
}
5165

5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
5178 5179 5180 5181 5182
 * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
 * call cgroup_exit() while the task is still competent to handle
 * notify_on_release(), then leave the task attached to the root cgroup in
 * each hierarchy for the remainder of its exit.  No need to bother with
 * init_css_set refcnting.  init_css_set never goes away and we can't race
5183
 * with migration path - PF_EXITING is visible to migration path.
5184
 */
5185
void cgroup_exit(struct task_struct *tsk)
5186
{
5187
	struct cgroup_subsys *ss;
5188
	struct css_set *cset;
5189
	bool put_cset = false;
5190
	int i;
5191 5192

	/*
5193 5194
	 * Unlink from @tsk from its css_set.  As migration path can't race
	 * with us, we can check cg_list without grabbing css_set_rwsem.
5195 5196
	 */
	if (!list_empty(&tsk->cg_list)) {
5197
		down_write(&css_set_rwsem);
5198
		list_del_init(&tsk->cg_list);
5199
		up_write(&css_set_rwsem);
5200
		put_cset = true;
5201 5202
	}

5203
	/* Reassign the task to the init_css_set. */
5204 5205
	cset = task_css_set(tsk);
	RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5206

5207
	if (need_forkexit_callback) {
T
Tejun Heo 已提交
5208 5209
		/* see cgroup_post_fork() for details */
		for_each_subsys(ss, i) {
5210
			if (ss->exit) {
5211 5212
				struct cgroup_subsys_state *old_css = cset->subsys[i];
				struct cgroup_subsys_state *css = task_css(tsk, i);
5213

5214
				ss->exit(css, old_css, tsk);
5215 5216 5217 5218
			}
		}
	}

5219 5220
	if (put_cset)
		put_css_set(cset, true);
5221
}
5222

5223
static void check_for_release(struct cgroup *cgrp)
5224
{
5225 5226
	if (cgroup_is_releasable(cgrp) && list_empty(&cgrp->cset_links) &&
	    !css_has_online_children(&cgrp->self)) {
5227 5228
		/*
		 * Control Group is currently removeable. If it's not
5229
		 * already queued for a userspace notification, queue
5230 5231
		 * it now
		 */
5232
		int need_schedule_work = 0;
5233

5234
		raw_spin_lock(&release_list_lock);
5235
		if (!cgroup_is_dead(cgrp) &&
5236 5237
		    list_empty(&cgrp->release_list)) {
			list_add(&cgrp->release_list, &release_list);
5238 5239
			need_schedule_work = 1;
		}
5240
		raw_spin_unlock(&release_list_lock);
5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272
		if (need_schedule_work)
			schedule_work(&release_agent_work);
	}
}

/*
 * Notify userspace when a cgroup is released, by running the
 * configured release agent with the name of the cgroup (path
 * relative to the root of cgroup file system) as the argument.
 *
 * Most likely, this user command will try to rmdir this cgroup.
 *
 * This races with the possibility that some other task will be
 * attached to this cgroup before it is removed, or that some other
 * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
 * unused, and this cgroup will be reprieved from its death sentence,
 * to continue to serve a useful existence.  Next time it's released,
 * we will get notified again, if it still has 'notify_on_release' set.
 *
 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
 * means only wait until the task is successfully execve()'d.  The
 * separate release agent task is forked by call_usermodehelper(),
 * then control in this thread returns here, without waiting for the
 * release agent task.  We don't bother to wait because the caller of
 * this routine has no use for the exit status of the release agent
 * task, so no sense holding our caller up for that.
 */
static void cgroup_release_agent(struct work_struct *work)
{
	BUG_ON(work != &release_agent_work);
	mutex_lock(&cgroup_mutex);
5273
	raw_spin_lock(&release_list_lock);
5274 5275 5276
	while (!list_empty(&release_list)) {
		char *argv[3], *envp[3];
		int i;
T
Tejun Heo 已提交
5277
		char *pathbuf = NULL, *agentbuf = NULL, *path;
5278
		struct cgroup *cgrp = list_entry(release_list.next,
5279 5280
						    struct cgroup,
						    release_list);
5281
		list_del_init(&cgrp->release_list);
5282
		raw_spin_unlock(&release_list_lock);
T
Tejun Heo 已提交
5283
		pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5284 5285
		if (!pathbuf)
			goto continue_free;
T
Tejun Heo 已提交
5286 5287
		path = cgroup_path(cgrp, pathbuf, PATH_MAX);
		if (!path)
5288 5289 5290 5291
			goto continue_free;
		agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
		if (!agentbuf)
			goto continue_free;
5292 5293

		i = 0;
5294
		argv[i++] = agentbuf;
T
Tejun Heo 已提交
5295
		argv[i++] = path;
5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309
		argv[i] = NULL;

		i = 0;
		/* minimal command environment */
		envp[i++] = "HOME=/";
		envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
		envp[i] = NULL;

		/* Drop the lock while we invoke the usermode helper,
		 * since the exec could involve hitting disk and hence
		 * be a slow process */
		mutex_unlock(&cgroup_mutex);
		call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
		mutex_lock(&cgroup_mutex);
5310 5311 5312
 continue_free:
		kfree(pathbuf);
		kfree(agentbuf);
5313
		raw_spin_lock(&release_list_lock);
5314
	}
5315
	raw_spin_unlock(&release_list_lock);
5316 5317
	mutex_unlock(&cgroup_mutex);
}
5318 5319 5320

static int __init cgroup_disable(char *str)
{
5321
	struct cgroup_subsys *ss;
5322
	char *token;
5323
	int i;
5324 5325 5326 5327

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
5328

T
Tejun Heo 已提交
5329
		for_each_subsys(ss, i) {
5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340
			if (!strcmp(token, ss->name)) {
				ss->disabled = 1;
				printk(KERN_INFO "Disabling %s control group"
					" subsystem\n", ss->name);
				break;
			}
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
5341

5342 5343 5344 5345 5346 5347 5348 5349
static int __init cgroup_set_legacy_files_on_dfl(char *str)
{
	printk("cgroup: using legacy files on the default hierarchy\n");
	cgroup_legacy_files_on_dfl = true;
	return 0;
}
__setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl);

5350
/**
5351
 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5352 5353
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
5354
 *
5355 5356 5357
 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
 * to get the corresponding css and return it.  If such css doesn't exist
 * or can't be pinned, an ERR_PTR value is returned.
S
Stephane Eranian 已提交
5358
 */
5359 5360
struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
						       struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
5361
{
T
Tejun Heo 已提交
5362 5363
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
	struct cgroup_subsys_state *css = NULL;
S
Stephane Eranian 已提交
5364 5365
	struct cgroup *cgrp;

5366
	/* is @dentry a cgroup dir? */
T
Tejun Heo 已提交
5367 5368
	if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
	    kernfs_type(kn) != KERNFS_DIR)
S
Stephane Eranian 已提交
5369 5370
		return ERR_PTR(-EBADF);

5371 5372
	rcu_read_lock();

T
Tejun Heo 已提交
5373 5374 5375
	/*
	 * This path doesn't originate from kernfs and @kn could already
	 * have been or be removed at any point.  @kn->priv is RCU
5376
	 * protected for this access.  See cgroup_rmdir() for details.
T
Tejun Heo 已提交
5377 5378 5379 5380
	 */
	cgrp = rcu_dereference(kn->priv);
	if (cgrp)
		css = cgroup_css(cgrp, ss);
5381

5382
	if (!css || !css_tryget_online(css))
5383 5384 5385 5386
		css = ERR_PTR(-ENOENT);

	rcu_read_unlock();
	return css;
S
Stephane Eranian 已提交
5387 5388
}

5389 5390 5391 5392 5393 5394 5395 5396 5397 5398
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
5399
	WARN_ON_ONCE(!rcu_read_lock_held());
5400
	return idr_find(&ss->css_idr, id);
S
Stephane Eranian 已提交
5401 5402
}

5403
#ifdef CONFIG_CGROUP_DEBUG
5404 5405
static struct cgroup_subsys_state *
debug_css_alloc(struct cgroup_subsys_state *parent_css)
5406 5407 5408 5409 5410 5411 5412 5413 5414
{
	struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);

	if (!css)
		return ERR_PTR(-ENOMEM);

	return css;
}

5415
static void debug_css_free(struct cgroup_subsys_state *css)
5416
{
5417
	kfree(css);
5418 5419
}

5420 5421
static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5422
{
5423
	return cgroup_task_count(css->cgroup);
5424 5425
}

5426 5427
static u64 current_css_set_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5428 5429 5430 5431
{
	return (u64)(unsigned long)current->cgroups;
}

5432
static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
L
Li Zefan 已提交
5433
					 struct cftype *cft)
5434 5435 5436 5437
{
	u64 count;

	rcu_read_lock();
5438
	count = atomic_read(&task_css_set(current)->refcount);
5439 5440 5441 5442
	rcu_read_unlock();
	return count;
}

5443
static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5444
{
5445
	struct cgrp_cset_link *link;
5446
	struct css_set *cset;
T
Tejun Heo 已提交
5447 5448 5449 5450 5451
	char *name_buf;

	name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
	if (!name_buf)
		return -ENOMEM;
5452

5453
	down_read(&css_set_rwsem);
5454
	rcu_read_lock();
5455
	cset = rcu_dereference(current->cgroups);
5456
	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5457 5458
		struct cgroup *c = link->cgrp;

T
Tejun Heo 已提交
5459
		cgroup_name(c, name_buf, NAME_MAX + 1);
5460
		seq_printf(seq, "Root %d group %s\n",
T
Tejun Heo 已提交
5461
			   c->root->hierarchy_id, name_buf);
5462 5463
	}
	rcu_read_unlock();
5464
	up_read(&css_set_rwsem);
T
Tejun Heo 已提交
5465
	kfree(name_buf);
5466 5467 5468 5469
	return 0;
}

#define MAX_TASKS_SHOWN_PER_CSS 25
5470
static int cgroup_css_links_read(struct seq_file *seq, void *v)
5471
{
5472
	struct cgroup_subsys_state *css = seq_css(seq);
5473
	struct cgrp_cset_link *link;
5474

5475
	down_read(&css_set_rwsem);
5476
	list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5477
		struct css_set *cset = link->cset;
5478 5479
		struct task_struct *task;
		int count = 0;
T
Tejun Heo 已提交
5480

5481
		seq_printf(seq, "css_set %p\n", cset);
T
Tejun Heo 已提交
5482

5483
		list_for_each_entry(task, &cset->tasks, cg_list) {
T
Tejun Heo 已提交
5484 5485 5486 5487 5488 5489 5490 5491 5492
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
		}

		list_for_each_entry(task, &cset->mg_tasks, cg_list) {
			if (count++ > MAX_TASKS_SHOWN_PER_CSS)
				goto overflow;
			seq_printf(seq, "  task %d\n", task_pid_vnr(task));
5493
		}
T
Tejun Heo 已提交
5494 5495 5496
		continue;
	overflow:
		seq_puts(seq, "  ...\n");
5497
	}
5498
	up_read(&css_set_rwsem);
5499 5500 5501
	return 0;
}

5502
static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5503
{
5504
	return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522
}

static struct cftype debug_files[] =  {
	{
		.name = "taskcount",
		.read_u64 = debug_taskcount_read,
	},

	{
		.name = "current_css_set",
		.read_u64 = current_css_set_read,
	},

	{
		.name = "current_css_set_refcount",
		.read_u64 = current_css_set_refcount_read,
	},

5523 5524
	{
		.name = "current_css_set_cg_links",
5525
		.seq_show = current_css_set_cg_links_read,
5526 5527 5528 5529
	},

	{
		.name = "cgroup_css_links",
5530
		.seq_show = cgroup_css_links_read,
5531 5532
	},

5533 5534 5535 5536 5537
	{
		.name = "releasable",
		.read_u64 = releasable_read,
	},

5538 5539
	{ }	/* terminate */
};
5540

5541
struct cgroup_subsys debug_cgrp_subsys = {
5542 5543
	.css_alloc = debug_css_alloc,
	.css_free = debug_css_free,
5544
	.legacy_cftypes = debug_files,
5545 5546
};
#endif /* CONFIG_CGROUP_DEBUG */