cgroup.c 151.4 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31 32
#include "cgroup-internal.h"

33
#include <linux/cred.h>
34
#include <linux/errno.h>
35
#include <linux/init_task.h>
36
#include <linux/kernel.h>
37
#include <linux/magic.h>
38 39 40
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
41
#include <linux/proc_fs.h>
42 43
#include <linux/rcupdate.h>
#include <linux/sched.h>
44
#include <linux/sched/task.h>
45 46
#include <linux/slab.h>
#include <linux/spinlock.h>
47
#include <linux/percpu-rwsem.h>
48
#include <linux/string.h>
49
#include <linux/hashtable.h>
50
#include <linux/idr.h>
51
#include <linux/kthread.h>
A
Arun Sharma 已提交
52
#include <linux/atomic.h>
53
#include <linux/cpuset.h>
54 55
#include <linux/proc_ns.h>
#include <linux/nsproxy.h>
56
#include <linux/file.h>
T
Tejun Heo 已提交
57
#include <net/sock.h>
58

59 60 61
#define CREATE_TRACE_POINTS
#include <trace/events/cgroup.h>

T
Tejun Heo 已提交
62 63 64
#define CGROUP_FILE_NAME_MAX		(MAX_CGROUP_TYPE_NAMELEN +	\
					 MAX_CFTYPE_NAME + 2)

T
Tejun Heo 已提交
65 66 67 68
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
69
 * css_set_lock protects task->cgroups pointer, the list of css_set
70
 * objects, and the chain of tasks off each css_set.
T
Tejun Heo 已提交
71
 *
72 73
 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 * cgroup.h can use them for lockdep annotations.
T
Tejun Heo 已提交
74
 */
T
Tejun Heo 已提交
75
DEFINE_MUTEX(cgroup_mutex);
76
DEFINE_SPINLOCK(css_set_lock);
77 78

#ifdef CONFIG_PROVE_RCU
79
EXPORT_SYMBOL_GPL(cgroup_mutex);
80
EXPORT_SYMBOL_GPL(css_set_lock);
T
Tejun Heo 已提交
81 82
#endif

83
/*
84 85
 * Protects cgroup_idr and css_idr so that IDs can be released without
 * grabbing cgroup_mutex.
86 87 88
 */
static DEFINE_SPINLOCK(cgroup_idr_lock);

89 90 91 92 93 94
/*
 * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
 * against file removal/re-creation across css hiding.
 */
static DEFINE_SPINLOCK(cgroup_file_kn_lock);

95 96
struct percpu_rw_semaphore cgroup_threadgroup_rwsem;

T
Tejun Heo 已提交
97
#define cgroup_assert_mutex_or_rcu_locked()				\
98 99
	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
			   !lockdep_is_held(&cgroup_mutex),		\
T
Tejun Heo 已提交
100
			   "cgroup_mutex or RCU read lock required");
101

102 103 104 105 106 107 108 109
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

T
Tejun Heo 已提交
110
/* generate an array of cgroup subsystem pointers */
111
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
112
struct cgroup_subsys *cgroup_subsys[] = {
113 114
#include <linux/cgroup_subsys.h>
};
115 116 117 118 119
#undef SUBSYS

/* array of cgroup subsystem names */
#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
static const char *cgroup_subsys_name[] = {
120 121
#include <linux/cgroup_subsys.h>
};
122
#undef SUBSYS
123

124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
#define SUBSYS(_x)								\
	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key);			\
	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key);			\
	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key);			\
	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
#include <linux/cgroup_subsys.h>
#undef SUBSYS

#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
static struct static_key_true *cgroup_subsys_enabled_key[] = {
#include <linux/cgroup_subsys.h>
};
#undef SUBSYS

#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
#include <linux/cgroup_subsys.h>
};
#undef SUBSYS

145
/*
146
 * The default hierarchy, reserved for the subsystems that are otherwise
147 148
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
149
 */
T
Tejun Heo 已提交
150
struct cgroup_root cgrp_dfl_root;
T
Tejun Heo 已提交
151
EXPORT_SYMBOL_GPL(cgrp_dfl_root);
152

T
Tejun Heo 已提交
153 154 155 156
/*
 * The default hierarchy always exists but is hidden until mounted for the
 * first time.  This is for backward compatibility.
 */
T
Tejun Heo 已提交
157
static bool cgrp_dfl_visible;
158

159
/* some controllers are not supported in the default hierarchy */
T
Tejun Heo 已提交
160
static u16 cgrp_dfl_inhibit_ss_mask;
161

162
/* some controllers are implicitly enabled on the default hierarchy */
T
Tejun Heo 已提交
163
static u16 cgrp_dfl_implicit_ss_mask;
164

165 166 167
/* some controllers can be threaded on the default hierarchy */
static u16 cgrp_dfl_threaded_ss_mask;

168
/* The list of hierarchy roots */
169
LIST_HEAD(cgroup_roots);
170
static int cgroup_root_count;
171

T
Tejun Heo 已提交
172
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
173
static DEFINE_IDR(cgroup_hierarchy_idr);
174

175
/*
176 177 178 179 180
 * Assign a monotonically increasing serial number to csses.  It guarantees
 * cgroups with bigger numbers are newer than those with smaller numbers.
 * Also, as csses are always appended to the parent's ->children list, it
 * guarantees that sibling csses are always sorted in the ascending serial
 * number order on the list.  Protected by cgroup_mutex.
181
 */
182
static u64 css_serial_nr_next = 1;
183

184
/*
T
Tejun Heo 已提交
185 186
 * These bitmasks identify subsystems with specific features to avoid
 * having to do iterative checks repeatedly.
187
 */
188 189 190
static u16 have_fork_callback __read_mostly;
static u16 have_exit_callback __read_mostly;
static u16 have_free_callback __read_mostly;
T
Tejun Heo 已提交
191
static u16 have_canfork_callback __read_mostly;
192

193 194
/* cgroup namespace for init task */
struct cgroup_namespace init_cgroup_ns = {
195
	.count		= REFCOUNT_INIT(2),
196 197 198 199 200 201
	.user_ns	= &init_user_ns,
	.ns.ops		= &cgroupns_operations,
	.ns.inum	= PROC_CGROUP_INIT_INO,
	.root_cset	= &init_css_set,
};

202
static struct file_system_type cgroup2_fs_type;
203
static struct cftype cgroup_base_files[];
204

205 206
static int cgroup_apply_control(struct cgroup *cgrp);
static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
207
static void css_task_iter_advance(struct css_task_iter *it);
208
static int cgroup_destroy_locked(struct cgroup *cgrp);
209 210
static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
					      struct cgroup_subsys *ss);
211
static void css_release(struct percpu_ref *ref);
212
static void kill_css(struct cgroup_subsys_state *css);
213 214
static int cgroup_addrm_files(struct cgroup_subsys_state *css,
			      struct cgroup *cgrp, struct cftype cfts[],
215
			      bool is_add);
216

217 218 219 220 221 222 223 224
/**
 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
 * @ssid: subsys ID of interest
 *
 * cgroup_subsys_enabled() can only be used with literal subsys names which
 * is fine for individual subsystems but unsuitable for cgroup core.  This
 * is slower static_key_enabled() based test indexed by @ssid.
 */
225
bool cgroup_ssid_enabled(int ssid)
226
{
227 228 229
	if (CGROUP_SUBSYS_COUNT == 0)
		return false;

230 231 232
	return static_key_enabled(cgroup_subsys_enabled_key[ssid]);
}

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
/**
 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
 * @cgrp: the cgroup of interest
 *
 * The default hierarchy is the v2 interface of cgroup and this function
 * can be used to test whether a cgroup is on the default hierarchy for
 * cases where a subsystem should behave differnetly depending on the
 * interface version.
 *
 * The set of behaviors which change on the default hierarchy are still
 * being determined and the mount option is prefixed with __DEVEL__.
 *
 * List of changed behaviors:
 *
 * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
 *   and "name" are disallowed.
 *
 * - When mounting an existing superblock, mount options should match.
 *
 * - Remount is disallowed.
 *
 * - rename(2) is disallowed.
 *
 * - "tasks" is removed.  Everything should be at process granularity.  Use
 *   "cgroup.procs" instead.
 *
 * - "cgroup.procs" is not sorted.  pids will be unique unless they got
 *   recycled inbetween reads.
 *
 * - "release_agent" and "notify_on_release" are removed.  Replacement
 *   notification mechanism will be implemented.
 *
 * - "cgroup.clone_children" is removed.
 *
 * - "cgroup.subtree_populated" is available.  Its value is 0 if the cgroup
 *   and its descendants contain no task; otherwise, 1.  The file also
 *   generates kernfs notification which can be monitored through poll and
 *   [di]notify when the value of the file changes.
 *
 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
 *   take masks of ancestors with non-empty cpus/mems, instead of being
 *   moved to an ancestor.
 *
 * - cpuset: a task can be moved into an empty cpuset, and again it takes
 *   masks of ancestors.
 *
 * - memcg: use_hierarchy is on by default and the cgroup file for the flag
 *   is not created.
 *
 * - blkcg: blk-throttle becomes properly hierarchical.
 *
 * - debug: disallowed on the default hierarchy.
 */
286
bool cgroup_on_dfl(const struct cgroup *cgrp)
287 288 289 290
{
	return cgrp->root == &cgrp_dfl_root;
}

291 292 293 294 295 296 297
/* IDR wrappers which synchronize using cgroup_idr_lock */
static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
			    gfp_t gfp_mask)
{
	int ret;

	idr_preload(gfp_mask);
T
Tejun Heo 已提交
298
	spin_lock_bh(&cgroup_idr_lock);
299
	ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM);
T
Tejun Heo 已提交
300
	spin_unlock_bh(&cgroup_idr_lock);
301 302 303 304 305 306 307 308
	idr_preload_end();
	return ret;
}

static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
{
	void *ret;

T
Tejun Heo 已提交
309
	spin_lock_bh(&cgroup_idr_lock);
310
	ret = idr_replace(idr, ptr, id);
T
Tejun Heo 已提交
311
	spin_unlock_bh(&cgroup_idr_lock);
312 313 314 315 316
	return ret;
}

static void cgroup_idr_remove(struct idr *idr, int id)
{
T
Tejun Heo 已提交
317
	spin_lock_bh(&cgroup_idr_lock);
318
	idr_remove(idr, id);
T
Tejun Heo 已提交
319
	spin_unlock_bh(&cgroup_idr_lock);
320 321
}

322
static bool cgroup_has_tasks(struct cgroup *cgrp)
T
Tejun Heo 已提交
323
{
324 325
	return cgrp->nr_populated_csets;
}
T
Tejun Heo 已提交
326

327
bool cgroup_is_threaded(struct cgroup *cgrp)
328 329 330 331
{
	return cgrp->dom_cgrp != cgrp;
}

332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
/* can @cgrp host both domain and threaded children? */
static bool cgroup_is_mixable(struct cgroup *cgrp)
{
	/*
	 * Root isn't under domain level resource control exempting it from
	 * the no-internal-process constraint, so it can serve as a thread
	 * root and a parent of resource domains at the same time.
	 */
	return !cgroup_parent(cgrp);
}

/* can @cgrp become a thread root? should always be true for a thread root */
static bool cgroup_can_be_thread_root(struct cgroup *cgrp)
{
	/* mixables don't care */
	if (cgroup_is_mixable(cgrp))
		return true;

	/* domain roots can't be nested under threaded */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* can only have either domain or threaded children */
	if (cgrp->nr_populated_domain_children)
		return false;

	/* and no domain controllers can be enabled */
	if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)
		return false;

	return true;
}

/* is @cgrp root of a threaded subtree? */
366
bool cgroup_is_thread_root(struct cgroup *cgrp)
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
{
	/* thread root should be a domain */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* a domain w/ threaded children is a thread root */
	if (cgrp->nr_threaded_children)
		return true;

	/*
	 * A domain which has tasks and explicit threaded controllers
	 * enabled is a thread root.
	 */
	if (cgroup_has_tasks(cgrp) &&
	    (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask))
		return true;

	return false;
}

/* a domain which isn't connected to the root w/o brekage can't be used */
static bool cgroup_is_valid_domain(struct cgroup *cgrp)
{
	/* the cgroup itself can be a thread root */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* but the ancestors can't be unless mixable */
	while ((cgrp = cgroup_parent(cgrp))) {
		if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp))
			return false;
		if (cgroup_is_threaded(cgrp))
			return false;
	}

	return true;
T
Tejun Heo 已提交
403 404
}

405 406 407 408 409 410
/* subsystems visibly enabled on a cgroup */
static u16 cgroup_control(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);
	u16 root_ss_mask = cgrp->root->subsys_mask;

411 412 413 414 415 416 417 418
	if (parent) {
		u16 ss_mask = parent->subtree_control;

		/* threaded cgroups can only have threaded controllers */
		if (cgroup_is_threaded(cgrp))
			ss_mask &= cgrp_dfl_threaded_ss_mask;
		return ss_mask;
	}
419 420

	if (cgroup_on_dfl(cgrp))
421 422
		root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask |
				  cgrp_dfl_implicit_ss_mask);
423 424 425 426 427 428 429 430
	return root_ss_mask;
}

/* subsystems enabled on a cgroup */
static u16 cgroup_ss_mask(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);

431 432 433 434 435 436 437 438
	if (parent) {
		u16 ss_mask = parent->subtree_ss_mask;

		/* threaded cgroups can only have threaded controllers */
		if (cgroup_is_threaded(cgrp))
			ss_mask &= cgrp_dfl_threaded_ss_mask;
		return ss_mask;
	}
439 440 441 442

	return cgrp->root->subsys_mask;
}

T
Tejun Heo 已提交
443 444 445
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
446
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
T
Tejun Heo 已提交
447
 *
448 449 450 451 452
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
453 454
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
455
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
456
{
457
	if (ss)
458
		return rcu_dereference_check(cgrp->subsys[ss->id],
T
Tejun Heo 已提交
459
					lockdep_is_held(&cgroup_mutex));
460
	else
461
		return &cgrp->self;
T
Tejun Heo 已提交
462
}
463

464 465 466
/**
 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
467
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
468
 *
C
Chen Hanxiao 已提交
469
 * Similar to cgroup_css() but returns the effective css, which is defined
470 471 472 473 474 475 476 477 478 479
 * as the matching css of the nearest ancestor including self which has @ss
 * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
 * function is guaranteed to return non-NULL css.
 */
static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
						struct cgroup_subsys *ss)
{
	lockdep_assert_held(&cgroup_mutex);

	if (!ss)
480
		return &cgrp->self;
481

T
Tejun Heo 已提交
482 483
	/*
	 * This function is used while updating css associations and thus
484
	 * can't test the csses directly.  Test ss_mask.
T
Tejun Heo 已提交
485
	 */
486
	while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) {
T
Tejun Heo 已提交
487
		cgrp = cgroup_parent(cgrp);
488 489 490
		if (!cgrp)
			return NULL;
	}
491 492

	return cgroup_css(cgrp, ss);
T
Tejun Heo 已提交
493
}
494

T
Tejun Heo 已提交
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
/**
 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
 * @ss: the subsystem of interest
 *
 * Find and get the effective css of @cgrp for @ss.  The effective css is
 * defined as the matching css of the nearest ancestor including self which
 * has @ss enabled.  If @ss is not mounted on the hierarchy @cgrp is on,
 * the root css is returned, so this function always returns a valid css.
 * The returned css must be put using css_put().
 */
struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp,
					     struct cgroup_subsys *ss)
{
	struct cgroup_subsys_state *css;

	rcu_read_lock();

	do {
		css = cgroup_css(cgrp, ss);

		if (css && css_tryget_online(css))
			goto out_unlock;
		cgrp = cgroup_parent(cgrp);
	} while (cgrp);

	css = init_css_set.subsys[ss->id];
	css_get(css);
out_unlock:
	rcu_read_unlock();
	return css;
}

528
static void cgroup_get_live(struct cgroup *cgrp)
T
Tejun Heo 已提交
529 530 531 532 533
{
	WARN_ON_ONCE(cgroup_is_dead(cgrp));
	css_get(&cgrp->self);
}

T
Tejun Heo 已提交
534
struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
535
{
T
Tejun Heo 已提交
536
	struct cgroup *cgrp = of->kn->parent->priv;
T
Tejun Heo 已提交
537
	struct cftype *cft = of_cft(of);
T
Tejun Heo 已提交
538 539 540 541 542 543 544 545 546 547 548 549

	/*
	 * This is open and unprotected implementation of cgroup_css().
	 * seq_css() is only called from a kernfs file operation which has
	 * an active reference on the file.  Because all the subsystem
	 * files are drained before a css is disassociated with a cgroup,
	 * the matching css from the cgroup's subsys table is guaranteed to
	 * be and stay valid until the enclosing operation is complete.
	 */
	if (cft->ss)
		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
	else
550
		return &cgrp->self;
551
}
T
Tejun Heo 已提交
552
EXPORT_SYMBOL_GPL(of_css);
553

T
Tejun Heo 已提交
554 555 556 557 558 559
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
560
 * Should be called under cgroup_[tree_]mutex.
T
Tejun Heo 已提交
561 562 563 564 565 566 567 568
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

569 570 571 572 573 574 575 576 577 578 579 580 581 582
/**
 * for_each_e_css - iterate all effective css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_[tree_]mutex.
 */
#define for_each_e_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
			;						\
		else

583
/**
584
 * do_each_subsys_mask - filter for_each_subsys with a bitmask
585 586
 * @ss: the iteration cursor
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
587
 * @ss_mask: the bitmask
588 589
 *
 * The block will only run for cases where the ssid-th bit (1 << ssid) of
590
 * @ss_mask is set.
591
 */
592 593 594
#define do_each_subsys_mask(ss, ssid, ss_mask) do {			\
	unsigned long __ss_mask = (ss_mask);				\
	if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */	\
595
		(ssid) = 0;						\
596 597 598 599 600 601 602 603 604 605
		break;							\
	}								\
	for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) {	\
		(ss) = cgroup_subsys[ssid];				\
		{

#define while_each_subsys_mask()					\
		}							\
	}								\
} while (false)
606

607 608
/* iterate over child cgrps, lock should be held throughout iteration */
#define cgroup_for_each_live_child(child, cgrp)				\
609
	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
T
Tejun Heo 已提交
610
		if (({ lockdep_assert_held(&cgroup_mutex);		\
611 612 613
		       cgroup_is_dead(child); }))			\
			;						\
		else
614

615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
/* walk live descendants in preorder */
#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp)		\
	css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL))	\
		if (({ lockdep_assert_held(&cgroup_mutex);		\
		       (dsct) = (d_css)->cgroup;			\
		       cgroup_is_dead(dsct); }))			\
			;						\
		else

/* walk live descendants in postorder */
#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp)		\
	css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL))	\
		if (({ lockdep_assert_held(&cgroup_mutex);		\
		       (dsct) = (d_css)->cgroup;			\
		       cgroup_is_dead(dsct); }))			\
			;						\
		else

633 634
/*
 * The default css_set - used by init and its children prior to any
635 636 637 638 639
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */
T
Tejun Heo 已提交
640
struct css_set init_css_set = {
641
	.refcount		= REFCOUNT_INIT(1),
642
	.dom_cset		= &init_css_set,
643 644
	.tasks			= LIST_HEAD_INIT(init_css_set.tasks),
	.mg_tasks		= LIST_HEAD_INIT(init_css_set.mg_tasks),
T
Tejun Heo 已提交
645
	.task_iters		= LIST_HEAD_INIT(init_css_set.task_iters),
646
	.threaded_csets		= LIST_HEAD_INIT(init_css_set.threaded_csets),
T
Tejun Heo 已提交
647
	.cgrp_links		= LIST_HEAD_INIT(init_css_set.cgrp_links),
648 649 650
	.mg_preload_node	= LIST_HEAD_INIT(init_css_set.mg_preload_node),
	.mg_node		= LIST_HEAD_INIT(init_css_set.mg_node),
};
651

652
static int css_set_count	= 1;	/* 1 for init_css_set */
653

654 655 656 657 658
static bool css_set_threaded(struct css_set *cset)
{
	return cset->dom_cset != cset;
}

659 660 661
/**
 * css_set_populated - does a css_set contain any tasks?
 * @cset: target css_set
662 663 664 665 666
 *
 * css_set_populated() should be the same as !!cset->nr_tasks at steady
 * state. However, css_set_populated() can be called while a task is being
 * added to or removed from the linked list before the nr_tasks is
 * properly updated. Hence, we can't just look at ->nr_tasks here.
667 668 669
 */
static bool css_set_populated(struct css_set *cset)
{
670
	lockdep_assert_held(&css_set_lock);
671 672 673 674

	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);
}

675
/**
676
 * cgroup_update_populated - update the populated count of a cgroup
677 678 679
 * @cgrp: the target cgroup
 * @populated: inc or dec populated count
 *
680
 * One of the css_sets associated with @cgrp is either getting its first
681 682 683 684
 * task or losing the last.  Update @cgrp->nr_populated_* accordingly.  The
 * count is propagated towards root so that a given cgroup's
 * nr_populated_children is zero iff none of its descendants contain any
 * tasks.
685
 *
686 687 688 689 690
 * @cgrp's interface file "cgroup.populated" is zero if both
 * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
 * 1 otherwise.  When the sum changes from or to zero, userland is notified
 * that the content of the interface file has changed.  This can be used to
 * detect when @cgrp and its descendants become populated or empty.
691 692 693
 */
static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
{
694 695 696
	struct cgroup *child = NULL;
	int adj = populated ? 1 : -1;

697
	lockdep_assert_held(&css_set_lock);
698 699

	do {
700
		bool was_populated = cgroup_is_populated(cgrp);
701

702
		if (!child) {
703
			cgrp->nr_populated_csets += adj;
704 705 706 707 708 709
		} else {
			if (cgroup_is_threaded(child))
				cgrp->nr_populated_threaded_children += adj;
			else
				cgrp->nr_populated_domain_children += adj;
		}
710

711
		if (was_populated == cgroup_is_populated(cgrp))
712 713
			break;

714
		cgroup1_check_for_release(cgrp);
715 716
		cgroup_file_notify(&cgrp->events_file);

717
		child = cgrp;
T
Tejun Heo 已提交
718
		cgrp = cgroup_parent(cgrp);
719 720 721
	} while (cgrp);
}

722 723 724 725 726 727
/**
 * css_set_update_populated - update populated state of a css_set
 * @cset: target css_set
 * @populated: whether @cset is populated or depopulated
 *
 * @cset is either getting the first task or losing the last.  Update the
728
 * populated counters of all associated cgroups accordingly.
729 730 731 732 733
 */
static void css_set_update_populated(struct css_set *cset, bool populated)
{
	struct cgrp_cset_link *link;

734
	lockdep_assert_held(&css_set_lock);
735 736 737 738 739

	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
		cgroup_update_populated(link->cgrp, populated);
}

T
Tejun Heo 已提交
740 741 742 743 744 745 746 747 748 749 750
/**
 * css_set_move_task - move a task from one css_set to another
 * @task: task being moved
 * @from_cset: css_set @task currently belongs to (may be NULL)
 * @to_cset: new css_set @task is being moved to (may be NULL)
 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
 *
 * Move @task from @from_cset to @to_cset.  If @task didn't belong to any
 * css_set, @from_cset can be NULL.  If @task is being disassociated
 * instead of moved, @to_cset can be NULL.
 *
751
 * This function automatically handles populated counter updates and
752 753
 * css_task_iter adjustments but the caller is responsible for managing
 * @from_cset and @to_cset's reference counts.
T
Tejun Heo 已提交
754 755 756 757 758
 */
static void css_set_move_task(struct task_struct *task,
			      struct css_set *from_cset, struct css_set *to_cset,
			      bool use_mg_tasks)
{
759
	lockdep_assert_held(&css_set_lock);
T
Tejun Heo 已提交
760

761 762 763
	if (to_cset && !css_set_populated(to_cset))
		css_set_update_populated(to_cset, true);

T
Tejun Heo 已提交
764
	if (from_cset) {
765 766
		struct css_task_iter *it, *pos;

T
Tejun Heo 已提交
767
		WARN_ON_ONCE(list_empty(&task->cg_list));
768 769 770 771 772 773 774 775 776 777 778 779 780

		/*
		 * @task is leaving, advance task iterators which are
		 * pointing to it so that they can resume at the next
		 * position.  Advancing an iterator might remove it from
		 * the list, use safe walk.  See css_task_iter_advance*()
		 * for details.
		 */
		list_for_each_entry_safe(it, pos, &from_cset->task_iters,
					 iters_node)
			if (it->task_pos == &task->cg_list)
				css_task_iter_advance(it);

T
Tejun Heo 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
		list_del_init(&task->cg_list);
		if (!css_set_populated(from_cset))
			css_set_update_populated(from_cset, false);
	} else {
		WARN_ON_ONCE(!list_empty(&task->cg_list));
	}

	if (to_cset) {
		/*
		 * We are synchronized through cgroup_threadgroup_rwsem
		 * against PF_EXITING setting such that we can't race
		 * against cgroup_exit() changing the css_set to
		 * init_css_set and dropping the old one.
		 */
		WARN_ON_ONCE(task->flags & PF_EXITING);

		rcu_assign_pointer(task->cgroups, to_cset);
		list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :
							     &to_cset->tasks);
	}
}

803 804 805 806 807
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
808
#define CSS_SET_HASH_BITS	7
809
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
810

811
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
812
{
813
	unsigned long key = 0UL;
814 815
	struct cgroup_subsys *ss;
	int i;
816

817
	for_each_subsys(ss, i)
818 819
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
820

821
	return key;
822 823
}

824
void put_css_set_locked(struct css_set *cset)
825
{
826
	struct cgrp_cset_link *link, *tmp_link;
T
Tejun Heo 已提交
827 828
	struct cgroup_subsys *ss;
	int ssid;
829

830
	lockdep_assert_held(&css_set_lock);
831

832
	if (!refcount_dec_and_test(&cset->refcount))
833
		return;
834

835 836
	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));

837 838
	/* This css_set is dead. unlink it and release cgroup and css refs */
	for_each_subsys(ss, ssid) {
T
Tejun Heo 已提交
839
		list_del(&cset->e_cset_node[ssid]);
840 841
		css_put(cset->subsys[ssid]);
	}
842
	hash_del(&cset->hlist);
843 844
	css_set_count--;

845 846 847
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
848 849
		if (cgroup_parent(link->cgrp))
			cgroup_put(link->cgrp);
850
		kfree(link);
851
	}
852

853 854 855 856 857
	if (css_set_threaded(cset)) {
		list_del(&cset->threaded_csets_node);
		put_css_set_locked(cset->dom_cset);
	}

858
	kfree_rcu(cset, rcu_head);
859 860
}

861
/**
862
 * compare_css_sets - helper function for find_existing_css_set().
863 864
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
865 866 867
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
868
 * Returns true if "cset" matches "old_cset" except for the hierarchy
869 870
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
871 872
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
873 874 875
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
876
	struct cgroup *new_dfl_cgrp;
877 878
	struct list_head *l1, *l2;

879 880 881 882 883 884
	/*
	 * On the default hierarchy, there can be csets which are
	 * associated with the same set of cgroups but different csses.
	 * Let's first ensure that csses match.
	 */
	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
885 886
		return false;

887 888 889 890 891 892 893 894 895 896

	/* @cset's domain should match the default cgroup's */
	if (cgroup_on_dfl(new_cgrp))
		new_dfl_cgrp = new_cgrp;
	else
		new_dfl_cgrp = old_cset->dfl_cgrp;

	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)
		return false;

897 898
	/*
	 * Compare cgroup pointers in order to distinguish between
899 900 901
	 * different cgroups in hierarchies.  As different cgroups may
	 * share the same effective css, this comparison is always
	 * necessary.
902
	 */
903 904
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
905
	while (1) {
906
		struct cgrp_cset_link *link1, *link2;
907
		struct cgroup *cgrp1, *cgrp2;
908 909 910 911

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
912 913
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
914 915
			break;
		} else {
916
			BUG_ON(l2 == &old_cset->cgrp_links);
917 918
		}
		/* Locate the cgroups associated with these links. */
919 920 921 922
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
923
		/* Hierarchies should be linked in the same order. */
924
		BUG_ON(cgrp1->root != cgrp2->root);
925 926 927 928 929 930 931 932

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
933 934
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
935 936
				return false;
		} else {
937
			if (cgrp1 != cgrp2)
938 939 940 941 942 943
				return false;
		}
	}
	return true;
}

944 945 946 947 948
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
949
 */
950 951 952
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
953
{
954
	struct cgroup_root *root = cgrp->root;
955
	struct cgroup_subsys *ss;
956
	struct css_set *cset;
957
	unsigned long key;
958
	int i;
959

B
Ben Blum 已提交
960 961 962 963 964
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
965
	for_each_subsys(ss, i) {
966
		if (root->subsys_mask & (1UL << i)) {
967 968 969 970 971
			/*
			 * @ss is in this hierarchy, so we want the
			 * effective css from @cgrp.
			 */
			template[i] = cgroup_e_css(cgrp, ss);
972
		} else {
973 974 975 976
			/*
			 * @ss is not in this hierarchy, so we don't want
			 * to change the css.
			 */
977
			template[i] = old_cset->subsys[i];
978 979 980
		}
	}

981
	key = css_set_hash(template);
982 983
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
984 985 986
			continue;

		/* This css_set matches what we need */
987
		return cset;
988
	}
989 990 991 992 993

	/* No existing cgroup group matched */
	return NULL;
}

994
static void free_cgrp_cset_links(struct list_head *links_to_free)
995
{
996
	struct cgrp_cset_link *link, *tmp_link;
997

998 999
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
1000 1001 1002 1003
		kfree(link);
	}
}

1004 1005 1006 1007 1008 1009 1010
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
1011
 */
1012
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
1013
{
1014
	struct cgrp_cset_link *link;
1015
	int i;
1016 1017 1018

	INIT_LIST_HEAD(tmp_links);

1019
	for (i = 0; i < count; i++) {
1020
		link = kzalloc(sizeof(*link), GFP_KERNEL);
1021
		if (!link) {
1022
			free_cgrp_cset_links(tmp_links);
1023 1024
			return -ENOMEM;
		}
1025
		list_add(&link->cset_link, tmp_links);
1026 1027 1028 1029
	}
	return 0;
}

1030 1031
/**
 * link_css_set - a helper function to link a css_set to a cgroup
1032
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
1033
 * @cset: the css_set to be linked
1034 1035
 * @cgrp: the destination cgroup
 */
1036 1037
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
1038
{
1039
	struct cgrp_cset_link *link;
1040

1041
	BUG_ON(list_empty(tmp_links));
T
Tejun Heo 已提交
1042 1043 1044 1045

	if (cgroup_on_dfl(cgrp))
		cset->dfl_cgrp = cgrp;

1046 1047
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
1048
	link->cgrp = cgrp;
1049

1050
	/*
1051 1052
	 * Always add links to the tail of the lists so that the lists are
	 * in choronological order.
1053
	 */
1054
	list_move_tail(&link->cset_link, &cgrp->cset_links);
1055
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
1056 1057

	if (cgroup_parent(cgrp))
1058
		cgroup_get_live(cgrp);
1059 1060
}

1061 1062 1063 1064 1065 1066 1067
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
1068
 */
1069 1070
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
1071
{
1072
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
1073
	struct css_set *cset;
1074 1075
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
1076
	struct cgroup_subsys *ss;
1077
	unsigned long key;
T
Tejun Heo 已提交
1078
	int ssid;
1079

1080 1081
	lockdep_assert_held(&cgroup_mutex);

1082 1083
	/* First see if we already have a cgroup group that matches
	 * the desired set */
1084
	spin_lock_irq(&css_set_lock);
1085 1086 1087
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
1088
	spin_unlock_irq(&css_set_lock);
1089

1090 1091
	if (cset)
		return cset;
1092

1093
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
1094
	if (!cset)
1095 1096
		return NULL;

1097
	/* Allocate all the cgrp_cset_link objects that we'll need */
1098
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
1099
		kfree(cset);
1100 1101 1102
		return NULL;
	}

1103
	refcount_set(&cset->refcount, 1);
1104
	cset->dom_cset = cset;
1105
	INIT_LIST_HEAD(&cset->tasks);
T
Tejun Heo 已提交
1106
	INIT_LIST_HEAD(&cset->mg_tasks);
1107
	INIT_LIST_HEAD(&cset->task_iters);
1108
	INIT_LIST_HEAD(&cset->threaded_csets);
1109
	INIT_HLIST_NODE(&cset->hlist);
T
Tejun Heo 已提交
1110 1111 1112
	INIT_LIST_HEAD(&cset->cgrp_links);
	INIT_LIST_HEAD(&cset->mg_preload_node);
	INIT_LIST_HEAD(&cset->mg_node);
1113 1114 1115

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
1116
	memcpy(cset->subsys, template, sizeof(cset->subsys));
1117

1118
	spin_lock_irq(&css_set_lock);
1119
	/* Add reference counts and links from the new css_set. */
1120
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
1121
		struct cgroup *c = link->cgrp;
1122

1123 1124
		if (c->root == cgrp->root)
			c = cgrp;
1125
		link_css_set(&tmp_links, cset, c);
1126
	}
1127

1128
	BUG_ON(!list_empty(&tmp_links));
1129 1130

	css_set_count++;
1131

T
Tejun Heo 已提交
1132
	/* Add @cset to the hash table */
1133 1134
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
1135

1136 1137 1138
	for_each_subsys(ss, ssid) {
		struct cgroup_subsys_state *css = cset->subsys[ssid];

T
Tejun Heo 已提交
1139
		list_add_tail(&cset->e_cset_node[ssid],
1140 1141 1142
			      &css->cgroup->e_csets[ssid]);
		css_get(css);
	}
T
Tejun Heo 已提交
1143

1144
	spin_unlock_irq(&css_set_lock);
1145

1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
	/*
	 * If @cset should be threaded, look up the matching dom_cset and
	 * link them up.  We first fully initialize @cset then look for the
	 * dom_cset.  It's simpler this way and safe as @cset is guaranteed
	 * to stay empty until we return.
	 */
	if (cgroup_is_threaded(cset->dfl_cgrp)) {
		struct css_set *dcset;

		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);
		if (!dcset) {
			put_css_set(cset);
			return NULL;
		}

		spin_lock_irq(&css_set_lock);
		cset->dom_cset = dcset;
		list_add_tail(&cset->threaded_csets_node,
			      &dcset->threaded_csets);
		spin_unlock_irq(&css_set_lock);
	}

1168
	return cset;
1169 1170
}

1171
struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
1172
{
1173
	struct cgroup *root_cgrp = kf_root->kn->priv;
T
Tejun Heo 已提交
1174

1175
	return root_cgrp->root;
T
Tejun Heo 已提交
1176 1177
}

1178
static int cgroup_init_root_id(struct cgroup_root *root)
1179 1180 1181 1182 1183
{
	int id;

	lockdep_assert_held(&cgroup_mutex);

1184
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
1185 1186 1187 1188 1189 1190 1191
	if (id < 0)
		return id;

	root->hierarchy_id = id;
	return 0;
}

1192
static void cgroup_exit_root_id(struct cgroup_root *root)
1193 1194 1195
{
	lockdep_assert_held(&cgroup_mutex);

1196
	idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1197 1198
}

1199
void cgroup_free_root(struct cgroup_root *root)
1200 1201 1202 1203 1204 1205 1206
{
	if (root) {
		idr_destroy(&root->cgroup_idr);
		kfree(root);
	}
}

1207
static void cgroup_destroy_root(struct cgroup_root *root)
1208
{
1209
	struct cgroup *cgrp = &root->cgrp;
1210 1211
	struct cgrp_cset_link *link, *tmp_link;

1212 1213
	trace_cgroup_destroy_root(root);

1214
	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
1215

T
Tejun Heo 已提交
1216
	BUG_ON(atomic_read(&root->nr_cgrps));
1217
	BUG_ON(!list_empty(&cgrp->self.children));
1218 1219

	/* Rebind all subsystems back to the default hierarchy */
1220
	WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask));
1221 1222

	/*
1223 1224
	 * Release all the links from cset_links to this hierarchy's
	 * root cgroup
1225
	 */
1226
	spin_lock_irq(&css_set_lock);
1227 1228 1229 1230 1231 1232

	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
		kfree(link);
	}
1233

1234
	spin_unlock_irq(&css_set_lock);
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244

	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
		cgroup_root_count--;
	}

	cgroup_exit_root_id(root);

	mutex_unlock(&cgroup_mutex);

T
Tejun Heo 已提交
1245
	kernfs_destroy_root(root->kf_root);
1246 1247 1248
	cgroup_free_root(root);
}

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
/*
 * look up cgroup associated with current task's cgroup namespace on the
 * specified hierarchy
 */
static struct cgroup *
current_cgns_cgroup_from_root(struct cgroup_root *root)
{
	struct cgroup *res = NULL;
	struct css_set *cset;

	lockdep_assert_held(&css_set_lock);

	rcu_read_lock();

	cset = current->nsproxy->cgroup_ns->root_cset;
	if (cset == &init_css_set) {
		res = &root->cgrp;
	} else {
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
			struct cgroup *c = link->cgrp;

			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
	rcu_read_unlock();

	BUG_ON(!res);
	return res;
}

1284 1285
/* look up cgroup associated with given css_set on the specified hierarchy */
static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
1286
					    struct cgroup_root *root)
1287 1288 1289
{
	struct cgroup *res = NULL;

1290
	lockdep_assert_held(&cgroup_mutex);
1291
	lockdep_assert_held(&css_set_lock);
1292

1293
	if (cset == &init_css_set) {
1294
		res = &root->cgrp;
1295 1296
	} else if (root == &cgrp_dfl_root) {
		res = cset->dfl_cgrp;
1297
	} else {
1298 1299 1300
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
1301
			struct cgroup *c = link->cgrp;
1302

1303 1304 1305 1306 1307 1308
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
1309

1310 1311 1312 1313
	BUG_ON(!res);
	return res;
}

1314
/*
1315
 * Return the cgroup for "task" from the given hierarchy. Must be
1316
 * called with cgroup_mutex and css_set_lock held.
1317
 */
1318 1319
struct cgroup *task_cgroup_from_root(struct task_struct *task,
				     struct cgroup_root *root)
1320 1321 1322 1323 1324 1325 1326 1327 1328
{
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
	return cset_cgroup_from_root(task_css_set(task), root);
}

1329 1330 1331 1332 1333 1334
/*
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
1335
 * cgroup_attach_task() can increment it again.  Because a count of zero
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
1347
 * least one task in the system (init, pid == 1), therefore, root cgroup
1348
 * always has either children cgroups and/or using tasks.  So we don't
1349
 * need a special hack to ensure that root cgroup cannot be deleted.
1350 1351
 *
 * P.S.  One more locking exception.  RCU is used to guard the
1352
 * update of a tasks cgroup pointer by cgroup_attach_task()
1353 1354
 */

T
Tejun Heo 已提交
1355
static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
1356

T
Tejun Heo 已提交
1357 1358
static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
			      char *buf)
1359
{
1360 1361
	struct cgroup_subsys *ss = cft->ss;

T
Tejun Heo 已提交
1362 1363 1364
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
		snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
1365 1366
			 cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,
			 cft->name);
T
Tejun Heo 已提交
1367 1368 1369
	else
		strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
	return buf;
1370 1371
}

1372 1373 1374 1375
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
1376
 * S_IRUGO for read, S_IWUSR for write.
1377 1378
 */
static umode_t cgroup_file_mode(const struct cftype *cft)
1379
{
1380
	umode_t mode = 0;
1381

1382 1383 1384
	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
		mode |= S_IRUGO;

1385 1386 1387 1388 1389 1390
	if (cft->write_u64 || cft->write_s64 || cft->write) {
		if (cft->flags & CFTYPE_WORLD_WRITABLE)
			mode |= S_IWUGO;
		else
			mode |= S_IWUSR;
	}
1391 1392

	return mode;
1393 1394
}

1395
/**
1396
 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1397
 * @subtree_control: the new subtree_control mask to consider
1398
 * @this_ss_mask: available subsystems
1399 1400 1401 1402 1403
 *
 * On the default hierarchy, a subsystem may request other subsystems to be
 * enabled together through its ->depends_on mask.  In such cases, more
 * subsystems than specified in "cgroup.subtree_control" may be enabled.
 *
1404
 * This function calculates which subsystems need to be enabled if
1405
 * @subtree_control is to be applied while restricted to @this_ss_mask.
1406
 */
1407
static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask)
1408
{
1409
	u16 cur_ss_mask = subtree_control;
1410 1411 1412 1413 1414
	struct cgroup_subsys *ss;
	int ssid;

	lockdep_assert_held(&cgroup_mutex);

1415 1416
	cur_ss_mask |= cgrp_dfl_implicit_ss_mask;

1417
	while (true) {
1418
		u16 new_ss_mask = cur_ss_mask;
1419

1420
		do_each_subsys_mask(ss, ssid, cur_ss_mask) {
1421
			new_ss_mask |= ss->depends_on;
1422
		} while_each_subsys_mask();
1423 1424 1425 1426 1427 1428

		/*
		 * Mask out subsystems which aren't available.  This can
		 * happen only if some depended-upon subsystems were bound
		 * to non-default hierarchies.
		 */
1429
		new_ss_mask &= this_ss_mask;
1430 1431 1432 1433 1434 1435

		if (new_ss_mask == cur_ss_mask)
			break;
		cur_ss_mask = new_ss_mask;
	}

1436 1437 1438
	return cur_ss_mask;
}

1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
/**
 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper undoes cgroup_kn_lock_live() and should be invoked before
 * the method finishes if locking succeeded.  Note that once this function
 * returns the cgroup returned by cgroup_kn_lock_live() may become
 * inaccessible any time.  If the caller intends to continue to access the
 * cgroup, it should pin it before invoking this function.
 */
1449
void cgroup_kn_unlock(struct kernfs_node *kn)
1450
{
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;

	mutex_unlock(&cgroup_mutex);

	kernfs_unbreak_active_protection(kn);
	cgroup_put(cgrp);
1462 1463
}

1464 1465 1466
/**
 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
1467
 * @drain_offline: perform offline draining on the cgroup
1468 1469 1470 1471 1472
 *
 * This helper is to be used by a cgroup kernfs method currently servicing
 * @kn.  It breaks the active protection, performs cgroup locking and
 * verifies that the associated cgroup is alive.  Returns the cgroup if
 * alive; otherwise, %NULL.  A successful return should be undone by a
1473 1474
 * matching cgroup_kn_unlock() invocation.  If @drain_offline is %true, the
 * cgroup is drained of offlining csses before return.
1475 1476 1477 1478 1479 1480
 *
 * Any cgroup kernfs method implementation which requires locking the
 * associated cgroup should use this helper.  It avoids nesting cgroup
 * locking under kernfs active protection and allows all kernfs operations
 * including self-removal.
 */
1481
struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline)
T
Tejun Heo 已提交
1482
{
1483 1484 1485 1486 1487 1488
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;
T
Tejun Heo 已提交
1489

1490
	/*
1491
	 * We're gonna grab cgroup_mutex which nests outside kernfs
1492 1493 1494
	 * active_ref.  cgroup liveliness check alone provides enough
	 * protection against removal.  Ensure @cgrp stays accessible and
	 * break the active_ref protection.
1495
	 */
1496 1497
	if (!cgroup_tryget(cgrp))
		return NULL;
1498 1499
	kernfs_break_active_protection(kn);

1500 1501 1502 1503
	if (drain_offline)
		cgroup_lock_and_drain_offline(cgrp);
	else
		mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1504

1505 1506 1507 1508 1509
	if (!cgroup_is_dead(cgrp))
		return cgrp;

	cgroup_kn_unlock(kn);
	return NULL;
1510
}
T
Tejun Heo 已提交
1511

1512
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
1513
{
T
Tejun Heo 已提交
1514
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
1515

1516
	lockdep_assert_held(&cgroup_mutex);
1517 1518 1519 1520 1521 1522 1523 1524 1525 1526

	if (cft->file_offset) {
		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
		struct cgroup_file *cfile = (void *)css + cft->file_offset;

		spin_lock_irq(&cgroup_file_kn_lock);
		cfile->kn = NULL;
		spin_unlock_irq(&cgroup_file_kn_lock);
	}

T
Tejun Heo 已提交
1527
	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
T
Tejun Heo 已提交
1528 1529
}

1530
/**
1531 1532
 * css_clear_dir - remove subsys files in a cgroup directory
 * @css: taget css
1533
 */
1534
static void css_clear_dir(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
1535
{
1536
	struct cgroup *cgrp = css->cgroup;
1537
	struct cftype *cfts;
T
Tejun Heo 已提交
1538

1539 1540 1541 1542 1543
	if (!(css->flags & CSS_VISIBLE))
		return;

	css->flags &= ~CSS_VISIBLE;

1544 1545
	list_for_each_entry(cfts, &css->ss->cfts, node)
		cgroup_addrm_files(css, cgrp, cfts, false);
1546 1547
}

1548
/**
1549 1550
 * css_populate_dir - create subsys files in a cgroup directory
 * @css: target css
1551 1552 1553
 *
 * On failure, no file is added.
 */
1554
static int css_populate_dir(struct cgroup_subsys_state *css)
1555
{
1556
	struct cgroup *cgrp = css->cgroup;
1557 1558
	struct cftype *cfts, *failed_cfts;
	int ret;
1559

1560
	if ((css->flags & CSS_VISIBLE) || !cgrp->kn)
1561 1562
		return 0;

1563 1564
	if (!css->ss) {
		if (cgroup_on_dfl(cgrp))
1565
			cfts = cgroup_base_files;
1566
		else
1567
			cfts = cgroup1_base_files;
1568

1569 1570
		return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true);
	}
1571

1572 1573 1574 1575 1576
	list_for_each_entry(cfts, &css->ss->cfts, node) {
		ret = cgroup_addrm_files(css, cgrp, cfts, true);
		if (ret < 0) {
			failed_cfts = cfts;
			goto err;
1577 1578
		}
	}
1579 1580 1581

	css->flags |= CSS_VISIBLE;

1582 1583
	return 0;
err:
1584 1585 1586 1587 1588
	list_for_each_entry(cfts, &css->ss->cfts, node) {
		if (cfts == failed_cfts)
			break;
		cgroup_addrm_files(css, cgrp, cfts, false);
	}
1589 1590 1591
	return ret;
}

1592
int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
1593
{
1594
	struct cgroup *dcgrp = &dst_root->cgrp;
1595
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1596
	int ssid, i, ret;
1597

T
Tejun Heo 已提交
1598
	lockdep_assert_held(&cgroup_mutex);
1599

1600
	do_each_subsys_mask(ss, ssid, ss_mask) {
1601 1602 1603 1604 1605 1606 1607
		/*
		 * If @ss has non-root csses attached to it, can't move.
		 * If @ss is an implicit controller, it is exempt from this
		 * rule and can be stolen.
		 */
		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) &&
		    !ss->implicit_on_dfl)
T
Tejun Heo 已提交
1608
			return -EBUSY;
1609

1610
		/* can't move between two non-dummy roots either */
1611
		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1612
			return -EBUSY;
1613
	} while_each_subsys_mask();
1614

1615
	do_each_subsys_mask(ss, ssid, ss_mask) {
1616 1617 1618
		struct cgroup_root *src_root = ss->root;
		struct cgroup *scgrp = &src_root->cgrp;
		struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);
T
Tejun Heo 已提交
1619
		struct css_set *cset;
1620

1621
		WARN_ON(!css || cgroup_css(dcgrp, ss));
1622

1623 1624 1625 1626
		/* disable from the source */
		src_root->subsys_mask &= ~(1 << ssid);
		WARN_ON(cgroup_apply_control(scgrp));
		cgroup_finalize_control(scgrp, 0);
1627

1628
		/* rebind */
1629 1630
		RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
		rcu_assign_pointer(dcgrp->subsys[ssid], css);
1631
		ss->root = dst_root;
1632
		css->cgroup = dcgrp;
1633

1634
		spin_lock_irq(&css_set_lock);
T
Tejun Heo 已提交
1635 1636
		hash_for_each(css_set_table, i, cset, hlist)
			list_move_tail(&cset->e_cset_node[ss->id],
1637
				       &dcgrp->e_csets[ss->id]);
1638
		spin_unlock_irq(&css_set_lock);
T
Tejun Heo 已提交
1639

1640
		/* default hierarchy doesn't enable controllers by default */
1641
		dst_root->subsys_mask |= 1 << ssid;
1642 1643 1644
		if (dst_root == &cgrp_dfl_root) {
			static_branch_enable(cgroup_subsys_on_dfl_key[ssid]);
		} else {
1645
			dcgrp->subtree_control |= 1 << ssid;
1646
			static_branch_disable(cgroup_subsys_on_dfl_key[ssid]);
1647
		}
1648

1649 1650 1651 1652 1653
		ret = cgroup_apply_control(dcgrp);
		if (ret)
			pr_warn("partial failure to rebind %s controller (err=%d)\n",
				ss->name, ret);

1654 1655
		if (ss->bind)
			ss->bind(css);
1656
	} while_each_subsys_mask();
1657

1658
	kernfs_activate(dcgrp->kn);
1659 1660 1661
	return 0;
}

1662 1663
int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node,
		     struct kernfs_root *kf_root)
1664
{
F
Felipe Balbi 已提交
1665
	int len = 0;
1666 1667 1668 1669 1670 1671 1672 1673
	char *buf = NULL;
	struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root);
	struct cgroup *ns_cgroup;

	buf = kmalloc(PATH_MAX, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

1674
	spin_lock_irq(&css_set_lock);
1675 1676
	ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot);
	len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX);
1677
	spin_unlock_irq(&css_set_lock);
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688

	if (len >= PATH_MAX)
		len = -ERANGE;
	else if (len > 0) {
		seq_escape(sf, buf, " \t\n\\");
		len = 0;
	}
	kfree(buf);
	return len;
}

1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727
static int parse_cgroup_root_flags(char *data, unsigned int *root_flags)
{
	char *token;

	*root_flags = 0;

	if (!data)
		return 0;

	while ((token = strsep(&data, ",")) != NULL) {
		if (!strcmp(token, "nsdelegate")) {
			*root_flags |= CGRP_ROOT_NS_DELEGATE;
			continue;
		}

		pr_err("cgroup2: unknown option \"%s\"\n", token);
		return -EINVAL;
	}

	return 0;
}

static void apply_cgroup_root_flags(unsigned int root_flags)
{
	if (current->nsproxy->cgroup_ns == &init_cgroup_ns) {
		if (root_flags & CGRP_ROOT_NS_DELEGATE)
			cgrp_dfl_root.flags |= CGRP_ROOT_NS_DELEGATE;
		else
			cgrp_dfl_root.flags &= ~CGRP_ROOT_NS_DELEGATE;
	}
}

static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root)
{
	if (cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE)
		seq_puts(seq, ",nsdelegate");
	return 0;
}

T
Tejun Heo 已提交
1728
static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1729
{
1730 1731 1732 1733 1734 1735 1736 1737 1738
	unsigned int root_flags;
	int ret;

	ret = parse_cgroup_root_flags(data, &root_flags);
	if (ret)
		return ret;

	apply_cgroup_root_flags(root_flags);
	return 0;
1739 1740
}

1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
/*
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
 * words after the first mount.
 */
static bool use_task_css_set_links __read_mostly;

static void cgroup_enable_task_cg_lists(void)
{
	struct task_struct *p, *g;

1753
	spin_lock_irq(&css_set_lock);
1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775

	if (use_task_css_set_links)
		goto out_unlock;

	use_task_css_set_links = true;

	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
	do_each_thread(g, p) {
		WARN_ON_ONCE(!list_empty(&p->cg_list) ||
			     task_css_set(p) != &init_css_set);

		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
1776 1777
		 * Do it while holding siglock so that we don't end up
		 * racing against cgroup_exit().
1778 1779 1780 1781
		 *
		 * Interrupts were already disabled while acquiring
		 * the css_set_lock, so we do not need to disable it
		 * again when acquiring the sighand->siglock here.
1782
		 */
1783
		spin_lock(&p->sighand->siglock);
1784 1785 1786
		if (!(p->flags & PF_EXITING)) {
			struct css_set *cset = task_css_set(p);

1787 1788
			if (!css_set_populated(cset))
				css_set_update_populated(cset, true);
1789
			list_add_tail(&p->cg_list, &cset->tasks);
1790
			get_css_set(cset);
1791
			cset->nr_tasks++;
1792
		}
1793
		spin_unlock(&p->sighand->siglock);
1794 1795 1796
	} while_each_thread(g, p);
	read_unlock(&tasklist_lock);
out_unlock:
1797
	spin_unlock_irq(&css_set_lock);
1798
}
1799

1800 1801
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
T
Tejun Heo 已提交
1802 1803 1804
	struct cgroup_subsys *ss;
	int ssid;

1805 1806
	INIT_LIST_HEAD(&cgrp->self.sibling);
	INIT_LIST_HEAD(&cgrp->self.children);
1807
	INIT_LIST_HEAD(&cgrp->cset_links);
1808 1809
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
1810
	cgrp->self.cgroup = cgrp;
1811
	cgrp->self.flags |= CSS_ONLINE;
1812
	cgrp->dom_cgrp = cgrp;
1813 1814
	cgrp->max_descendants = INT_MAX;
	cgrp->max_depth = INT_MAX;
T
Tejun Heo 已提交
1815 1816 1817

	for_each_subsys(ss, ssid)
		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1818 1819

	init_waitqueue_head(&cgrp->offline_waitq);
1820
	INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);
1821
}
1822

1823
void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts)
1824
{
1825
	struct cgroup *cgrp = &root->cgrp;
1826

1827
	INIT_LIST_HEAD(&root->root_list);
1828
	atomic_set(&root->nr_cgrps, 1);
1829
	cgrp->root = root;
1830
	init_cgroup_housekeeping(cgrp);
1831
	idr_init(&root->cgroup_idr);
1832 1833 1834 1835 1836 1837

	root->flags = opts->flags;
	if (opts->release_agent)
		strcpy(root->release_agent_path, opts->release_agent);
	if (opts->name)
		strcpy(root->name, opts->name);
1838
	if (opts->cpuset_clone_children)
1839
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1840 1841
}

1842
int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags)
1843
{
1844
	LIST_HEAD(tmp_links);
1845
	struct cgroup *root_cgrp = &root->cgrp;
1846
	struct kernfs_syscall_ops *kf_sops;
1847 1848
	struct css_set *cset;
	int i, ret;
1849

1850
	lockdep_assert_held(&cgroup_mutex);
1851

V
Vladimir Davydov 已提交
1852
	ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL);
1853
	if (ret < 0)
T
Tejun Heo 已提交
1854
		goto out;
1855
	root_cgrp->id = ret;
1856
	root_cgrp->ancestor_ids[0] = ret;
1857

1858 1859
	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release,
			      ref_flags, GFP_KERNEL);
1860 1861 1862
	if (ret)
		goto out;

1863
	/*
1864
	 * We're accessing css_set_count without locking css_set_lock here,
1865
	 * but that's OK - it can only be increased by someone holding
1866 1867 1868
	 * cgroup_lock, and that's us.  Later rebinding may disable
	 * controllers on the default hierarchy and thus create new csets,
	 * which can't be more than the existing ones.  Allocate 2x.
1869
	 */
1870
	ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links);
1871
	if (ret)
1872
		goto cancel_ref;
1873

1874
	ret = cgroup_init_root_id(root);
1875
	if (ret)
1876
		goto cancel_ref;
1877

1878 1879 1880 1881
	kf_sops = root == &cgrp_dfl_root ?
		&cgroup_kf_syscall_ops : &cgroup1_kf_syscall_ops;

	root->kf_root = kernfs_create_root(kf_sops,
S
Shaohua Li 已提交
1882 1883
					   KERNFS_ROOT_CREATE_DEACTIVATED |
					   KERNFS_ROOT_SUPPORT_EXPORTOP,
T
Tejun Heo 已提交
1884 1885 1886 1887 1888 1889
					   root_cgrp);
	if (IS_ERR(root->kf_root)) {
		ret = PTR_ERR(root->kf_root);
		goto exit_root_id;
	}
	root_cgrp->kn = root->kf_root->kn;
1890

1891
	ret = css_populate_dir(&root_cgrp->self);
1892
	if (ret)
T
Tejun Heo 已提交
1893
		goto destroy_root;
1894

1895
	ret = rebind_subsystems(root, ss_mask);
1896
	if (ret)
T
Tejun Heo 已提交
1897
		goto destroy_root;
1898

1899 1900
	trace_cgroup_setup_root(root);

1901 1902 1903 1904 1905 1906 1907
	/*
	 * There must be no failure case after here, since rebinding takes
	 * care of subsystems' refcounts, which are explicitly dropped in
	 * the failure exit path.
	 */
	list_add(&root->root_list, &cgroup_roots);
	cgroup_root_count++;
A
Al Viro 已提交
1908

1909
	/*
1910
	 * Link the root cgroup in this hierarchy into all the css_set
1911 1912
	 * objects.
	 */
1913
	spin_lock_irq(&css_set_lock);
1914
	hash_for_each(css_set_table, i, cset, hlist) {
1915
		link_css_set(&tmp_links, cset, root_cgrp);
1916 1917 1918
		if (css_set_populated(cset))
			cgroup_update_populated(root_cgrp, true);
	}
1919
	spin_unlock_irq(&css_set_lock);
1920

1921
	BUG_ON(!list_empty(&root_cgrp->self.children));
1922
	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1923

T
Tejun Heo 已提交
1924
	kernfs_activate(root_cgrp->kn);
1925
	ret = 0;
T
Tejun Heo 已提交
1926
	goto out;
1927

T
Tejun Heo 已提交
1928 1929 1930 1931
destroy_root:
	kernfs_destroy_root(root->kf_root);
	root->kf_root = NULL;
exit_root_id:
1932
	cgroup_exit_root_id(root);
1933
cancel_ref:
1934
	percpu_ref_exit(&root_cgrp->self.refcnt);
T
Tejun Heo 已提交
1935
out:
1936 1937
	free_cgrp_cset_links(&tmp_links);
	return ret;
1938 1939
}

1940 1941 1942
struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags,
			       struct cgroup_root *root, unsigned long magic,
			       struct cgroup_namespace *ns)
1943
{
T
Tejun Heo 已提交
1944
	struct dentry *dentry;
L
Li Zefan 已提交
1945
	bool new_sb;
1946

1947
	dentry = kernfs_mount(fs_type, flags, root->kf_root, magic, &new_sb);
1948

1949
	/*
1950 1951
	 * In non-init cgroup namespace, instead of root cgroup's dentry,
	 * we return the dentry corresponding to the cgroupns->root_cgrp.
1952
	 */
1953 1954 1955
	if (!IS_ERR(dentry) && ns != &init_cgroup_ns) {
		struct dentry *nsdentry;
		struct cgroup *cgrp;
1956

1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
		mutex_lock(&cgroup_mutex);
		spin_lock_irq(&css_set_lock);

		cgrp = cset_cgroup_from_root(ns->root_cset, root);

		spin_unlock_irq(&css_set_lock);
		mutex_unlock(&cgroup_mutex);

		nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb);
		dput(dentry);
		dentry = nsdentry;
1968 1969
	}

1970 1971 1972 1973 1974 1975
	if (IS_ERR(dentry) || !new_sb)
		cgroup_put(&root->cgrp);

	return dentry;
}

A
Al Viro 已提交
1976
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1977
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
1978
			 void *data)
1979
{
1980
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
T
Tejun Heo 已提交
1981
	struct dentry *dentry;
1982
	int ret;
1983

1984 1985 1986 1987 1988 1989 1990 1991
	get_cgroup_ns(ns);

	/* Check if the caller has permission to mount. */
	if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) {
		put_cgroup_ns(ns);
		return ERR_PTR(-EPERM);
	}

1992 1993 1994 1995 1996 1997
	/*
	 * The first time anyone tries to mount a cgroup, enable the list
	 * linking each css_set to its tasks and fix up all existing tasks.
	 */
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();
1998

1999
	if (fs_type == &cgroup2_fs_type) {
2000 2001 2002 2003
		unsigned int root_flags;

		ret = parse_cgroup_root_flags(data, &root_flags);
		if (ret) {
2004
			put_cgroup_ns(ns);
2005
			return ERR_PTR(ret);
2006
		}
2007

T
Tejun Heo 已提交
2008
		cgrp_dfl_visible = true;
2009
		cgroup_get_live(&cgrp_dfl_root.cgrp);
2010 2011 2012

		dentry = cgroup_do_mount(&cgroup2_fs_type, flags, &cgrp_dfl_root,
					 CGROUP2_SUPER_MAGIC, ns);
2013 2014
		if (!IS_ERR(dentry))
			apply_cgroup_root_flags(root_flags);
2015 2016 2017
	} else {
		dentry = cgroup1_mount(&cgroup_fs_type, flags, data,
				       CGROUP_SUPER_MAGIC, ns);
2018 2019
	}

2020
	put_cgroup_ns(ns);
T
Tejun Heo 已提交
2021 2022
	return dentry;
}
2023

T
Tejun Heo 已提交
2024 2025 2026
static void cgroup_kill_sb(struct super_block *sb)
{
	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
2027
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
2028

2029
	/*
2030 2031 2032
	 * If @root doesn't have any mounts or children, start killing it.
	 * This prevents new mounts by disabling percpu_ref_tryget_live().
	 * cgroup_mount() may wait for @root's release.
2033 2034
	 *
	 * And don't kill the default root.
2035
	 */
2036
	if (!list_empty(&root->cgrp.self.children) ||
2037
	    root == &cgrp_dfl_root)
2038 2039 2040
		cgroup_put(&root->cgrp);
	else
		percpu_ref_kill(&root->cgrp.self.refcnt);
2041

T
Tejun Heo 已提交
2042
	kernfs_kill_sb(sb);
2043
}
2044

2045
struct file_system_type cgroup_fs_type = {
2046
	.name = "cgroup",
A
Al Viro 已提交
2047
	.mount = cgroup_mount,
2048
	.kill_sb = cgroup_kill_sb,
S
Serge Hallyn 已提交
2049
	.fs_flags = FS_USERNS_MOUNT,
2050
};
2051

2052 2053 2054 2055
static struct file_system_type cgroup2_fs_type = {
	.name = "cgroup2",
	.mount = cgroup_mount,
	.kill_sb = cgroup_kill_sb,
S
Serge Hallyn 已提交
2056
	.fs_flags = FS_USERNS_MOUNT,
2057
};
2058

2059 2060
int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen,
			  struct cgroup_namespace *ns)
2061 2062 2063
{
	struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root);

2064
	return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen);
2065 2066
}

2067 2068
int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
		   struct cgroup_namespace *ns)
2069
{
2070
	int ret;
2071 2072

	mutex_lock(&cgroup_mutex);
2073
	spin_lock_irq(&css_set_lock);
2074 2075 2076

	ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns);

2077
	spin_unlock_irq(&css_set_lock);
2078 2079 2080 2081 2082 2083
	mutex_unlock(&cgroup_mutex);

	return ret;
}
EXPORT_SYMBOL_GPL(cgroup_path_ns);

2084
/**
2085
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2086 2087 2088 2089
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
2090 2091 2092 2093 2094
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
T
Tejun Heo 已提交
2095
 * Return value is the same as kernfs_path().
2096
 */
2097
int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
2098
{
2099
	struct cgroup_root *root;
2100
	struct cgroup *cgrp;
T
Tejun Heo 已提交
2101
	int hierarchy_id = 1;
2102
	int ret;
2103 2104

	mutex_lock(&cgroup_mutex);
2105
	spin_lock_irq(&css_set_lock);
2106

2107 2108
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

2109 2110
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
2111
		ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns);
2112 2113
	} else {
		/* if no hierarchy exists, everyone is in "/" */
2114
		ret = strlcpy(buf, "/", buflen);
2115 2116
	}

2117
	spin_unlock_irq(&css_set_lock);
2118
	mutex_unlock(&cgroup_mutex);
2119
	return ret;
2120
}
2121
EXPORT_SYMBOL_GPL(task_cgroup_path);
2122

2123
/**
2124
 * cgroup_migrate_add_task - add a migration target task to a migration context
2125
 * @task: target task
2126
 * @mgctx: target migration context
2127
 *
2128 2129 2130 2131
 * Add @task, which is a migration target, to @mgctx->tset.  This function
 * becomes noop if @task doesn't need to be migrated.  @task's css_set
 * should have been added as a migration source and @task->cg_list will be
 * moved from the css_set's tasks list to mg_tasks one.
2132
 */
2133 2134
static void cgroup_migrate_add_task(struct task_struct *task,
				    struct cgroup_mgctx *mgctx)
2135 2136 2137
{
	struct css_set *cset;

2138
	lockdep_assert_held(&css_set_lock);
2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151

	/* @task either already exited or can't exit until the end */
	if (task->flags & PF_EXITING)
		return;

	/* leave @task alone if post_fork() hasn't linked it yet */
	if (list_empty(&task->cg_list))
		return;

	cset = task_css_set(task);
	if (!cset->mg_src_cgrp)
		return;

2152 2153
	mgctx->tset.nr_tasks++;

2154 2155
	list_move_tail(&task->cg_list, &cset->mg_tasks);
	if (list_empty(&cset->mg_node))
2156 2157
		list_add_tail(&cset->mg_node,
			      &mgctx->tset.src_csets);
2158
	if (list_empty(&cset->mg_dst_cset->mg_node))
2159
		list_add_tail(&cset->mg_dst_cset->mg_node,
2160
			      &mgctx->tset.dst_csets);
2161 2162
}

2163 2164 2165
/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
2166
 * @dst_cssp: output variable for the destination css
2167 2168 2169
 *
 * @tset iteration is initialized and the first task is returned.
 */
2170 2171
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
					 struct cgroup_subsys_state **dst_cssp)
2172
{
2173 2174 2175
	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
	tset->cur_task = NULL;

2176
	return cgroup_taskset_next(tset, dst_cssp);
2177 2178 2179 2180 2181
}

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
2182
 * @dst_cssp: output variable for the destination css
2183 2184 2185 2186
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
2187 2188
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
					struct cgroup_subsys_state **dst_cssp)
2189
{
2190 2191
	struct css_set *cset = tset->cur_cset;
	struct task_struct *task = tset->cur_task;
2192

2193 2194 2195 2196 2197 2198
	while (&cset->mg_node != tset->csets) {
		if (!task)
			task = list_first_entry(&cset->mg_tasks,
						struct task_struct, cg_list);
		else
			task = list_next_entry(task, cg_list);
2199

2200 2201 2202
		if (&task->cg_list != &cset->mg_tasks) {
			tset->cur_cset = cset;
			tset->cur_task = task;
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214

			/*
			 * This function may be called both before and
			 * after cgroup_taskset_migrate().  The two cases
			 * can be distinguished by looking at whether @cset
			 * has its ->mg_dst_cset set.
			 */
			if (cset->mg_dst_cset)
				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];
			else
				*dst_cssp = cset->subsys[tset->ssid];

2215 2216
			return task;
		}
2217

2218 2219 2220
		cset = list_next_entry(cset, mg_node);
		task = NULL;
	}
2221

2222
	return NULL;
2223 2224
}

2225
/**
2226
 * cgroup_taskset_migrate - migrate a taskset
2227
 * @mgctx: migration context
2228
 *
2229
 * Migrate tasks in @mgctx as setup by migration preparation functions.
2230
 * This function fails iff one of the ->can_attach callbacks fails and
2231 2232
 * guarantees that either all or none of the tasks in @mgctx are migrated.
 * @mgctx is consumed regardless of success.
2233
 */
2234
static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
2235
{
2236
	struct cgroup_taskset *tset = &mgctx->tset;
2237
	struct cgroup_subsys *ss;
2238 2239
	struct task_struct *task, *tmp_task;
	struct css_set *cset, *tmp_cset;
2240
	int ssid, failed_ssid, ret;
2241 2242

	/* check that we can legitimately attach to the cgroup */
2243 2244 2245 2246 2247 2248 2249 2250 2251
	if (tset->nr_tasks) {
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ss->can_attach) {
				tset->ssid = ssid;
				ret = ss->can_attach(tset);
				if (ret) {
					failed_ssid = ssid;
					goto out_cancel_attach;
				}
2252
			}
2253 2254
		} while_each_subsys_mask();
	}
2255 2256 2257 2258 2259 2260

	/*
	 * Now that we're guaranteed success, proceed to move all tasks to
	 * the new cgroup.  There are no failure cases after here, so this
	 * is the commit point.
	 */
2261
	spin_lock_irq(&css_set_lock);
2262
	list_for_each_entry(cset, &tset->src_csets, mg_node) {
T
Tejun Heo 已提交
2263 2264 2265 2266 2267
		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {
			struct css_set *from_cset = task_css_set(task);
			struct css_set *to_cset = cset->mg_dst_cset;

			get_css_set(to_cset);
2268
			to_cset->nr_tasks++;
T
Tejun Heo 已提交
2269 2270
			css_set_move_task(task, from_cset, to_cset, true);
			put_css_set_locked(from_cset);
2271
			from_cset->nr_tasks--;
T
Tejun Heo 已提交
2272
		}
2273
	}
2274
	spin_unlock_irq(&css_set_lock);
2275 2276 2277 2278 2279 2280 2281 2282

	/*
	 * Migration is committed, all target tasks are now on dst_csets.
	 * Nothing is sensitive to fork() after this point.  Notify
	 * controllers that migration is complete.
	 */
	tset->csets = &tset->dst_csets;

2283 2284 2285 2286 2287 2288 2289 2290
	if (tset->nr_tasks) {
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ss->attach) {
				tset->ssid = ssid;
				ss->attach(tset);
			}
		} while_each_subsys_mask();
	}
2291 2292 2293 2294 2295

	ret = 0;
	goto out_release_tset;

out_cancel_attach:
2296 2297 2298 2299 2300 2301 2302 2303 2304 2305
	if (tset->nr_tasks) {
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ssid == failed_ssid)
				break;
			if (ss->cancel_attach) {
				tset->ssid = ssid;
				ss->cancel_attach(tset);
			}
		} while_each_subsys_mask();
	}
2306
out_release_tset:
2307
	spin_lock_irq(&css_set_lock);
2308 2309 2310 2311 2312
	list_splice_init(&tset->dst_csets, &tset->src_csets);
	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {
		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
		list_del_init(&cset->mg_node);
	}
2313
	spin_unlock_irq(&css_set_lock);
2314 2315 2316
	return ret;
}

2317
/**
2318
 * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2319 2320
 * @dst_cgrp: destination cgroup to test
 *
2321 2322 2323 2324
 * On the default hierarchy, except for the mixable, (possible) thread root
 * and threaded cgroups, subtree_control must be zero for migration
 * destination cgroups with tasks so that child cgroups don't compete
 * against tasks.
2325
 */
2326
int cgroup_migrate_vet_dst(struct cgroup *dst_cgrp)
2327
{
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351
	/* v1 doesn't have any restriction */
	if (!cgroup_on_dfl(dst_cgrp))
		return 0;

	/* verify @dst_cgrp can host resources */
	if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp))
		return -EOPNOTSUPP;

	/* mixables don't care */
	if (cgroup_is_mixable(dst_cgrp))
		return 0;

	/*
	 * If @dst_cgrp is already or can become a thread root or is
	 * threaded, it doesn't matter.
	 */
	if (cgroup_can_be_thread_root(dst_cgrp) || cgroup_is_threaded(dst_cgrp))
		return 0;

	/* apply no-internal-process constraint */
	if (dst_cgrp->subtree_control)
		return -EBUSY;

	return 0;
2352 2353
}

L
Li Zefan 已提交
2354
/**
2355
 * cgroup_migrate_finish - cleanup after attach
2356
 * @mgctx: migration context
B
Ben Blum 已提交
2357
 *
2358 2359
 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
 * those functions for details.
B
Ben Blum 已提交
2360
 */
2361
void cgroup_migrate_finish(struct cgroup_mgctx *mgctx)
B
Ben Blum 已提交
2362
{
2363
	LIST_HEAD(preloaded);
2364
	struct css_set *cset, *tmp_cset;
B
Ben Blum 已提交
2365

2366 2367
	lockdep_assert_held(&cgroup_mutex);

2368
	spin_lock_irq(&css_set_lock);
2369 2370 2371 2372 2373

	list_splice_tail_init(&mgctx->preloaded_src_csets, &preloaded);
	list_splice_tail_init(&mgctx->preloaded_dst_csets, &preloaded);

	list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) {
2374
		cset->mg_src_cgrp = NULL;
2375
		cset->mg_dst_cgrp = NULL;
2376 2377
		cset->mg_dst_cset = NULL;
		list_del_init(&cset->mg_preload_node);
Z
Zefan Li 已提交
2378
		put_css_set_locked(cset);
2379
	}
2380

2381
	spin_unlock_irq(&css_set_lock);
2382 2383 2384 2385 2386 2387
}

/**
 * cgroup_migrate_add_src - add a migration source css_set
 * @src_cset: the source css_set to add
 * @dst_cgrp: the destination cgroup
2388
 * @mgctx: migration context
2389 2390
 *
 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
2391
 * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2392 2393
 * up by cgroup_migrate_finish().
 *
2394 2395 2396 2397 2398
 * This function may be called without holding cgroup_threadgroup_rwsem
 * even if the target is a process.  Threads may be created and destroyed
 * but as long as cgroup_mutex is not dropped, no new css_set can be put
 * into play and the preloaded css_sets are guaranteed to cover all
 * migrations.
2399
 */
2400 2401
void cgroup_migrate_add_src(struct css_set *src_cset,
			    struct cgroup *dst_cgrp,
2402
			    struct cgroup_mgctx *mgctx)
2403 2404 2405 2406
{
	struct cgroup *src_cgrp;

	lockdep_assert_held(&cgroup_mutex);
2407
	lockdep_assert_held(&css_set_lock);
2408

2409 2410 2411 2412 2413 2414 2415 2416
	/*
	 * If ->dead, @src_set is associated with one or more dead cgroups
	 * and doesn't contain any migratable tasks.  Ignore it early so
	 * that the rest of migration path doesn't get confused by it.
	 */
	if (src_cset->dead)
		return;

2417 2418 2419 2420 2421 2422
	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);

	if (!list_empty(&src_cset->mg_preload_node))
		return;

	WARN_ON(src_cset->mg_src_cgrp);
2423
	WARN_ON(src_cset->mg_dst_cgrp);
2424 2425 2426 2427
	WARN_ON(!list_empty(&src_cset->mg_tasks));
	WARN_ON(!list_empty(&src_cset->mg_node));

	src_cset->mg_src_cgrp = src_cgrp;
2428
	src_cset->mg_dst_cgrp = dst_cgrp;
2429
	get_css_set(src_cset);
2430
	list_add_tail(&src_cset->mg_preload_node, &mgctx->preloaded_src_csets);
2431 2432 2433 2434
}

/**
 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2435
 * @mgctx: migration context
2436
 *
2437
 * Tasks are about to be moved and all the source css_sets have been
2438 2439 2440
 * preloaded to @mgctx->preloaded_src_csets.  This function looks up and
 * pins all destination css_sets, links each to its source, and append them
 * to @mgctx->preloaded_dst_csets.
2441 2442 2443 2444
 *
 * This function must be called after cgroup_migrate_add_src() has been
 * called on each migration source css_set.  After migration is performed
 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2445
 * @mgctx.
2446
 */
2447
int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx)
2448
{
2449
	struct css_set *src_cset, *tmp_cset;
2450 2451 2452 2453

	lockdep_assert_held(&cgroup_mutex);

	/* look up the dst cset for each src cset and link it to src */
2454 2455
	list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets,
				 mg_preload_node) {
2456
		struct css_set *dst_cset;
2457 2458
		struct cgroup_subsys *ss;
		int ssid;
2459

2460
		dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);
2461 2462 2463 2464
		if (!dst_cset)
			goto err;

		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2465 2466 2467 2468 2469 2470 2471 2472

		/*
		 * If src cset equals dst, it's noop.  Drop the src.
		 * cgroup_migrate() will skip the cset too.  Note that we
		 * can't handle src == dst as some nodes are used by both.
		 */
		if (src_cset == dst_cset) {
			src_cset->mg_src_cgrp = NULL;
2473
			src_cset->mg_dst_cgrp = NULL;
2474
			list_del_init(&src_cset->mg_preload_node);
Z
Zefan Li 已提交
2475 2476
			put_css_set(src_cset);
			put_css_set(dst_cset);
2477 2478 2479
			continue;
		}

2480 2481 2482
		src_cset->mg_dst_cset = dst_cset;

		if (list_empty(&dst_cset->mg_preload_node))
2483 2484
			list_add_tail(&dst_cset->mg_preload_node,
				      &mgctx->preloaded_dst_csets);
2485
		else
Z
Zefan Li 已提交
2486
			put_css_set(dst_cset);
2487 2488 2489 2490

		for_each_subsys(ss, ssid)
			if (src_cset->subsys[ssid] != dst_cset->subsys[ssid])
				mgctx->ss_mask |= 1 << ssid;
2491 2492 2493 2494
	}

	return 0;
err:
2495
	cgroup_migrate_finish(mgctx);
2496 2497 2498 2499 2500 2501 2502
	return -ENOMEM;
}

/**
 * cgroup_migrate - migrate a process or task to a cgroup
 * @leader: the leader of the process or the task to migrate
 * @threadgroup: whether @leader points to the whole process or a single task
2503
 * @mgctx: migration context
2504
 *
2505 2506 2507
 * Migrate a process or task denoted by @leader.  If migrating a process,
 * the caller must be holding cgroup_threadgroup_rwsem.  The caller is also
 * responsible for invoking cgroup_migrate_add_src() and
2508 2509 2510 2511 2512 2513 2514 2515 2516
 * cgroup_migrate_prepare_dst() on the targets before invoking this
 * function and following up with cgroup_migrate_finish().
 *
 * As long as a controller's ->can_attach() doesn't fail, this function is
 * guaranteed to succeed.  This means that, excluding ->can_attach()
 * failure, when migrating multiple targets, the success or failure can be
 * decided for all targets by invoking group_migrate_prepare_dst() before
 * actually starting migrating.
 */
2517
int cgroup_migrate(struct task_struct *leader, bool threadgroup,
2518
		   struct cgroup_mgctx *mgctx)
B
Ben Blum 已提交
2519
{
2520
	struct task_struct *task;
B
Ben Blum 已提交
2521

2522 2523 2524 2525 2526
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
2527
	spin_lock_irq(&css_set_lock);
2528
	rcu_read_lock();
2529
	task = leader;
B
Ben Blum 已提交
2530
	do {
2531
		cgroup_migrate_add_task(task, mgctx);
2532 2533
		if (!threadgroup)
			break;
2534
	} while_each_thread(leader, task);
2535
	rcu_read_unlock();
2536
	spin_unlock_irq(&css_set_lock);
B
Ben Blum 已提交
2537

2538
	return cgroup_migrate_execute(mgctx);
B
Ben Blum 已提交
2539 2540
}

2541 2542 2543 2544 2545 2546
/**
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
 * @dst_cgrp: the cgroup to attach to
 * @leader: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
 *
2547
 * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
2548
 */
2549 2550
int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,
		       bool threadgroup)
2551
{
2552
	DEFINE_CGROUP_MGCTX(mgctx);
2553 2554 2555
	struct task_struct *task;
	int ret;

2556 2557 2558
	ret = cgroup_migrate_vet_dst(dst_cgrp);
	if (ret)
		return ret;
2559

2560
	/* look up all src csets */
2561
	spin_lock_irq(&css_set_lock);
2562 2563 2564
	rcu_read_lock();
	task = leader;
	do {
2565
		cgroup_migrate_add_src(task_css_set(task), dst_cgrp, &mgctx);
2566 2567 2568 2569
		if (!threadgroup)
			break;
	} while_each_thread(leader, task);
	rcu_read_unlock();
2570
	spin_unlock_irq(&css_set_lock);
2571 2572

	/* prepare dst csets and commit */
2573
	ret = cgroup_migrate_prepare_dst(&mgctx);
2574
	if (!ret)
2575
		ret = cgroup_migrate(leader, threadgroup, &mgctx);
2576

2577
	cgroup_migrate_finish(&mgctx);
2578 2579 2580 2581

	if (!ret)
		trace_cgroup_attach_task(dst_cgrp, leader, threadgroup);

2582
	return ret;
B
Ben Blum 已提交
2583 2584
}

2585 2586
struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup)
	__acquires(&cgroup_threadgroup_rwsem)
2587 2588
{
	struct task_struct *tsk;
2589
	pid_t pid;
2590

2591
	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2592
		return ERR_PTR(-EINVAL);
B
Ben Blum 已提交
2593

T
Tejun Heo 已提交
2594
	percpu_down_write(&cgroup_threadgroup_rwsem);
2595

2596
	rcu_read_lock();
2597
	if (pid) {
2598
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2599
		if (!tsk) {
2600 2601
			tsk = ERR_PTR(-ESRCH);
			goto out_unlock_threadgroup;
2602
		}
2603
	} else {
2604
		tsk = current;
2605
	}
2606 2607

	if (threadgroup)
2608
		tsk = tsk->group_leader;
2609 2610

	/*
2611 2612 2613 2614
	 * kthreads may acquire PF_NO_SETAFFINITY during initialization.
	 * If userland migrates such a kthread to a non-root cgroup, it can
	 * become trapped in a cpuset, or RT kthread may be born in a
	 * cgroup with no rt_runtime allocated.  Just say no.
2615
	 */
2616
	if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) {
2617 2618
		tsk = ERR_PTR(-EINVAL);
		goto out_unlock_threadgroup;
2619 2620
	}

2621
	get_task_struct(tsk);
2622 2623 2624 2625 2626
	goto out_unlock_rcu;

out_unlock_threadgroup:
	percpu_up_write(&cgroup_threadgroup_rwsem);
out_unlock_rcu:
2627
	rcu_read_unlock();
2628 2629
	return tsk;
}
2630

2631 2632 2633 2634 2635
void cgroup_procs_write_finish(struct task_struct *task)
	__releases(&cgroup_threadgroup_rwsem)
{
	struct cgroup_subsys *ss;
	int ssid;
2636

2637 2638
	/* release reference from cgroup_procs_write_start() */
	put_task_struct(task);
T
Tejun Heo 已提交
2639 2640

	percpu_up_write(&cgroup_threadgroup_rwsem);
2641 2642 2643
	for_each_subsys(ss, ssid)
		if (ss->post_attach)
			ss->post_attach();
2644 2645
}

2646
static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask)
2647
{
2648 2649 2650
	struct cgroup_subsys *ss;
	bool printed = false;
	int ssid;
2651

2652
	do_each_subsys_mask(ss, ssid, ss_mask) {
2653 2654 2655 2656
		if (printed)
			seq_putc(seq, ' ');
		seq_printf(seq, "%s", ss->name);
		printed = true;
2657
	} while_each_subsys_mask();
2658 2659
	if (printed)
		seq_putc(seq, '\n');
2660 2661
}

2662 2663
/* show controllers which are enabled from the parent */
static int cgroup_controllers_show(struct seq_file *seq, void *v)
2664
{
2665 2666
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2667
	cgroup_print_ss_mask(seq, cgroup_control(cgrp));
2668
	return 0;
2669 2670
}

2671 2672
/* show controllers which are enabled for a given cgroup's children */
static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2673
{
2674 2675
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2676
	cgroup_print_ss_mask(seq, cgrp->subtree_control);
2677 2678 2679 2680 2681 2682 2683
	return 0;
}

/**
 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
 * @cgrp: root of the subtree to update csses for
 *
2684 2685 2686 2687
 * @cgrp's control masks have changed and its subtree's css associations
 * need to be updated accordingly.  This function looks up all css_sets
 * which are attached to the subtree, creates the matching updated css_sets
 * and migrates the tasks to the new ones.
2688 2689 2690
 */
static int cgroup_update_dfl_csses(struct cgroup *cgrp)
{
2691
	DEFINE_CGROUP_MGCTX(mgctx);
2692 2693
	struct cgroup_subsys_state *d_css;
	struct cgroup *dsct;
2694 2695 2696 2697 2698
	struct css_set *src_cset;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

T
Tejun Heo 已提交
2699 2700
	percpu_down_write(&cgroup_threadgroup_rwsem);

2701
	/* look up all csses currently attached to @cgrp's subtree */
2702
	spin_lock_irq(&css_set_lock);
2703
	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
2704 2705
		struct cgrp_cset_link *link;

2706
		list_for_each_entry(link, &dsct->cset_links, cset_link)
2707
			cgroup_migrate_add_src(link->cset, dsct, &mgctx);
2708
	}
2709
	spin_unlock_irq(&css_set_lock);
2710 2711

	/* NULL dst indicates self on default hierarchy */
2712
	ret = cgroup_migrate_prepare_dst(&mgctx);
2713 2714 2715
	if (ret)
		goto out_finish;

2716
	spin_lock_irq(&css_set_lock);
2717
	list_for_each_entry(src_cset, &mgctx.preloaded_src_csets, mg_preload_node) {
2718
		struct task_struct *task, *ntask;
2719

2720 2721
		/* all tasks in src_csets need to be migrated */
		list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)
2722
			cgroup_migrate_add_task(task, &mgctx);
2723
	}
2724
	spin_unlock_irq(&css_set_lock);
2725

2726
	ret = cgroup_migrate_execute(&mgctx);
2727
out_finish:
2728
	cgroup_migrate_finish(&mgctx);
T
Tejun Heo 已提交
2729
	percpu_up_write(&cgroup_threadgroup_rwsem);
2730 2731 2732
	return ret;
}

2733
/**
2734
 * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
2735
 * @cgrp: root of the target subtree
2736 2737
 *
 * Because css offlining is asynchronous, userland may try to re-enable a
2738 2739
 * controller while the previous css is still around.  This function grabs
 * cgroup_mutex and drains the previous css instances of @cgrp's subtree.
2740
 */
2741
void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
2742
	__acquires(&cgroup_mutex)
2743 2744
{
	struct cgroup *dsct;
2745
	struct cgroup_subsys_state *d_css;
2746 2747 2748
	struct cgroup_subsys *ss;
	int ssid;

2749 2750
restart:
	mutex_lock(&cgroup_mutex);
2751

2752
	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
2753 2754 2755 2756
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
			DEFINE_WAIT(wait);

2757
			if (!css || !percpu_ref_is_dying(&css->refcnt))
2758 2759
				continue;

2760
			cgroup_get_live(dsct);
2761 2762 2763 2764 2765 2766 2767 2768
			prepare_to_wait(&dsct->offline_waitq, &wait,
					TASK_UNINTERRUPTIBLE);

			mutex_unlock(&cgroup_mutex);
			schedule();
			finish_wait(&dsct->offline_waitq, &wait);

			cgroup_put(dsct);
2769
			goto restart;
2770 2771 2772 2773
		}
	}
}

2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806
/**
 * cgroup_save_control - save control masks of a subtree
 * @cgrp: root of the target subtree
 *
 * Save ->subtree_control and ->subtree_ss_mask to the respective old_
 * prefixed fields for @cgrp's subtree including @cgrp itself.
 */
static void cgroup_save_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
		dsct->old_subtree_control = dsct->subtree_control;
		dsct->old_subtree_ss_mask = dsct->subtree_ss_mask;
	}
}

/**
 * cgroup_propagate_control - refresh control masks of a subtree
 * @cgrp: root of the target subtree
 *
 * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
 * ->subtree_control and propagate controller availability through the
 * subtree so that descendants don't have unavailable controllers enabled.
 */
static void cgroup_propagate_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
		dsct->subtree_control &= cgroup_control(dsct);
2807 2808 2809
		dsct->subtree_ss_mask =
			cgroup_calc_subtree_ss_mask(dsct->subtree_control,
						    cgroup_ss_mask(dsct));
2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830
	}
}

/**
 * cgroup_restore_control - restore control masks of a subtree
 * @cgrp: root of the target subtree
 *
 * Restore ->subtree_control and ->subtree_ss_mask from the respective old_
 * prefixed fields for @cgrp's subtree including @cgrp itself.
 */
static void cgroup_restore_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
		dsct->subtree_control = dsct->old_subtree_control;
		dsct->subtree_ss_mask = dsct->old_subtree_ss_mask;
	}
}

2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842
static bool css_visible(struct cgroup_subsys_state *css)
{
	struct cgroup_subsys *ss = css->ss;
	struct cgroup *cgrp = css->cgroup;

	if (cgroup_control(cgrp) & (1 << ss->id))
		return true;
	if (!(cgroup_ss_mask(cgrp) & (1 << ss->id)))
		return false;
	return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl;
}

2843 2844
/**
 * cgroup_apply_control_enable - enable or show csses according to control
2845
 * @cgrp: root of the target subtree
2846
 *
2847
 * Walk @cgrp's subtree and create new csses or make the existing ones
2848 2849 2850 2851 2852 2853
 * visible.  A css is created invisible if it's being implicitly enabled
 * through dependency.  An invisible css is made visible when the userland
 * explicitly enables it.
 *
 * Returns 0 on success, -errno on failure.  On failure, csses which have
 * been processed already aren't cleaned up.  The caller is responsible for
2854
 * cleaning up with cgroup_apply_control_disable().
2855 2856 2857 2858
 */
static int cgroup_apply_control_enable(struct cgroup *cgrp)
{
	struct cgroup *dsct;
2859
	struct cgroup_subsys_state *d_css;
2860 2861 2862
	struct cgroup_subsys *ss;
	int ssid, ret;

2863
	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
2864 2865 2866
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);

2867 2868
			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));

2869 2870 2871 2872 2873 2874 2875 2876 2877
			if (!(cgroup_ss_mask(dsct) & (1 << ss->id)))
				continue;

			if (!css) {
				css = css_create(dsct, ss);
				if (IS_ERR(css))
					return PTR_ERR(css);
			}

2878
			if (css_visible(css)) {
2879
				ret = css_populate_dir(css);
2880 2881 2882 2883 2884 2885 2886 2887 2888
				if (ret)
					return ret;
			}
		}
	}

	return 0;
}

2889 2890
/**
 * cgroup_apply_control_disable - kill or hide csses according to control
2891
 * @cgrp: root of the target subtree
2892
 *
2893
 * Walk @cgrp's subtree and kill and hide csses so that they match
2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904
 * cgroup_ss_mask() and cgroup_visible_mask().
 *
 * A css is hidden when the userland requests it to be disabled while other
 * subsystems are still depending on it.  The css must not actively control
 * resources and be in the vanilla state if it's made visible again later.
 * Controllers which may be depended upon should provide ->css_reset() for
 * this purpose.
 */
static void cgroup_apply_control_disable(struct cgroup *cgrp)
{
	struct cgroup *dsct;
2905
	struct cgroup_subsys_state *d_css;
2906 2907 2908
	struct cgroup_subsys *ss;
	int ssid;

2909
	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
2910 2911 2912
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);

2913 2914
			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));

2915 2916 2917
			if (!css)
				continue;

2918 2919
			if (css->parent &&
			    !(cgroup_ss_mask(dsct) & (1 << ss->id))) {
2920
				kill_css(css);
2921
			} else if (!css_visible(css)) {
2922
				css_clear_dir(css);
2923 2924 2925 2926 2927 2928 2929
				if (ss->css_reset)
					ss->css_reset(css);
			}
		}
	}
}

2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985
/**
 * cgroup_apply_control - apply control mask updates to the subtree
 * @cgrp: root of the target subtree
 *
 * subsystems can be enabled and disabled in a subtree using the following
 * steps.
 *
 * 1. Call cgroup_save_control() to stash the current state.
 * 2. Update ->subtree_control masks in the subtree as desired.
 * 3. Call cgroup_apply_control() to apply the changes.
 * 4. Optionally perform other related operations.
 * 5. Call cgroup_finalize_control() to finish up.
 *
 * This function implements step 3 and propagates the mask changes
 * throughout @cgrp's subtree, updates csses accordingly and perform
 * process migrations.
 */
static int cgroup_apply_control(struct cgroup *cgrp)
{
	int ret;

	cgroup_propagate_control(cgrp);

	ret = cgroup_apply_control_enable(cgrp);
	if (ret)
		return ret;

	/*
	 * At this point, cgroup_e_css() results reflect the new csses
	 * making the following cgroup_update_dfl_csses() properly update
	 * css associations of all tasks in the subtree.
	 */
	ret = cgroup_update_dfl_csses(cgrp);
	if (ret)
		return ret;

	return 0;
}

/**
 * cgroup_finalize_control - finalize control mask update
 * @cgrp: root of the target subtree
 * @ret: the result of the update
 *
 * Finalize control mask update.  See cgroup_apply_control() for more info.
 */
static void cgroup_finalize_control(struct cgroup *cgrp, int ret)
{
	if (ret) {
		cgroup_restore_control(cgrp);
		cgroup_propagate_control(cgrp);
	}

	cgroup_apply_control_disable(cgrp);
}

2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025
static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable)
{
	u16 domain_enable = enable & ~cgrp_dfl_threaded_ss_mask;

	/* if nothing is getting enabled, nothing to worry about */
	if (!enable)
		return 0;

	/* can @cgrp host any resources? */
	if (!cgroup_is_valid_domain(cgrp->dom_cgrp))
		return -EOPNOTSUPP;

	/* mixables don't care */
	if (cgroup_is_mixable(cgrp))
		return 0;

	if (domain_enable) {
		/* can't enable domain controllers inside a thread subtree */
		if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp))
			return -EOPNOTSUPP;
	} else {
		/*
		 * Threaded controllers can handle internal competitions
		 * and are always allowed inside a (prospective) thread
		 * subtree.
		 */
		if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp))
			return 0;
	}

	/*
	 * Controllers can't be enabled for a cgroup with tasks to avoid
	 * child cgroups competing against tasks.
	 */
	if (cgroup_has_tasks(cgrp))
		return -EBUSY;

	return 0;
}

3026
/* change the enabled child controllers for a cgroup in the default hierarchy */
3027 3028 3029
static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
3030
{
3031
	u16 enable = 0, disable = 0;
3032
	struct cgroup *cgrp, *child;
3033
	struct cgroup_subsys *ss;
3034
	char *tok;
3035 3036 3037
	int ssid, ret;

	/*
3038 3039
	 * Parse input - space separated list of subsystem names prefixed
	 * with either + or -.
3040
	 */
3041 3042
	buf = strstrip(buf);
	while ((tok = strsep(&buf, " "))) {
3043 3044
		if (tok[0] == '\0')
			continue;
T
Tejun Heo 已提交
3045
		do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) {
3046 3047
			if (!cgroup_ssid_enabled(ssid) ||
			    strcmp(tok + 1, ss->name))
3048 3049 3050
				continue;

			if (*tok == '+') {
3051 3052
				enable |= 1 << ssid;
				disable &= ~(1 << ssid);
3053
			} else if (*tok == '-') {
3054 3055
				disable |= 1 << ssid;
				enable &= ~(1 << ssid);
3056 3057 3058 3059
			} else {
				return -EINVAL;
			}
			break;
3060
		} while_each_subsys_mask();
3061 3062 3063 3064
		if (ssid == CGROUP_SUBSYS_COUNT)
			return -EINVAL;
	}

3065
	cgrp = cgroup_kn_lock_live(of->kn, true);
3066 3067
	if (!cgrp)
		return -ENODEV;
3068 3069 3070

	for_each_subsys(ss, ssid) {
		if (enable & (1 << ssid)) {
3071
			if (cgrp->subtree_control & (1 << ssid)) {
3072 3073 3074 3075
				enable &= ~(1 << ssid);
				continue;
			}

3076
			if (!(cgroup_control(cgrp) & (1 << ssid))) {
3077 3078 3079
				ret = -ENOENT;
				goto out_unlock;
			}
3080
		} else if (disable & (1 << ssid)) {
3081
			if (!(cgrp->subtree_control & (1 << ssid))) {
3082 3083 3084 3085 3086 3087
				disable &= ~(1 << ssid);
				continue;
			}

			/* a child has it enabled? */
			cgroup_for_each_live_child(child, cgrp) {
3088
				if (child->subtree_control & (1 << ssid)) {
3089
					ret = -EBUSY;
3090
					goto out_unlock;
3091 3092 3093 3094 3095 3096 3097
				}
			}
		}
	}

	if (!enable && !disable) {
		ret = 0;
3098
		goto out_unlock;
3099 3100
	}

3101 3102
	ret = cgroup_vet_subtree_control_enable(cgrp, enable);
	if (ret)
3103
		goto out_unlock;
3104

3105 3106
	/* save and update control masks and prepare csses */
	cgroup_save_control(cgrp);
3107

3108 3109
	cgrp->subtree_control |= enable;
	cgrp->subtree_control &= ~disable;
3110

3111 3112
	ret = cgroup_apply_control(cgrp);
	cgroup_finalize_control(cgrp, ret);
3113 3114
	if (ret)
		goto out_unlock;
3115 3116 3117

	kernfs_activate(cgrp->kn);
out_unlock:
3118
	cgroup_kn_unlock(of->kn);
3119
	return ret ?: nbytes;
3120 3121
}

3122 3123 3124 3125 3126 3127 3128 3129 3130
/**
 * cgroup_enable_threaded - make @cgrp threaded
 * @cgrp: the target cgroup
 *
 * Called when "threaded" is written to the cgroup.type interface file and
 * tries to make @cgrp threaded and join the parent's resource domain.
 * This function is never called on the root cgroup as cgroup.type doesn't
 * exist on it.
 */
3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201
static int cgroup_enable_threaded(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);
	struct cgroup *dom_cgrp = parent->dom_cgrp;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* noop if already threaded */
	if (cgroup_is_threaded(cgrp))
		return 0;

	/* we're joining the parent's domain, ensure its validity */
	if (!cgroup_is_valid_domain(dom_cgrp) ||
	    !cgroup_can_be_thread_root(dom_cgrp))
		return -EOPNOTSUPP;

	/*
	 * The following shouldn't cause actual migrations and should
	 * always succeed.
	 */
	cgroup_save_control(cgrp);

	cgrp->dom_cgrp = dom_cgrp;
	ret = cgroup_apply_control(cgrp);
	if (!ret)
		parent->nr_threaded_children++;
	else
		cgrp->dom_cgrp = cgrp;

	cgroup_finalize_control(cgrp, ret);
	return ret;
}

static int cgroup_type_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;

	if (cgroup_is_threaded(cgrp))
		seq_puts(seq, "threaded\n");
	else if (!cgroup_is_valid_domain(cgrp))
		seq_puts(seq, "domain invalid\n");
	else if (cgroup_is_thread_root(cgrp))
		seq_puts(seq, "domain threaded\n");
	else
		seq_puts(seq, "domain\n");

	return 0;
}

static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	int ret;

	/* only switching to threaded mode is supported */
	if (strcmp(strstrip(buf), "threaded"))
		return -EINVAL;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	/* threaded can only be enabled */
	ret = cgroup_enable_threaded(cgrp);

	cgroup_kn_unlock(of->kn);
	return ret ?: nbytes;
}

3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
static int cgroup_max_descendants_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;
	int descendants = READ_ONCE(cgrp->max_descendants);

	if (descendants == INT_MAX)
		seq_puts(seq, "max\n");
	else
		seq_printf(seq, "%d\n", descendants);

	return 0;
}

static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of,
					   char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	int descendants;
	ssize_t ret;

	buf = strstrip(buf);
	if (!strcmp(buf, "max")) {
		descendants = INT_MAX;
	} else {
		ret = kstrtoint(buf, 0, &descendants);
		if (ret)
			return ret;
	}

D
Dan Carpenter 已提交
3231
	if (descendants < 0)
3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273
		return -ERANGE;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	cgrp->max_descendants = descendants;

	cgroup_kn_unlock(of->kn);

	return nbytes;
}

static int cgroup_max_depth_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;
	int depth = READ_ONCE(cgrp->max_depth);

	if (depth == INT_MAX)
		seq_puts(seq, "max\n");
	else
		seq_printf(seq, "%d\n", depth);

	return 0;
}

static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of,
				      char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	ssize_t ret;
	int depth;

	buf = strstrip(buf);
	if (!strcmp(buf, "max")) {
		depth = INT_MAX;
	} else {
		ret = kstrtoint(buf, 0, &depth);
		if (ret)
			return ret;
	}

D
Dan Carpenter 已提交
3274
	if (depth < 0)
3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287
		return -ERANGE;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	cgrp->max_depth = depth;

	cgroup_kn_unlock(of->kn);

	return nbytes;
}

3288
static int cgroup_events_show(struct seq_file *seq, void *v)
3289
{
3290
	seq_printf(seq, "populated %d\n",
3291
		   cgroup_is_populated(seq_css(seq)->cgroup));
3292 3293 3294
	return 0;
}

T
Tejun Heo 已提交
3295
static int cgroup_stat_show(struct seq_file *seq, void *v)
3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
{
	struct cgroup *cgroup = seq_css(seq)->cgroup;

	seq_printf(seq, "nr_descendants %d\n",
		   cgroup->nr_descendants);
	seq_printf(seq, "nr_dying_descendants %d\n",
		   cgroup->nr_dying_descendants);

	return 0;
}

3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323
static int cgroup_file_open(struct kernfs_open_file *of)
{
	struct cftype *cft = of->kn->priv;

	if (cft->open)
		return cft->open(of);
	return 0;
}

static void cgroup_file_release(struct kernfs_open_file *of)
{
	struct cftype *cft = of->kn->priv;

	if (cft->release)
		cft->release(of);
}

T
Tejun Heo 已提交
3324 3325
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
3326
{
3327
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
T
Tejun Heo 已提交
3328 3329 3330
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = of->kn->priv;
	struct cgroup_subsys_state *css;
3331
	int ret;
3332

3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343
	/*
	 * If namespaces are delegation boundaries, disallow writes to
	 * files in an non-init namespace root from inside the namespace
	 * except for the files explicitly marked delegatable -
	 * cgroup.procs and cgroup.subtree_control.
	 */
	if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) &&
	    !(cft->flags & CFTYPE_NS_DELEGATABLE) &&
	    ns != &init_cgroup_ns && ns->root_cset->dfl_cgrp == cgrp)
		return -EPERM;

T
Tejun Heo 已提交
3344 3345 3346
	if (cft->write)
		return cft->write(of, buf, nbytes, off);

T
Tejun Heo 已提交
3347 3348 3349 3350 3351 3352 3353 3354 3355
	/*
	 * kernfs guarantees that a file isn't deleted with operations in
	 * flight, which means that the matching css is and stays alive and
	 * doesn't need to be pinned.  The RCU locking is not necessary
	 * either.  It's just for the convenience of using cgroup_css().
	 */
	rcu_read_lock();
	css = cgroup_css(cgrp, cft->ss);
	rcu_read_unlock();
3356

3357
	if (cft->write_u64) {
3358 3359 3360 3361 3362 3363 3364 3365 3366
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
3367
	} else {
3368
		ret = -EINVAL;
3369
	}
T
Tejun Heo 已提交
3370

3371
	return ret ?: nbytes;
3372 3373
}

3374
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
3375
{
T
Tejun Heo 已提交
3376
	return seq_cft(seq)->seq_start(seq, ppos);
3377 3378
}

3379
static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
3380
{
T
Tejun Heo 已提交
3381
	return seq_cft(seq)->seq_next(seq, v, ppos);
3382 3383
}

3384
static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
3385
{
3386 3387
	if (seq_cft(seq)->seq_stop)
		seq_cft(seq)->seq_stop(seq, v);
3388 3389
}

3390
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
3391
{
3392 3393
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
3394

3395 3396
	if (cft->seq_show)
		return cft->seq_show(m, arg);
3397

3398
	if (cft->read_u64)
3399 3400 3401 3402 3403 3404
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
3405 3406
}

T
Tejun Heo 已提交
3407 3408
static struct kernfs_ops cgroup_kf_single_ops = {
	.atomic_write_len	= PAGE_SIZE,
3409 3410
	.open			= cgroup_file_open,
	.release		= cgroup_file_release,
T
Tejun Heo 已提交
3411 3412
	.write			= cgroup_file_write,
	.seq_show		= cgroup_seqfile_show,
3413 3414
};

T
Tejun Heo 已提交
3415 3416
static struct kernfs_ops cgroup_kf_ops = {
	.atomic_write_len	= PAGE_SIZE,
3417 3418
	.open			= cgroup_file_open,
	.release		= cgroup_file_release,
T
Tejun Heo 已提交
3419 3420 3421 3422 3423 3424
	.write			= cgroup_file_write,
	.seq_start		= cgroup_seqfile_start,
	.seq_next		= cgroup_seqfile_next,
	.seq_stop		= cgroup_seqfile_stop,
	.seq_show		= cgroup_seqfile_show,
};
3425

3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439
/* set uid and gid of cgroup dirs and files to that of the creator */
static int cgroup_kn_set_ugid(struct kernfs_node *kn)
{
	struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
			       .ia_uid = current_fsuid(),
			       .ia_gid = current_fsgid(), };

	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
	    gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
		return 0;

	return kernfs_setattr(kn, &iattr);
}

3440 3441
static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
			   struct cftype *cft)
3442
{
T
Tejun Heo 已提交
3443
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
3444 3445
	struct kernfs_node *kn;
	struct lock_class_key *key = NULL;
3446
	int ret;
T
Tejun Heo 已提交
3447

T
Tejun Heo 已提交
3448 3449 3450 3451 3452
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	key = &cft->lockdep_key;
#endif
	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
				  cgroup_file_mode(cft), 0, cft->kf_ops, cft,
T
Tejun Heo 已提交
3453
				  NULL, key);
3454 3455 3456 3457
	if (IS_ERR(kn))
		return PTR_ERR(kn);

	ret = cgroup_kn_set_ugid(kn);
3458
	if (ret) {
3459
		kernfs_remove(kn);
3460 3461 3462
		return ret;
	}

3463 3464 3465
	if (cft->file_offset) {
		struct cgroup_file *cfile = (void *)css + cft->file_offset;

3466
		spin_lock_irq(&cgroup_file_kn_lock);
3467
		cfile->kn = kn;
3468
		spin_unlock_irq(&cgroup_file_kn_lock);
3469 3470
	}

3471
	return 0;
3472 3473
}

3474 3475
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
3476 3477
 * @css: the target css
 * @cgrp: the target cgroup (usually css->cgroup)
3478 3479 3480 3481
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3482
 * For removals, this function never fails.
3483
 */
3484 3485
static int cgroup_addrm_files(struct cgroup_subsys_state *css,
			      struct cgroup *cgrp, struct cftype cfts[],
3486
			      bool is_add)
3487
{
3488
	struct cftype *cft, *cft_end = NULL;
3489
	int ret = 0;
3490

3491
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
3492

3493 3494
restart:
	for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {
3495
		/* does cft->flags tell us to skip this file on @cgrp? */
3496
		if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
T
Tejun Heo 已提交
3497
			continue;
3498
		if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
3499
			continue;
T
Tejun Heo 已提交
3500
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3501
			continue;
T
Tejun Heo 已提交
3502
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3503 3504
			continue;

3505
		if (is_add) {
3506
			ret = cgroup_add_file(css, cgrp, cft);
3507
			if (ret) {
3508 3509
				pr_warn("%s: failed to add %s, err=%d\n",
					__func__, cft->name, ret);
3510 3511 3512
				cft_end = cft;
				is_add = false;
				goto restart;
3513
			}
3514 3515
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
3516
		}
3517
	}
3518
	return ret;
3519 3520
}

3521
static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3522
{
3523
	struct cgroup_subsys *ss = cfts[0].ss;
3524
	struct cgroup *root = &ss->root->cgrp;
3525
	struct cgroup_subsys_state *css;
3526
	int ret = 0;
3527

3528
	lockdep_assert_held(&cgroup_mutex);
3529 3530

	/* add/rm files for all cgroups created before */
3531
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3532 3533
		struct cgroup *cgrp = css->cgroup;

3534
		if (!(css->flags & CSS_VISIBLE))
3535 3536
			continue;

3537
		ret = cgroup_addrm_files(css, cgrp, cfts, is_add);
3538 3539
		if (ret)
			break;
3540
	}
3541 3542 3543

	if (is_add && !ret)
		kernfs_activate(root->kn);
3544
	return ret;
3545 3546
}

3547
static void cgroup_exit_cftypes(struct cftype *cfts)
3548
{
3549
	struct cftype *cft;
3550

T
Tejun Heo 已提交
3551 3552 3553 3554 3555
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		/* free copy for custom atomic_write_len, see init_cftypes() */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
			kfree(cft->kf_ops);
		cft->kf_ops = NULL;
3556
		cft->ss = NULL;
3557 3558

		/* revert flags set by cgroup core while adding @cfts */
3559
		cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
T
Tejun Heo 已提交
3560
	}
3561 3562
}

T
Tejun Heo 已提交
3563
static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3564 3565 3566
{
	struct cftype *cft;

T
Tejun Heo 已提交
3567 3568 3569
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		struct kernfs_ops *kf_ops;

T
Tejun Heo 已提交
3570 3571
		WARN_ON(cft->ss || cft->kf_ops);

T
Tejun Heo 已提交
3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588
		if (cft->seq_start)
			kf_ops = &cgroup_kf_ops;
		else
			kf_ops = &cgroup_kf_single_ops;

		/*
		 * Ugh... if @cft wants a custom max_write_len, we need to
		 * make a copy of kf_ops to set its atomic_write_len.
		 */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
			kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
			if (!kf_ops) {
				cgroup_exit_cftypes(cfts);
				return -ENOMEM;
			}
			kf_ops->atomic_write_len = cft->max_write_len;
		}
3589

T
Tejun Heo 已提交
3590
		cft->kf_ops = kf_ops;
3591
		cft->ss = ss;
T
Tejun Heo 已提交
3592
	}
3593

T
Tejun Heo 已提交
3594
	return 0;
3595 3596
}

3597 3598
static int cgroup_rm_cftypes_locked(struct cftype *cfts)
{
3599
	lockdep_assert_held(&cgroup_mutex);
3600 3601 3602 3603 3604 3605 3606 3607

	if (!cfts || !cfts[0].ss)
		return -ENOENT;

	list_del(&cfts->node);
	cgroup_apply_cftypes(cfts, false);
	cgroup_exit_cftypes(cfts);
	return 0;
3608 3609
}

3610 3611 3612 3613
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
3614 3615 3616
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
3617 3618
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3619
 * registered.
3620
 */
3621
int cgroup_rm_cftypes(struct cftype *cfts)
3622
{
3623
	int ret;
3624

3625
	mutex_lock(&cgroup_mutex);
3626
	ret = cgroup_rm_cftypes_locked(cfts);
3627
	mutex_unlock(&cgroup_mutex);
3628
	return ret;
T
Tejun Heo 已提交
3629 3630
}

3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644
/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
3645
static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3646
{
3647
	int ret;
3648

3649
	if (!cgroup_ssid_enabled(ss->id))
3650 3651
		return 0;

3652 3653
	if (!cfts || cfts[0].name[0] == '\0')
		return 0;
3654

T
Tejun Heo 已提交
3655 3656 3657
	ret = cgroup_init_cftypes(ss, cfts);
	if (ret)
		return ret;
3658

3659
	mutex_lock(&cgroup_mutex);
3660

T
Tejun Heo 已提交
3661
	list_add_tail(&cfts->node, &ss->cfts);
3662
	ret = cgroup_apply_cftypes(cfts, true);
3663
	if (ret)
3664
		cgroup_rm_cftypes_locked(cfts);
3665

3666
	mutex_unlock(&cgroup_mutex);
3667
	return ret;
3668 3669
}

3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682
/**
 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the default hierarchy.
 */
int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
	struct cftype *cft;

	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3683
		cft->flags |= __CFTYPE_ONLY_ON_DFL;
3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694
	return cgroup_add_cftypes(ss, cfts);
}

/**
 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the legacy hierarchies.
 */
3695 3696
int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
3697 3698
	struct cftype *cft;

3699 3700
	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
		cft->flags |= __CFTYPE_NOT_ON_DFL;
3701 3702 3703
	return cgroup_add_cftypes(ss, cfts);
}

3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719
/**
 * cgroup_file_notify - generate a file modified event for a cgroup_file
 * @cfile: target cgroup_file
 *
 * @cfile must have been obtained by setting cftype->file_offset.
 */
void cgroup_file_notify(struct cgroup_file *cfile)
{
	unsigned long flags;

	spin_lock_irqsave(&cgroup_file_kn_lock, flags);
	if (cfile->kn)
		kernfs_notify(cfile->kn);
	spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
}

3720
/**
3721
 * css_next_child - find the next child of a given css
3722 3723
 * @pos: the current position (%NULL to initiate traversal)
 * @parent: css whose children to walk
3724
 *
3725
 * This function returns the next child of @parent and should be called
3726
 * under either cgroup_mutex or RCU read lock.  The only requirement is
3727 3728 3729 3730 3731 3732 3733 3734 3735
 * that @parent and @pos are accessible.  The next sibling is guaranteed to
 * be returned regardless of their states.
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3736
 */
3737 3738
struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
					   struct cgroup_subsys_state *parent)
3739
{
3740
	struct cgroup_subsys_state *next;
3741

T
Tejun Heo 已提交
3742
	cgroup_assert_mutex_or_rcu_locked();
3743 3744

	/*
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754
	 * @pos could already have been unlinked from the sibling list.
	 * Once a cgroup is removed, its ->sibling.next is no longer
	 * updated when its next sibling changes.  CSS_RELEASED is set when
	 * @pos is taken off list, at which time its next pointer is valid,
	 * and, as releases are serialized, the one pointed to by the next
	 * pointer is guaranteed to not have started release yet.  This
	 * implies that if we observe !CSS_RELEASED on @pos in this RCU
	 * critical section, the one pointed to by its next pointer is
	 * guaranteed to not have finished its RCU grace period even if we
	 * have dropped rcu_read_lock() inbetween iterations.
3755
	 *
3756 3757 3758 3759 3760 3761 3762
	 * If @pos has CSS_RELEASED set, its next pointer can't be
	 * dereferenced; however, as each css is given a monotonically
	 * increasing unique serial number and always appended to the
	 * sibling list, the next one can be found by walking the parent's
	 * children until the first css with higher serial number than
	 * @pos's.  While this path can be slower, it happens iff iteration
	 * races against release and the race window is very small.
3763
	 */
3764
	if (!pos) {
3765 3766 3767
		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
	} else if (likely(!(pos->flags & CSS_RELEASED))) {
		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3768
	} else {
3769
		list_for_each_entry_rcu(next, &parent->children, sibling)
3770 3771
			if (next->serial_nr > pos->serial_nr)
				break;
3772 3773
	}

3774 3775
	/*
	 * @next, if not pointing to the head, can be dereferenced and is
3776
	 * the next sibling.
3777
	 */
3778 3779
	if (&next->sibling != &parent->children)
		return next;
3780
	return NULL;
3781 3782
}

3783
/**
3784
 * css_next_descendant_pre - find the next descendant for pre-order walk
3785
 * @pos: the current position (%NULL to initiate traversal)
3786
 * @root: css whose descendants to walk
3787
 *
3788
 * To be used by css_for_each_descendant_pre().  Find the next descendant
3789 3790
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
3791
 *
3792 3793 3794 3795
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
3796 3797 3798 3799 3800 3801 3802
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3803
 */
3804 3805 3806
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
3807
{
3808
	struct cgroup_subsys_state *next;
3809

T
Tejun Heo 已提交
3810
	cgroup_assert_mutex_or_rcu_locked();
3811

3812
	/* if first iteration, visit @root */
3813
	if (!pos)
3814
		return root;
3815 3816

	/* visit the first child if exists */
3817
	next = css_next_child(NULL, pos);
3818 3819 3820 3821
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
3822
	while (pos != root) {
T
Tejun Heo 已提交
3823
		next = css_next_child(pos, pos->parent);
3824
		if (next)
3825
			return next;
T
Tejun Heo 已提交
3826
		pos = pos->parent;
3827
	}
3828 3829 3830 3831

	return NULL;
}

3832
/**
3833 3834
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
3835
 *
3836 3837
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
3838
 * subtree of @pos.
3839
 *
3840 3841 3842 3843
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
3844
 */
3845 3846
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
3847
{
3848
	struct cgroup_subsys_state *last, *tmp;
3849

T
Tejun Heo 已提交
3850
	cgroup_assert_mutex_or_rcu_locked();
3851 3852 3853 3854 3855

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
3856
		css_for_each_child(tmp, last)
3857 3858 3859 3860 3861 3862
			pos = tmp;
	} while (pos);

	return last;
}

3863 3864
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
3865
{
3866
	struct cgroup_subsys_state *last;
3867 3868 3869

	do {
		last = pos;
3870
		pos = css_next_child(NULL, pos);
3871 3872 3873 3874 3875 3876
	} while (pos);

	return last;
}

/**
3877
 * css_next_descendant_post - find the next descendant for post-order walk
3878
 * @pos: the current position (%NULL to initiate traversal)
3879
 * @root: css whose descendants to walk
3880
 *
3881
 * To be used by css_for_each_descendant_post().  Find the next descendant
3882 3883
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
3884
 *
3885 3886 3887 3888 3889
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
3890 3891 3892 3893 3894 3895 3896
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3897
 */
3898 3899 3900
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
3901
{
3902
	struct cgroup_subsys_state *next;
3903

T
Tejun Heo 已提交
3904
	cgroup_assert_mutex_or_rcu_locked();
3905

3906 3907 3908
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
3909

3910 3911 3912 3913
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

3914
	/* if there's an unvisited sibling, visit its leftmost descendant */
T
Tejun Heo 已提交
3915
	next = css_next_child(pos, pos->parent);
3916
	if (next)
3917
		return css_leftmost_descendant(next);
3918 3919

	/* no sibling left, visit parent */
T
Tejun Heo 已提交
3920
	return pos->parent;
3921 3922
}

3923 3924 3925 3926 3927 3928 3929 3930 3931
/**
 * css_has_online_children - does a css have online children
 * @css: the target css
 *
 * Returns %true if @css has any online children; otherwise, %false.  This
 * function can be called from any context but the caller is responsible
 * for synchronizing against on/offlining as necessary.
 */
bool css_has_online_children(struct cgroup_subsys_state *css)
3932
{
3933 3934
	struct cgroup_subsys_state *child;
	bool ret = false;
3935 3936

	rcu_read_lock();
3937
	css_for_each_child(child, css) {
3938
		if (child->flags & CSS_ONLINE) {
3939 3940
			ret = true;
			break;
3941 3942 3943
		}
	}
	rcu_read_unlock();
3944
	return ret;
3945 3946
}

3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998
static struct css_set *css_task_iter_next_css_set(struct css_task_iter *it)
{
	struct list_head *l;
	struct cgrp_cset_link *link;
	struct css_set *cset;

	lockdep_assert_held(&css_set_lock);

	/* find the next threaded cset */
	if (it->tcset_pos) {
		l = it->tcset_pos->next;

		if (l != it->tcset_head) {
			it->tcset_pos = l;
			return container_of(l, struct css_set,
					    threaded_csets_node);
		}

		it->tcset_pos = NULL;
	}

	/* find the next cset */
	l = it->cset_pos;
	l = l->next;
	if (l == it->cset_head) {
		it->cset_pos = NULL;
		return NULL;
	}

	if (it->ss) {
		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);
	} else {
		link = list_entry(l, struct cgrp_cset_link, cset_link);
		cset = link->cset;
	}

	it->cset_pos = l;

	/* initialize threaded css_set walking */
	if (it->flags & CSS_TASK_ITER_THREADED) {
		if (it->cur_dcset)
			put_css_set_locked(it->cur_dcset);
		it->cur_dcset = cset;
		get_css_set(cset);

		it->tcset_head = &cset->threaded_csets;
		it->tcset_pos = &cset->threaded_csets;
	}

	return cset;
}

3999
/**
4000
 * css_task_iter_advance_css_set - advance a task itererator to the next css_set
4001 4002 4003
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
4004
 */
4005
static void css_task_iter_advance_css_set(struct css_task_iter *it)
4006 4007 4008
{
	struct css_set *cset;

4009
	lockdep_assert_held(&css_set_lock);
4010

4011 4012
	/* Advance to the next non-empty css_set */
	do {
4013 4014
		cset = css_task_iter_next_css_set(it);
		if (!cset) {
4015
			it->task_pos = NULL;
4016 4017
			return;
		}
4018
	} while (!css_set_populated(cset));
T
Tejun Heo 已提交
4019 4020

	if (!list_empty(&cset->tasks))
T
Tejun Heo 已提交
4021
		it->task_pos = cset->tasks.next;
T
Tejun Heo 已提交
4022
	else
T
Tejun Heo 已提交
4023 4024 4025 4026
		it->task_pos = cset->mg_tasks.next;

	it->tasks_head = &cset->tasks;
	it->mg_tasks_head = &cset->mg_tasks;
4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049

	/*
	 * We don't keep css_sets locked across iteration steps and thus
	 * need to take steps to ensure that iteration can be resumed after
	 * the lock is re-acquired.  Iteration is performed at two levels -
	 * css_sets and tasks in them.
	 *
	 * Once created, a css_set never leaves its cgroup lists, so a
	 * pinned css_set is guaranteed to stay put and we can resume
	 * iteration afterwards.
	 *
	 * Tasks may leave @cset across iteration steps.  This is resolved
	 * by registering each iterator with the css_set currently being
	 * walked and making css_set_move_task() advance iterators whose
	 * next task is leaving.
	 */
	if (it->cur_cset) {
		list_del(&it->iters_node);
		put_css_set_locked(it->cur_cset);
	}
	get_css_set(cset);
	it->cur_cset = cset;
	list_add(&it->iters_node, &cset->task_iters);
4050 4051
}

4052 4053 4054 4055
static void css_task_iter_advance(struct css_task_iter *it)
{
	struct list_head *l = it->task_pos;

4056
	lockdep_assert_held(&css_set_lock);
4057 4058
	WARN_ON_ONCE(!l);

4059
repeat:
4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073
	/*
	 * Advance iterator to find next entry.  cset->tasks is consumed
	 * first and then ->mg_tasks.  After ->mg_tasks, we move onto the
	 * next cset.
	 */
	l = l->next;

	if (l == it->tasks_head)
		l = it->mg_tasks_head->next;

	if (l == it->mg_tasks_head)
		css_task_iter_advance_css_set(it);
	else
		it->task_pos = l;
4074 4075 4076 4077 4078 4079

	/* if PROCS, skip over tasks which aren't group leaders */
	if ((it->flags & CSS_TASK_ITER_PROCS) && it->task_pos &&
	    !thread_group_leader(list_entry(it->task_pos, struct task_struct,
					    cg_list)))
		goto repeat;
4080 4081
}

4082
/**
4083 4084
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
4085
 * @flags: CSS_TASK_ITER_* flags
4086 4087
 * @it: the task iterator to use
 *
4088 4089 4090 4091
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
4092
 */
4093
void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
4094
			 struct css_task_iter *it)
4095
{
4096 4097
	/* no one should try to iterate before mounting cgroups */
	WARN_ON_ONCE(!use_task_css_set_links);
4098

4099 4100
	memset(it, 0, sizeof(*it));

4101
	spin_lock_irq(&css_set_lock);
4102

4103
	it->ss = css->ss;
4104
	it->flags = flags;
4105 4106 4107 4108 4109 4110

	if (it->ss)
		it->cset_pos = &css->cgroup->e_csets[css->ss->id];
	else
		it->cset_pos = &css->cgroup->cset_links;

T
Tejun Heo 已提交
4111
	it->cset_head = it->cset_pos;
4112

4113
	css_task_iter_advance_css_set(it);
4114

4115
	spin_unlock_irq(&css_set_lock);
4116 4117
}

4118
/**
4119
 * css_task_iter_next - return the next task for the iterator
4120 4121 4122
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
4123 4124
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
4125
 */
4126
struct task_struct *css_task_iter_next(struct css_task_iter *it)
4127
{
4128
	if (it->cur_task) {
4129
		put_task_struct(it->cur_task);
4130 4131
		it->cur_task = NULL;
	}
4132

4133
	spin_lock_irq(&css_set_lock);
4134

4135 4136 4137 4138 4139 4140
	if (it->task_pos) {
		it->cur_task = list_entry(it->task_pos, struct task_struct,
					  cg_list);
		get_task_struct(it->cur_task);
		css_task_iter_advance(it);
	}
4141

4142
	spin_unlock_irq(&css_set_lock);
4143 4144

	return it->cur_task;
4145 4146
}

4147
/**
4148
 * css_task_iter_end - finish task iteration
4149 4150
 * @it: the task iterator to finish
 *
4151
 * Finish task iteration started by css_task_iter_start().
4152
 */
4153
void css_task_iter_end(struct css_task_iter *it)
4154
{
4155
	if (it->cur_cset) {
4156
		spin_lock_irq(&css_set_lock);
4157 4158
		list_del(&it->iters_node);
		put_css_set_locked(it->cur_cset);
4159
		spin_unlock_irq(&css_set_lock);
4160 4161
	}

4162 4163 4164
	if (it->cur_dcset)
		put_css_set(it->cur_dcset);

4165 4166
	if (it->cur_task)
		put_task_struct(it->cur_task);
4167 4168
}

4169
static void cgroup_procs_release(struct kernfs_open_file *of)
4170
{
4171 4172 4173 4174 4175
	if (of->priv) {
		css_task_iter_end(of->priv);
		kfree(of->priv);
	}
}
4176

4177 4178 4179 4180
static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos)
{
	struct kernfs_open_file *of = s->private;
	struct css_task_iter *it = of->priv;
4181

4182
	return css_task_iter_next(it);
4183
}
4184

4185 4186
static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos,
				  unsigned int iter_flags)
4187 4188 4189 4190
{
	struct kernfs_open_file *of = s->private;
	struct cgroup *cgrp = seq_css(s)->cgroup;
	struct css_task_iter *it = of->priv;
4191

4192
	/*
4193 4194
	 * When a seq_file is seeked, it's always traversed sequentially
	 * from position 0, so we can simply keep iterating on !0 *pos.
4195
	 */
4196 4197 4198
	if (!it) {
		if (WARN_ON_ONCE((*pos)++))
			return ERR_PTR(-EINVAL);
4199

4200 4201 4202 4203
		it = kzalloc(sizeof(*it), GFP_KERNEL);
		if (!it)
			return ERR_PTR(-ENOMEM);
		of->priv = it;
4204
		css_task_iter_start(&cgrp->self, iter_flags, it);
4205 4206
	} else if (!(*pos)++) {
		css_task_iter_end(it);
4207
		css_task_iter_start(&cgrp->self, iter_flags, it);
4208
	}
4209

4210 4211
	return cgroup_procs_next(s, NULL, NULL);
}
4212

4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229
static void *cgroup_procs_start(struct seq_file *s, loff_t *pos)
{
	struct cgroup *cgrp = seq_css(s)->cgroup;

	/*
	 * All processes of a threaded subtree belong to the domain cgroup
	 * of the subtree.  Only threads can be distributed across the
	 * subtree.  Reject reads on cgroup.procs in the subtree proper.
	 * They're always empty anyway.
	 */
	if (cgroup_is_threaded(cgrp))
		return ERR_PTR(-EOPNOTSUPP);

	return __cgroup_procs_start(s, pos, CSS_TASK_ITER_PROCS |
					    CSS_TASK_ITER_THREADED);
}

4230
static int cgroup_procs_show(struct seq_file *s, void *v)
4231
{
4232
	seq_printf(s, "%d\n", task_pid_vnr(v));
4233 4234 4235
	return 0;
}

4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
static int cgroup_procs_write_permission(struct cgroup *src_cgrp,
					 struct cgroup *dst_cgrp,
					 struct super_block *sb)
{
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
	struct cgroup *com_cgrp = src_cgrp;
	struct inode *inode;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* find the common ancestor */
	while (!cgroup_is_descendant(dst_cgrp, com_cgrp))
		com_cgrp = cgroup_parent(com_cgrp);

	/* %current should be authorized to migrate to the common ancestor */
	inode = kernfs_get_inode(sb, com_cgrp->procs_file.kn);
	if (!inode)
		return -ENOMEM;

	ret = inode_permission(inode, MAY_WRITE);
	iput(inode);
	if (ret)
		return ret;

	/*
	 * If namespaces are delegation boundaries, %current must be able
	 * to see both source and destination cgroups from its namespace.
	 */
	if ((cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE) &&
	    (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) ||
	     !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp)))
		return -ENOENT;

	return 0;
}

static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *src_cgrp, *dst_cgrp;
	struct task_struct *task;
	ssize_t ret;

	dst_cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!dst_cgrp)
		return -ENODEV;

	task = cgroup_procs_write_start(buf, true);
	ret = PTR_ERR_OR_ZERO(task);
	if (ret)
		goto out_unlock;

	/* find the source cgroup */
	spin_lock_irq(&css_set_lock);
	src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
	spin_unlock_irq(&css_set_lock);

	ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp,
					    of->file->f_path.dentry->d_sb);
	if (ret)
		goto out_finish;

	ret = cgroup_attach_task(dst_cgrp, task, true);

out_finish:
	cgroup_procs_write_finish(task);
out_unlock:
	cgroup_kn_unlock(of->kn);

	return ret ?: nbytes;
}

4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357
static void *cgroup_threads_start(struct seq_file *s, loff_t *pos)
{
	return __cgroup_procs_start(s, pos, 0);
}

static ssize_t cgroup_threads_write(struct kernfs_open_file *of,
				    char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *src_cgrp, *dst_cgrp;
	struct task_struct *task;
	ssize_t ret;

	buf = strstrip(buf);

	dst_cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!dst_cgrp)
		return -ENODEV;

	task = cgroup_procs_write_start(buf, false);
	ret = PTR_ERR_OR_ZERO(task);
	if (ret)
		goto out_unlock;

	/* find the source cgroup */
	spin_lock_irq(&css_set_lock);
	src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
	spin_unlock_irq(&css_set_lock);

	/* thread migrations follow the cgroup.procs delegation rule */
	ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp,
					    of->file->f_path.dentry->d_sb);
	if (ret)
		goto out_finish;

	/* and must be contained in the same domain */
	ret = -EOPNOTSUPP;
	if (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp)
		goto out_finish;

	ret = cgroup_attach_task(dst_cgrp, task, false);

out_finish:
	cgroup_procs_write_finish(task);
out_unlock:
	cgroup_kn_unlock(of->kn);

	return ret ?: nbytes;
}

4358
/* cgroup core interface files for the default hierarchy */
4359
static struct cftype cgroup_base_files[] = {
4360 4361 4362 4363 4364 4365
	{
		.name = "cgroup.type",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_type_show,
		.write = cgroup_type_write,
	},
4366
	{
4367
		.name = "cgroup.procs",
4368
		.flags = CFTYPE_NS_DELEGATABLE,
4369
		.file_offset = offsetof(struct cgroup, procs_file),
4370 4371 4372 4373
		.release = cgroup_procs_release,
		.seq_start = cgroup_procs_start,
		.seq_next = cgroup_procs_next,
		.seq_show = cgroup_procs_show,
4374
		.write = cgroup_procs_write,
4375
	},
4376 4377 4378 4379 4380 4381 4382 4383
	{
		.name = "cgroup.threads",
		.release = cgroup_procs_release,
		.seq_start = cgroup_threads_start,
		.seq_next = cgroup_procs_next,
		.seq_show = cgroup_procs_show,
		.write = cgroup_threads_write,
	},
4384 4385 4386 4387 4388 4389
	{
		.name = "cgroup.controllers",
		.seq_show = cgroup_controllers_show,
	},
	{
		.name = "cgroup.subtree_control",
4390
		.flags = CFTYPE_NS_DELEGATABLE,
4391
		.seq_show = cgroup_subtree_control_show,
4392
		.write = cgroup_subtree_control_write,
4393
	},
4394
	{
4395
		.name = "cgroup.events",
4396
		.flags = CFTYPE_NOT_ON_ROOT,
4397
		.file_offset = offsetof(struct cgroup, events_file),
4398
		.seq_show = cgroup_events_show,
4399
	},
4400 4401 4402 4403 4404 4405 4406 4407 4408 4409
	{
		.name = "cgroup.max.descendants",
		.seq_show = cgroup_max_descendants_show,
		.write = cgroup_max_descendants_write,
	},
	{
		.name = "cgroup.max.depth",
		.seq_show = cgroup_max_depth_show,
		.write = cgroup_max_depth_write,
	},
4410 4411
	{
		.name = "cgroup.stat",
T
Tejun Heo 已提交
4412
		.seq_show = cgroup_stat_show,
4413
	},
4414 4415
	{ }	/* terminate */
};
4416

4417 4418 4419 4420 4421 4422 4423
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4424 4425 4426
 *    and thus css_tryget_online() is guaranteed to fail, the css can be
 *    offlined by invoking offline_css().  After offlining, the base ref is
 *    put.  Implemented in css_killed_work_fn().
4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
4439
static void css_free_work_fn(struct work_struct *work)
4440 4441
{
	struct cgroup_subsys_state *css =
4442
		container_of(work, struct cgroup_subsys_state, destroy_work);
4443
	struct cgroup_subsys *ss = css->ss;
4444
	struct cgroup *cgrp = css->cgroup;
4445

4446 4447
	percpu_ref_exit(&css->refcnt);

4448
	if (ss) {
4449
		/* css free path */
4450
		struct cgroup_subsys_state *parent = css->parent;
4451 4452 4453 4454
		int id = css->id;

		ss->css_free(css);
		cgroup_idr_remove(&ss->css_idr, id);
4455
		cgroup_put(cgrp);
4456 4457 4458

		if (parent)
			css_put(parent);
4459 4460 4461
	} else {
		/* cgroup free path */
		atomic_dec(&cgrp->root->nr_cgrps);
4462
		cgroup1_pidlist_destroy_all(cgrp);
4463
		cancel_work_sync(&cgrp->release_agent_work);
4464

T
Tejun Heo 已提交
4465
		if (cgroup_parent(cgrp)) {
4466 4467 4468 4469 4470 4471
			/*
			 * We get a ref to the parent, and put the ref when
			 * this cgroup is being freed, so it's guaranteed
			 * that the parent won't be destroyed before its
			 * children.
			 */
T
Tejun Heo 已提交
4472
			cgroup_put(cgroup_parent(cgrp));
4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483
			kernfs_put(cgrp->kn);
			kfree(cgrp);
		} else {
			/*
			 * This is root cgroup's refcnt reaching zero,
			 * which indicates that the root should be
			 * released.
			 */
			cgroup_destroy_root(cgrp->root);
		}
	}
4484 4485
}

4486
static void css_free_rcu_fn(struct rcu_head *rcu_head)
4487 4488
{
	struct cgroup_subsys_state *css =
4489
		container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4490

4491
	INIT_WORK(&css->destroy_work, css_free_work_fn);
4492
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4493 4494
}

4495
static void css_release_work_fn(struct work_struct *work)
4496 4497
{
	struct cgroup_subsys_state *css =
4498
		container_of(work, struct cgroup_subsys_state, destroy_work);
4499
	struct cgroup_subsys *ss = css->ss;
4500
	struct cgroup *cgrp = css->cgroup;
4501

4502 4503
	mutex_lock(&cgroup_mutex);

4504
	css->flags |= CSS_RELEASED;
4505 4506
	list_del_rcu(&css->sibling);

4507 4508
	if (ss) {
		/* css release path */
4509
		cgroup_idr_replace(&ss->css_idr, NULL, css->id);
4510 4511
		if (ss->css_released)
			ss->css_released(css);
4512
	} else {
4513 4514
		struct cgroup *tcgrp;

4515
		/* cgroup release path */
4516 4517
		trace_cgroup_release(cgrp);

4518 4519 4520 4521
		for (tcgrp = cgroup_parent(cgrp); tcgrp;
		     tcgrp = cgroup_parent(tcgrp))
			tcgrp->nr_dying_descendants--;

4522 4523
		cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
		cgrp->id = -1;
4524 4525 4526 4527 4528 4529 4530 4531

		/*
		 * There are two control paths which try to determine
		 * cgroup from dentry without going through kernfs -
		 * cgroupstats_build() and css_tryget_online_from_dir().
		 * Those are supported by RCU protecting clearing of
		 * cgrp->kn->priv backpointer.
		 */
4532 4533 4534
		if (cgrp->kn)
			RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv,
					 NULL);
4535 4536

		cgroup_bpf_put(cgrp);
4537
	}
4538

4539 4540
	mutex_unlock(&cgroup_mutex);

4541
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4542 4543 4544 4545 4546 4547 4548
}

static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4549 4550
	INIT_WORK(&css->destroy_work, css_release_work_fn);
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4551 4552
}

4553 4554
static void init_and_link_css(struct cgroup_subsys_state *css,
			      struct cgroup_subsys *ss, struct cgroup *cgrp)
4555
{
4556 4557
	lockdep_assert_held(&cgroup_mutex);

4558
	cgroup_get_live(cgrp);
4559

4560
	memset(css, 0, sizeof(*css));
4561
	css->cgroup = cgrp;
4562
	css->ss = ss;
4563
	css->id = -1;
4564 4565
	INIT_LIST_HEAD(&css->sibling);
	INIT_LIST_HEAD(&css->children);
4566
	css->serial_nr = css_serial_nr_next++;
4567
	atomic_set(&css->online_cnt, 0);
4568

T
Tejun Heo 已提交
4569 4570
	if (cgroup_parent(cgrp)) {
		css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4571 4572
		css_get(css->parent);
	}
4573

4574
	BUG_ON(cgroup_css(cgrp, ss));
4575 4576
}

4577
/* invoke ->css_online() on a new CSS and mark it online if successful */
4578
static int online_css(struct cgroup_subsys_state *css)
4579
{
4580
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
4581 4582
	int ret = 0;

4583 4584
	lockdep_assert_held(&cgroup_mutex);

4585
	if (ss->css_online)
4586
		ret = ss->css_online(css);
4587
	if (!ret) {
4588
		css->flags |= CSS_ONLINE;
4589
		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4590 4591 4592 4593

		atomic_inc(&css->online_cnt);
		if (css->parent)
			atomic_inc(&css->parent->online_cnt);
4594
	}
T
Tejun Heo 已提交
4595
	return ret;
4596 4597
}

4598
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4599
static void offline_css(struct cgroup_subsys_state *css)
4600
{
4601
	struct cgroup_subsys *ss = css->ss;
4602 4603 4604 4605 4606 4607

	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

4608
	if (ss->css_offline)
4609
		ss->css_offline(css);
4610

4611
	css->flags &= ~CSS_ONLINE;
4612
	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4613 4614

	wake_up_all(&css->cgroup->offline_waitq);
4615 4616
}

4617
/**
4618
 * css_create - create a cgroup_subsys_state
4619 4620 4621 4622
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
4623 4624
 * css is online and installed in @cgrp.  This function doesn't create the
 * interface files.  Returns 0 on success, -errno on failure.
4625
 */
4626 4627
static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
					      struct cgroup_subsys *ss)
4628
{
T
Tejun Heo 已提交
4629
	struct cgroup *parent = cgroup_parent(cgrp);
4630
	struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4631 4632 4633 4634 4635
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgroup_mutex);

4636
	css = ss->css_alloc(parent_css);
4637 4638
	if (!css)
		css = ERR_PTR(-ENOMEM);
4639
	if (IS_ERR(css))
4640
		return css;
4641

4642
	init_and_link_css(css, ss, cgrp);
4643

4644
	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
4645
	if (err)
4646
		goto err_free_css;
4647

V
Vladimir Davydov 已提交
4648
	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);
4649
	if (err < 0)
4650
		goto err_free_css;
4651
	css->id = err;
4652

4653
	/* @css is ready to be brought online now, make it visible */
4654
	list_add_tail_rcu(&css->sibling, &parent_css->children);
4655
	cgroup_idr_replace(&ss->css_idr, css, css->id);
4656 4657 4658

	err = online_css(css);
	if (err)
4659
		goto err_list_del;
4660

4661
	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
T
Tejun Heo 已提交
4662
	    cgroup_parent(parent)) {
4663
		pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4664
			current->comm, current->pid, ss->name);
4665
		if (!strcmp(ss->name, "memory"))
4666
			pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4667 4668 4669
		ss->warned_broken_hierarchy = true;
	}

4670
	return css;
4671

4672 4673
err_list_del:
	list_del_rcu(&css->sibling);
4674
err_free_css:
4675
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4676
	return ERR_PTR(err);
4677 4678
}

4679 4680 4681 4682 4683
/*
 * The returned cgroup is fully initialized including its control mask, but
 * it isn't associated with its kernfs_node and doesn't have the control
 * mask applied.
 */
4684
static struct cgroup *cgroup_create(struct cgroup *parent)
4685
{
4686 4687 4688
	struct cgroup_root *root = parent->root;
	struct cgroup *cgrp, *tcgrp;
	int level = parent->level + 1;
4689
	int ret;
4690

T
Tejun Heo 已提交
4691
	/* allocate the cgroup and its ID, 0 is reserved for the root */
4692 4693
	cgrp = kzalloc(sizeof(*cgrp) +
		       sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL);
4694 4695
	if (!cgrp)
		return ERR_PTR(-ENOMEM);
4696

4697
	ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
4698 4699 4700
	if (ret)
		goto out_free_cgrp;

4701 4702 4703 4704
	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
V
Vladimir Davydov 已提交
4705
	cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL);
4706
	if (cgrp->id < 0) {
T
Tejun Heo 已提交
4707
		ret = -ENOMEM;
4708
		goto out_cancel_ref;
4709 4710
	}

4711
	init_cgroup_housekeeping(cgrp);
4712

4713
	cgrp->self.parent = &parent->self;
T
Tejun Heo 已提交
4714
	cgrp->root = root;
4715 4716
	cgrp->level = level;

4717
	for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) {
4718
		cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
4719

4720 4721 4722 4723
		if (tcgrp != cgrp)
			tcgrp->nr_descendants++;
	}

4724 4725 4726
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

4727 4728
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4729

4730
	cgrp->self.serial_nr = css_serial_nr_next++;
4731

4732
	/* allocation complete, commit to creation */
4733
	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
4734
	atomic_inc(&root->nr_cgrps);
4735
	cgroup_get_live(parent);
4736

4737 4738 4739 4740
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
4741
	cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4742

4743 4744
	/*
	 * On the default hierarchy, a child doesn't automatically inherit
4745
	 * subtree_control from the parent.  Each is configured manually.
4746
	 */
4747
	if (!cgroup_on_dfl(cgrp))
4748
		cgrp->subtree_control = cgroup_control(cgrp);
4749

4750 4751 4752
	if (parent)
		cgroup_bpf_inherit(cgrp, parent);

4753 4754
	cgroup_propagate_control(cgrp);

4755 4756 4757 4758 4759 4760 4761 4762 4763
	return cgrp;

out_cancel_ref:
	percpu_ref_exit(&cgrp->self.refcnt);
out_free_cgrp:
	kfree(cgrp);
	return ERR_PTR(ret);
}

4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786
static bool cgroup_check_hierarchy_limits(struct cgroup *parent)
{
	struct cgroup *cgroup;
	int ret = false;
	int level = 1;

	lockdep_assert_held(&cgroup_mutex);

	for (cgroup = parent; cgroup; cgroup = cgroup_parent(cgroup)) {
		if (cgroup->nr_descendants >= cgroup->max_descendants)
			goto fail;

		if (level > cgroup->max_depth)
			goto fail;

		level++;
	}

	ret = true;
fail:
	return ret;
}

4787
int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
4788 4789 4790
{
	struct cgroup *parent, *cgrp;
	struct kernfs_node *kn;
4791
	int ret;
4792 4793 4794 4795 4796

	/* do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable */
	if (strchr(name, '\n'))
		return -EINVAL;

4797
	parent = cgroup_kn_lock_live(parent_kn, false);
4798 4799 4800
	if (!parent)
		return -ENODEV;

4801 4802 4803 4804 4805
	if (!cgroup_check_hierarchy_limits(parent)) {
		ret = -EAGAIN;
		goto out_unlock;
	}

4806 4807 4808 4809 4810 4811
	cgrp = cgroup_create(parent);
	if (IS_ERR(cgrp)) {
		ret = PTR_ERR(cgrp);
		goto out_unlock;
	}

4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829
	/* create the directory */
	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
	if (IS_ERR(kn)) {
		ret = PTR_ERR(kn);
		goto out_destroy;
	}
	cgrp->kn = kn;

	/*
	 * This extra ref will be put in cgroup_free_fn() and guarantees
	 * that @cgrp->kn is always accessible.
	 */
	kernfs_get(kn);

	ret = cgroup_kn_set_ugid(kn);
	if (ret)
		goto out_destroy;

4830
	ret = css_populate_dir(&cgrp->self);
4831 4832 4833
	if (ret)
		goto out_destroy;

4834 4835 4836
	ret = cgroup_apply_control_enable(cgrp);
	if (ret)
		goto out_destroy;
4837

4838 4839
	trace_cgroup_mkdir(cgrp);

4840
	/* let's create and online css's */
T
Tejun Heo 已提交
4841
	kernfs_activate(kn);
4842

T
Tejun Heo 已提交
4843 4844
	ret = 0;
	goto out_unlock;
4845

4846 4847
out_destroy:
	cgroup_destroy_locked(cgrp);
T
Tejun Heo 已提交
4848
out_unlock:
4849
	cgroup_kn_unlock(parent_kn);
T
Tejun Heo 已提交
4850
	return ret;
4851 4852
}

4853 4854
/*
 * This is called when the refcnt of a css is confirmed to be killed.
4855 4856
 * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
 * initate destruction and put the css ref from kill_css().
4857 4858
 */
static void css_killed_work_fn(struct work_struct *work)
4859
{
4860 4861
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
4862

4863
	mutex_lock(&cgroup_mutex);
4864

4865 4866 4867 4868 4869 4870 4871 4872
	do {
		offline_css(css);
		css_put(css);
		/* @css can't go away while we're holding cgroup_mutex */
		css = css->parent;
	} while (css && atomic_dec_and_test(&css->online_cnt));

	mutex_unlock(&cgroup_mutex);
4873 4874
}

4875 4876
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
4877 4878 4879 4880
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4881 4882 4883 4884
	if (atomic_dec_and_test(&css->online_cnt)) {
		INIT_WORK(&css->destroy_work, css_killed_work_fn);
		queue_work(cgroup_destroy_wq, &css->destroy_work);
	}
4885 4886
}

4887 4888 4889 4890 4891 4892
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
4893 4894
 * asynchronously once css_tryget_online() is guaranteed to fail and when
 * the reference count reaches zero, @css will be released.
4895 4896
 */
static void kill_css(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
4897
{
4898
	lockdep_assert_held(&cgroup_mutex);
4899

4900 4901 4902 4903 4904
	if (css->flags & CSS_DYING)
		return;

	css->flags |= CSS_DYING;

T
Tejun Heo 已提交
4905 4906 4907 4908
	/*
	 * This must happen before css is disassociated with its cgroup.
	 * See seq_css() for details.
	 */
4909
	css_clear_dir(css);
4910

T
Tejun Heo 已提交
4911 4912 4913 4914 4915 4916 4917 4918 4919
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
4920
	 * css_tryget_online().  We can't simply call percpu_ref_kill() and
T
Tejun Heo 已提交
4921 4922 4923 4924 4925 4926 4927
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4928 4929 4930 4931 4932 4933 4934 4935
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
4936 4937 4938
 * guarantee that css_tryget_online() won't succeed by the time
 * ->css_offline() is invoked.  To satisfy all the requirements,
 * destruction is implemented in the following two steps.
4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
4954 4955
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4956
{
4957
	struct cgroup *tcgrp, *parent = cgroup_parent(cgrp);
T
Tejun Heo 已提交
4958
	struct cgroup_subsys_state *css;
4959
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
4960
	int ssid;
4961

4962 4963
	lockdep_assert_held(&cgroup_mutex);

4964 4965 4966 4967 4968
	/*
	 * Only migration can raise populated from zero and we're already
	 * holding cgroup_mutex.
	 */
	if (cgroup_is_populated(cgrp))
4969
		return -EBUSY;
L
Li Zefan 已提交
4970

4971
	/*
4972 4973 4974
	 * Make sure there's no live children.  We can't test emptiness of
	 * ->self.children as dead children linger on it while being
	 * drained; otherwise, "rmdir parent/child parent" may fail.
4975
	 */
4976
	if (css_has_online_children(&cgrp->self))
4977 4978
		return -EBUSY;

4979
	/*
4980 4981 4982 4983
	 * Mark @cgrp and the associated csets dead.  The former prevents
	 * further task migration and child creation by disabling
	 * cgroup_lock_live_group().  The latter makes the csets ignored by
	 * the migration path.
4984
	 */
4985
	cgrp->self.flags &= ~CSS_ONLINE;
4986

4987
	spin_lock_irq(&css_set_lock);
4988 4989
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		link->cset->dead = true;
4990
	spin_unlock_irq(&css_set_lock);
4991

4992
	/* initiate massacre of all css's */
T
Tejun Heo 已提交
4993 4994
	for_each_css(css, ssid, cgrp)
		kill_css(css);
4995 4996

	/*
4997 4998
	 * Remove @cgrp directory along with the base files.  @cgrp has an
	 * extra ref on its kn.
4999
	 */
5000
	kernfs_remove(cgrp->kn);
5001

5002 5003 5004
	if (parent && cgroup_is_threaded(cgrp))
		parent->nr_threaded_children--;

5005 5006 5007 5008 5009
	for (tcgrp = cgroup_parent(cgrp); tcgrp; tcgrp = cgroup_parent(tcgrp)) {
		tcgrp->nr_descendants--;
		tcgrp->nr_dying_descendants++;
	}

5010
	cgroup1_check_for_release(parent);
T
Tejun Heo 已提交
5011

5012
	/* put the base reference */
5013
	percpu_ref_kill(&cgrp->self.refcnt);
5014

5015 5016 5017
	return 0;
};

5018
int cgroup_rmdir(struct kernfs_node *kn)
5019
{
5020
	struct cgroup *cgrp;
T
Tejun Heo 已提交
5021
	int ret = 0;
5022

5023
	cgrp = cgroup_kn_lock_live(kn, false);
5024 5025
	if (!cgrp)
		return 0;
5026

5027
	ret = cgroup_destroy_locked(cgrp);
5028

5029 5030 5031
	if (!ret)
		trace_cgroup_rmdir(cgrp);

5032
	cgroup_kn_unlock(kn);
5033
	return ret;
5034 5035
}

T
Tejun Heo 已提交
5036
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
5037
	.show_options		= cgroup_show_options,
T
Tejun Heo 已提交
5038 5039 5040
	.remount_fs		= cgroup_remount,
	.mkdir			= cgroup_mkdir,
	.rmdir			= cgroup_rmdir,
5041
	.show_path		= cgroup_show_path,
T
Tejun Heo 已提交
5042 5043
};

5044
static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
5045 5046
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
5047

5048
	pr_debug("Initializing cgroup subsys %s\n", ss->name);
5049

5050 5051
	mutex_lock(&cgroup_mutex);

5052
	idr_init(&ss->css_idr);
T
Tejun Heo 已提交
5053
	INIT_LIST_HEAD(&ss->cfts);
5054

5055 5056 5057
	/* Create the root cgroup state for this subsystem */
	ss->root = &cgrp_dfl_root;
	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
5058 5059
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
5060
	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
5061 5062 5063 5064 5065 5066 5067

	/*
	 * Root csses are never destroyed and we can't initialize
	 * percpu_ref during early init.  Disable refcnting.
	 */
	css->flags |= CSS_NO_REF;

5068
	if (early) {
5069
		/* allocation can't be done safely during early init */
5070 5071 5072 5073 5074
		css->id = 1;
	} else {
		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
		BUG_ON(css->id < 0);
	}
5075

L
Li Zefan 已提交
5076
	/* Update the init_css_set to contain a subsys
5077
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
5078
	 * newly registered, all tasks and hence the
5079
	 * init_css_set is in the subsystem's root cgroup. */
5080
	init_css_set.subsys[ss->id] = css;
5081

5082 5083
	have_fork_callback |= (bool)ss->fork << ss->id;
	have_exit_callback |= (bool)ss->exit << ss->id;
5084
	have_free_callback |= (bool)ss->free << ss->id;
5085
	have_canfork_callback |= (bool)ss->can_fork << ss->id;
5086

L
Li Zefan 已提交
5087 5088 5089 5090 5091
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

5092
	BUG_ON(online_css(css));
5093

B
Ben Blum 已提交
5094 5095 5096
	mutex_unlock(&cgroup_mutex);
}

5097
/**
L
Li Zefan 已提交
5098 5099 5100 5101
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
5102 5103 5104
 */
int __init cgroup_init_early(void)
{
5105
	static struct cgroup_sb_opts __initdata opts;
5106
	struct cgroup_subsys *ss;
5107
	int i;
5108

5109
	init_cgroup_root(&cgrp_dfl_root, &opts);
5110 5111
	cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;

5112
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
5113

T
Tejun Heo 已提交
5114
	for_each_subsys(ss, i) {
5115
		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
5116
		     "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n",
5117
		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
5118
		     ss->id, ss->name);
5119 5120 5121
		WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
		     "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);

5122
		ss->id = i;
5123
		ss->name = cgroup_subsys_name[i];
5124 5125
		if (!ss->legacy_name)
			ss->legacy_name = cgroup_subsys_name[i];
5126 5127

		if (ss->early_init)
5128
			cgroup_init_subsys(ss, true);
5129 5130 5131 5132
	}
	return 0;
}

5133
static u16 cgroup_disable_mask __initdata;
5134

5135
/**
L
Li Zefan 已提交
5136 5137 5138 5139
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
5140 5141 5142
 */
int __init cgroup_init(void)
{
5143
	struct cgroup_subsys *ss;
5144
	int ssid;
5145

5146
	BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16);
5147
	BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
5148 5149
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
	BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files));
5150

5151 5152 5153 5154 5155 5156
	/*
	 * The latency of the synchronize_sched() is too high for cgroups,
	 * avoid it at the cost of forcing all readers into the slow path.
	 */
	rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss);

5157 5158
	get_user_ns(init_cgroup_ns.user_ns);

T
Tejun Heo 已提交
5159 5160
	mutex_lock(&cgroup_mutex);

5161 5162 5163 5164 5165 5166
	/*
	 * Add init_css_set to the hash table so that dfl_root can link to
	 * it during init.
	 */
	hash_add(css_set_table, &init_css_set.hlist,
		 css_set_hash(init_css_set.subsys));
5167

5168
	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0, 0));
5169

T
Tejun Heo 已提交
5170 5171
	mutex_unlock(&cgroup_mutex);

5172
	for_each_subsys(ss, ssid) {
5173 5174 5175 5176 5177 5178 5179 5180 5181 5182
		if (ss->early_init) {
			struct cgroup_subsys_state *css =
				init_css_set.subsys[ss->id];

			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
						   GFP_KERNEL);
			BUG_ON(css->id < 0);
		} else {
			cgroup_init_subsys(ss, false);
		}
5183

T
Tejun Heo 已提交
5184 5185
		list_add_tail(&init_css_set.e_cset_node[ssid],
			      &cgrp_dfl_root.cgrp.e_csets[ssid]);
5186 5187

		/*
5188 5189 5190
		 * Setting dfl_root subsys_mask needs to consider the
		 * disabled flag and cftype registration needs kmalloc,
		 * both of which aren't available during early_init.
5191
		 */
5192 5193 5194 5195
		if (cgroup_disable_mask & (1 << ssid)) {
			static_branch_disable(cgroup_subsys_enabled_key[ssid]);
			printk(KERN_INFO "Disabling %s control group subsystem\n",
			       ss->name);
5196
			continue;
5197
		}
5198

5199
		if (cgroup1_ssid_disabled(ssid))
5200 5201 5202
			printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n",
			       ss->name);

5203 5204
		cgrp_dfl_root.subsys_mask |= 1 << ss->id;

5205 5206 5207
		/* implicit controllers must be threaded too */
		WARN_ON(ss->implicit_on_dfl && !ss->threaded);

5208 5209 5210
		if (ss->implicit_on_dfl)
			cgrp_dfl_implicit_ss_mask |= 1 << ss->id;
		else if (!ss->dfl_cftypes)
T
Tejun Heo 已提交
5211
			cgrp_dfl_inhibit_ss_mask |= 1 << ss->id;
5212

5213 5214 5215
		if (ss->threaded)
			cgrp_dfl_threaded_ss_mask |= 1 << ss->id;

5216 5217 5218 5219 5220
		if (ss->dfl_cftypes == ss->legacy_cftypes) {
			WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
		} else {
			WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
			WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
5221
		}
5222 5223 5224

		if (ss->bind)
			ss->bind(init_css_set.subsys[ssid]);
5225 5226 5227 5228

		mutex_lock(&cgroup_mutex);
		css_populate_dir(init_css_set.subsys[ssid]);
		mutex_unlock(&cgroup_mutex);
5229 5230
	}

5231 5232 5233 5234 5235
	/* init_css_set.subsys[] has been updated, re-hash */
	hash_del(&init_css_set.hlist);
	hash_add(css_set_table, &init_css_set.hlist,
		 css_set_hash(init_css_set.subsys));

5236 5237
	WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup"));
	WARN_ON(register_filesystem(&cgroup_fs_type));
5238
	WARN_ON(register_filesystem(&cgroup2_fs_type));
5239
	WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations));
5240

T
Tejun Heo 已提交
5241
	return 0;
5242
}
5243

5244 5245 5246 5247 5248
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
5249
	 * Use 1 for @max_active.
5250 5251 5252 5253
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
5254
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
5255 5256 5257 5258 5259
	BUG_ON(!cgroup_destroy_wq);
	return 0;
}
core_initcall(cgroup_wq_init);

5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271
void cgroup_path_from_kernfs_id(const union kernfs_node_id *id,
					char *buf, size_t buflen)
{
	struct kernfs_node *kn;

	kn = kernfs_get_node_by_id(cgrp_dfl_root.kf_root, id);
	if (!kn)
		return;
	kernfs_path(kn, buf, buflen);
	kernfs_put(kn);
}

5272 5273 5274 5275 5276
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 */
Z
Zefan Li 已提交
5277 5278
int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
		     struct pid *pid, struct task_struct *tsk)
5279
{
5280
	char *buf;
5281
	int retval;
5282
	struct cgroup_root *root;
5283 5284

	retval = -ENOMEM;
T
Tejun Heo 已提交
5285
	buf = kmalloc(PATH_MAX, GFP_KERNEL);
5286 5287 5288 5289
	if (!buf)
		goto out;

	mutex_lock(&cgroup_mutex);
5290
	spin_lock_irq(&css_set_lock);
5291

5292
	for_each_root(root) {
5293
		struct cgroup_subsys *ss;
5294
		struct cgroup *cgrp;
T
Tejun Heo 已提交
5295
		int ssid, count = 0;
5296

T
Tejun Heo 已提交
5297
		if (root == &cgrp_dfl_root && !cgrp_dfl_visible)
5298 5299
			continue;

5300
		seq_printf(m, "%d:", root->hierarchy_id);
5301 5302 5303 5304
		if (root != &cgrp_dfl_root)
			for_each_subsys(ss, ssid)
				if (root->subsys_mask & (1 << ssid))
					seq_printf(m, "%s%s", count++ ? "," : "",
5305
						   ss->legacy_name);
5306 5307 5308
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
5309
		seq_putc(m, ':');
5310

5311
		cgrp = task_cgroup_from_root(tsk, root);
5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322

		/*
		 * On traditional hierarchies, all zombie tasks show up as
		 * belonging to the root cgroup.  On the default hierarchy,
		 * while a zombie doesn't show up in "cgroup.procs" and
		 * thus can't be migrated, its /proc/PID/cgroup keeps
		 * reporting the cgroup it belonged to before exiting.  If
		 * the cgroup is removed before the zombie is reaped,
		 * " (deleted)" is appended to the cgroup path.
		 */
		if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {
5323
			retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX,
5324
						current->nsproxy->cgroup_ns);
5325
			if (retval >= PATH_MAX)
5326
				retval = -ENAMETOOLONG;
5327
			if (retval < 0)
5328
				goto out_unlock;
5329 5330

			seq_puts(m, buf);
5331
		} else {
5332
			seq_puts(m, "/");
T
Tejun Heo 已提交
5333
		}
5334 5335 5336 5337 5338

		if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp))
			seq_puts(m, " (deleted)\n");
		else
			seq_putc(m, '\n');
5339 5340
	}

Z
Zefan Li 已提交
5341
	retval = 0;
5342
out_unlock:
5343
	spin_unlock_irq(&css_set_lock);
5344 5345 5346 5347 5348 5349
	mutex_unlock(&cgroup_mutex);
	kfree(buf);
out:
	return retval;
}

5350
/**
5351
 * cgroup_fork - initialize cgroup related fields during copy_process()
L
Li Zefan 已提交
5352
 * @child: pointer to task_struct of forking parent process.
5353
 *
5354 5355 5356
 * A task is associated with the init_css_set until cgroup_post_fork()
 * attaches it to the parent's css_set.  Empty cg_list indicates that
 * @child isn't holding reference to its css_set.
5357 5358 5359
 */
void cgroup_fork(struct task_struct *child)
{
5360
	RCU_INIT_POINTER(child->cgroups, &init_css_set);
5361
	INIT_LIST_HEAD(&child->cg_list);
5362 5363
}

5364 5365 5366 5367 5368 5369 5370 5371
/**
 * cgroup_can_fork - called on a new task before the process is exposed
 * @child: the task in question.
 *
 * This calls the subsystem can_fork() callbacks. If the can_fork() callback
 * returns an error, the fork aborts with that error code. This allows for
 * a cgroup subsystem to conditionally allow or deny new forks.
 */
5372
int cgroup_can_fork(struct task_struct *child)
5373 5374 5375 5376
{
	struct cgroup_subsys *ss;
	int i, j, ret;

5377
	do_each_subsys_mask(ss, i, have_canfork_callback) {
5378
		ret = ss->can_fork(child);
5379 5380
		if (ret)
			goto out_revert;
5381
	} while_each_subsys_mask();
5382 5383 5384 5385 5386 5387 5388 5389

	return 0;

out_revert:
	for_each_subsys(ss, j) {
		if (j >= i)
			break;
		if (ss->cancel_fork)
5390
			ss->cancel_fork(child);
5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402
	}

	return ret;
}

/**
 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
 * @child: the task in question
 *
 * This calls the cancel_fork() callbacks if a fork failed *after*
 * cgroup_can_fork() succeded.
 */
5403
void cgroup_cancel_fork(struct task_struct *child)
5404 5405 5406 5407 5408 5409
{
	struct cgroup_subsys *ss;
	int i;

	for_each_subsys(ss, i)
		if (ss->cancel_fork)
5410
			ss->cancel_fork(child);
5411 5412
}

5413
/**
L
Li Zefan 已提交
5414 5415 5416
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
5417 5418 5419
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
5420
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5421
 * list.
L
Li Zefan 已提交
5422
 */
5423
void cgroup_post_fork(struct task_struct *child)
5424
{
5425
	struct cgroup_subsys *ss;
5426 5427
	int i;

5428
	/*
D
Dongsheng Yang 已提交
5429
	 * This may race against cgroup_enable_task_cg_lists().  As that
5430 5431 5432 5433 5434 5435 5436
	 * function sets use_task_css_set_links before grabbing
	 * tasklist_lock and we just went through tasklist_lock to add
	 * @child, it's guaranteed that either we see the set
	 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
	 * @child during its iteration.
	 *
	 * If we won the race, @child is associated with %current's
5437
	 * css_set.  Grabbing css_set_lock guarantees both that the
5438 5439 5440 5441 5442 5443
	 * association is stable, and, on completion of the parent's
	 * migration, @child is visible in the source of migration or
	 * already in the destination cgroup.  This guarantee is necessary
	 * when implementing operations which need to migrate all tasks of
	 * a cgroup to another.
	 *
D
Dongsheng Yang 已提交
5444
	 * Note that if we lose to cgroup_enable_task_cg_lists(), @child
5445 5446 5447
	 * will remain in init_css_set.  This is safe because all tasks are
	 * in the init_css_set before cg_links is enabled and there's no
	 * operation which transfers all tasks out of init_css_set.
5448
	 */
5449
	if (use_task_css_set_links) {
5450 5451
		struct css_set *cset;

5452
		spin_lock_irq(&css_set_lock);
5453
		cset = task_css_set(current);
5454 5455
		if (list_empty(&child->cg_list)) {
			get_css_set(cset);
5456
			cset->nr_tasks++;
T
Tejun Heo 已提交
5457
			css_set_move_task(child, NULL, cset, false);
5458
		}
5459
		spin_unlock_irq(&css_set_lock);
5460
	}
5461 5462 5463 5464 5465 5466

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
5467
	do_each_subsys_mask(ss, i, have_fork_callback) {
5468
		ss->fork(child);
5469
	} while_each_subsys_mask();
5470
}
5471

5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
5484 5485 5486 5487 5488
 * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
 * call cgroup_exit() while the task is still competent to handle
 * notify_on_release(), then leave the task attached to the root cgroup in
 * each hierarchy for the remainder of its exit.  No need to bother with
 * init_css_set refcnting.  init_css_set never goes away and we can't race
5489
 * with migration path - PF_EXITING is visible to migration path.
5490
 */
5491
void cgroup_exit(struct task_struct *tsk)
5492
{
5493
	struct cgroup_subsys *ss;
5494
	struct css_set *cset;
5495
	int i;
5496 5497

	/*
5498
	 * Unlink from @tsk from its css_set.  As migration path can't race
5499
	 * with us, we can check css_set and cg_list without synchronization.
5500
	 */
5501 5502
	cset = task_css_set(tsk);

5503
	if (!list_empty(&tsk->cg_list)) {
5504
		spin_lock_irq(&css_set_lock);
T
Tejun Heo 已提交
5505
		css_set_move_task(tsk, cset, NULL, false);
5506
		cset->nr_tasks--;
5507
		spin_unlock_irq(&css_set_lock);
5508 5509
	} else {
		get_css_set(cset);
5510 5511
	}

5512
	/* see cgroup_post_fork() for details */
5513
	do_each_subsys_mask(ss, i, have_exit_callback) {
5514
		ss->exit(tsk);
5515
	} while_each_subsys_mask();
5516
}
5517

5518 5519 5520
void cgroup_free(struct task_struct *task)
{
	struct css_set *cset = task_css_set(task);
5521 5522 5523
	struct cgroup_subsys *ss;
	int ssid;

5524
	do_each_subsys_mask(ss, ssid, have_free_callback) {
5525
		ss->free(task);
5526
	} while_each_subsys_mask();
5527

5528
	put_css_set(cset);
5529
}
5530

5531 5532
static int __init cgroup_disable(char *str)
{
5533
	struct cgroup_subsys *ss;
5534
	char *token;
5535
	int i;
5536 5537 5538 5539

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
5540

T
Tejun Heo 已提交
5541
		for_each_subsys(ss, i) {
5542 5543 5544
			if (strcmp(token, ss->name) &&
			    strcmp(token, ss->legacy_name))
				continue;
5545
			cgroup_disable_mask |= 1 << i;
5546 5547 5548 5549 5550
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
5551

5552
/**
5553
 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5554 5555
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
5556
 *
5557 5558 5559
 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
 * to get the corresponding css and return it.  If such css doesn't exist
 * or can't be pinned, an ERR_PTR value is returned.
S
Stephane Eranian 已提交
5560
 */
5561 5562
struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
						       struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
5563
{
T
Tejun Heo 已提交
5564
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
5565
	struct file_system_type *s_type = dentry->d_sb->s_type;
T
Tejun Heo 已提交
5566
	struct cgroup_subsys_state *css = NULL;
S
Stephane Eranian 已提交
5567 5568
	struct cgroup *cgrp;

5569
	/* is @dentry a cgroup dir? */
5570 5571
	if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) ||
	    !kn || kernfs_type(kn) != KERNFS_DIR)
S
Stephane Eranian 已提交
5572 5573
		return ERR_PTR(-EBADF);

5574 5575
	rcu_read_lock();

T
Tejun Heo 已提交
5576 5577 5578
	/*
	 * This path doesn't originate from kernfs and @kn could already
	 * have been or be removed at any point.  @kn->priv is RCU
5579
	 * protected for this access.  See css_release_work_fn() for details.
T
Tejun Heo 已提交
5580
	 */
5581
	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);
T
Tejun Heo 已提交
5582 5583
	if (cgrp)
		css = cgroup_css(cgrp, ss);
5584

5585
	if (!css || !css_tryget_online(css))
5586 5587 5588 5589
		css = ERR_PTR(-ENOENT);

	rcu_read_unlock();
	return css;
S
Stephane Eranian 已提交
5590 5591
}

5592 5593 5594 5595 5596 5597 5598 5599 5600 5601
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
5602
	WARN_ON_ONCE(!rcu_read_lock_held());
5603
	return idr_find(&ss->css_idr, id);
S
Stephane Eranian 已提交
5604 5605
}

5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625
/**
 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
 * @path: path on the default hierarchy
 *
 * Find the cgroup at @path on the default hierarchy, increment its
 * reference count and return it.  Returns pointer to the found cgroup on
 * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
 * if @path points to a non-directory.
 */
struct cgroup *cgroup_get_from_path(const char *path)
{
	struct kernfs_node *kn;
	struct cgroup *cgrp;

	mutex_lock(&cgroup_mutex);

	kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path);
	if (kn) {
		if (kernfs_type(kn) == KERNFS_DIR) {
			cgrp = kn->priv;
5626
			cgroup_get_live(cgrp);
5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639
		} else {
			cgrp = ERR_PTR(-ENOTDIR);
		}
		kernfs_put(kn);
	} else {
		cgrp = ERR_PTR(-ENOENT);
	}

	mutex_unlock(&cgroup_mutex);
	return cgrp;
}
EXPORT_SYMBOL_GPL(cgroup_get_from_path);

5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673
/**
 * cgroup_get_from_fd - get a cgroup pointer from a fd
 * @fd: fd obtained by open(cgroup2_dir)
 *
 * Find the cgroup from a fd which should be obtained
 * by opening a cgroup directory.  Returns a pointer to the
 * cgroup on success. ERR_PTR is returned if the cgroup
 * cannot be found.
 */
struct cgroup *cgroup_get_from_fd(int fd)
{
	struct cgroup_subsys_state *css;
	struct cgroup *cgrp;
	struct file *f;

	f = fget_raw(fd);
	if (!f)
		return ERR_PTR(-EBADF);

	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);
	fput(f);
	if (IS_ERR(css))
		return ERR_CAST(css);

	cgrp = css->cgroup;
	if (!cgroup_on_dfl(cgrp)) {
		cgroup_put(cgrp);
		return ERR_PTR(-EBADF);
	}

	return cgrp;
}
EXPORT_SYMBOL_GPL(cgroup_get_from_fd);

T
Tejun Heo 已提交
5674 5675 5676 5677 5678 5679 5680 5681
/*
 * sock->sk_cgrp_data handling.  For more info, see sock_cgroup_data
 * definition in cgroup-defs.h.
 */
#ifdef CONFIG_SOCK_CGROUP_DATA

#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)

5682
DEFINE_SPINLOCK(cgroup_sk_update_lock);
T
Tejun Heo 已提交
5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703
static bool cgroup_sk_alloc_disabled __read_mostly;

void cgroup_sk_alloc_disable(void)
{
	if (cgroup_sk_alloc_disabled)
		return;
	pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
	cgroup_sk_alloc_disabled = true;
}

#else

#define cgroup_sk_alloc_disabled	false

#endif

void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
{
	if (cgroup_sk_alloc_disabled)
		return;

5704 5705
	/* Socket clone path */
	if (skcd->val) {
5706 5707 5708 5709 5710
		/*
		 * We might be cloning a socket which is left in an empty
		 * cgroup and the cgroup might have already been rmdir'd.
		 * Don't use cgroup_get_live().
		 */
5711 5712 5713 5714
		cgroup_get(sock_cgroup_ptr(skcd));
		return;
	}

T
Tejun Heo 已提交
5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737
	rcu_read_lock();

	while (true) {
		struct css_set *cset;

		cset = task_css_set(current);
		if (likely(cgroup_tryget(cset->dfl_cgrp))) {
			skcd->val = (unsigned long)cset->dfl_cgrp;
			break;
		}
		cpu_relax();
	}

	rcu_read_unlock();
}

void cgroup_sk_free(struct sock_cgroup_data *skcd)
{
	cgroup_put(sock_cgroup_ptr(skcd));
}

#endif	/* CONFIG_SOCK_CGROUP_DATA */

5738
#ifdef CONFIG_CGROUP_BPF
5739 5740
int cgroup_bpf_update(struct cgroup *cgrp, struct bpf_prog *prog,
		      enum bpf_attach_type type, bool overridable)
5741 5742
{
	struct cgroup *parent = cgroup_parent(cgrp);
5743
	int ret;
5744 5745

	mutex_lock(&cgroup_mutex);
5746
	ret = __cgroup_bpf_update(cgrp, parent, prog, type, overridable);
5747
	mutex_unlock(&cgroup_mutex);
5748
	return ret;
5749 5750
}
#endif /* CONFIG_CGROUP_BPF */