cgroup.c 162.6 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31 32
#include "cgroup-internal.h"

33
#include <linux/cred.h>
34
#include <linux/errno.h>
35
#include <linux/init_task.h>
36
#include <linux/kernel.h>
37
#include <linux/magic.h>
38 39 40
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
41
#include <linux/proc_fs.h>
42 43
#include <linux/rcupdate.h>
#include <linux/sched.h>
44
#include <linux/sched/task.h>
45 46
#include <linux/slab.h>
#include <linux/spinlock.h>
47
#include <linux/percpu-rwsem.h>
48
#include <linux/string.h>
49
#include <linux/hashtable.h>
50
#include <linux/idr.h>
51
#include <linux/kthread.h>
A
Arun Sharma 已提交
52
#include <linux/atomic.h>
53
#include <linux/cpuset.h>
54 55
#include <linux/proc_ns.h>
#include <linux/nsproxy.h>
56
#include <linux/file.h>
57
#include <linux/sched/cputime.h>
58
#include <linux/backing-dev.h>
T
Tejun Heo 已提交
59
#include <net/sock.h>
J
Johannes Weiner 已提交
60
#include <linux/psi.h>
61

62 63 64
#define CREATE_TRACE_POINTS
#include <trace/events/cgroup.h>

T
Tejun Heo 已提交
65 66
#define CGROUP_FILE_NAME_MAX		(MAX_CGROUP_TYPE_NAMELEN +	\
					 MAX_CFTYPE_NAME + 2)
67 68
/* let's not notify more than 100 times per second */
#define CGROUP_FILE_NOTIFY_MIN_INTV	DIV_ROUND_UP(HZ, 100)
T
Tejun Heo 已提交
69

T
Tejun Heo 已提交
70 71 72 73
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
74
 * css_set_lock protects task->cgroups pointer, the list of css_set
75
 * objects, and the chain of tasks off each css_set.
T
Tejun Heo 已提交
76
 *
77 78
 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
 * cgroup.h can use them for lockdep annotations.
T
Tejun Heo 已提交
79
 */
T
Tejun Heo 已提交
80
DEFINE_MUTEX(cgroup_mutex);
81
DEFINE_SPINLOCK(css_set_lock);
82 83

#ifdef CONFIG_PROVE_RCU
84
EXPORT_SYMBOL_GPL(cgroup_mutex);
85
EXPORT_SYMBOL_GPL(css_set_lock);
T
Tejun Heo 已提交
86 87
#endif

88 89 90
DEFINE_SPINLOCK(trace_cgroup_path_lock);
char trace_cgroup_path[TRACE_CGROUP_PATH_LEN];

91
/*
92 93
 * Protects cgroup_idr and css_idr so that IDs can be released without
 * grabbing cgroup_mutex.
94 95 96
 */
static DEFINE_SPINLOCK(cgroup_idr_lock);

97 98 99 100 101 102
/*
 * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
 * against file removal/re-creation across css hiding.
 */
static DEFINE_SPINLOCK(cgroup_file_kn_lock);

103 104
struct percpu_rw_semaphore cgroup_threadgroup_rwsem;

T
Tejun Heo 已提交
105
#define cgroup_assert_mutex_or_rcu_locked()				\
106 107
	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
			   !lockdep_is_held(&cgroup_mutex),		\
T
Tejun Heo 已提交
108
			   "cgroup_mutex or RCU read lock required");
109

110 111 112 113 114 115 116 117
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

T
Tejun Heo 已提交
118
/* generate an array of cgroup subsystem pointers */
119
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
120
struct cgroup_subsys *cgroup_subsys[] = {
121 122
#include <linux/cgroup_subsys.h>
};
123 124 125 126 127
#undef SUBSYS

/* array of cgroup subsystem names */
#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
static const char *cgroup_subsys_name[] = {
128 129
#include <linux/cgroup_subsys.h>
};
130
#undef SUBSYS
131

132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
#define SUBSYS(_x)								\
	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key);			\
	DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key);			\
	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key);			\
	EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
#include <linux/cgroup_subsys.h>
#undef SUBSYS

#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
static struct static_key_true *cgroup_subsys_enabled_key[] = {
#include <linux/cgroup_subsys.h>
};
#undef SUBSYS

#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
#include <linux/cgroup_subsys.h>
};
#undef SUBSYS

T
Tejun Heo 已提交
153
static DEFINE_PER_CPU(struct cgroup_rstat_cpu, cgrp_dfl_root_rstat_cpu);
154

155
/*
156
 * The default hierarchy, reserved for the subsystems that are otherwise
157 158
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
159
 */
T
Tejun Heo 已提交
160
struct cgroup_root cgrp_dfl_root = { .cgrp.rstat_cpu = &cgrp_dfl_root_rstat_cpu };
T
Tejun Heo 已提交
161
EXPORT_SYMBOL_GPL(cgrp_dfl_root);
162

T
Tejun Heo 已提交
163 164 165 166
/*
 * The default hierarchy always exists but is hidden until mounted for the
 * first time.  This is for backward compatibility.
 */
T
Tejun Heo 已提交
167
static bool cgrp_dfl_visible;
168

169
/* some controllers are not supported in the default hierarchy */
T
Tejun Heo 已提交
170
static u16 cgrp_dfl_inhibit_ss_mask;
171

172
/* some controllers are implicitly enabled on the default hierarchy */
T
Tejun Heo 已提交
173
static u16 cgrp_dfl_implicit_ss_mask;
174

175 176 177
/* some controllers can be threaded on the default hierarchy */
static u16 cgrp_dfl_threaded_ss_mask;

178
/* The list of hierarchy roots */
179
LIST_HEAD(cgroup_roots);
180
static int cgroup_root_count;
181

T
Tejun Heo 已提交
182
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
183
static DEFINE_IDR(cgroup_hierarchy_idr);
184

185
/*
186 187 188 189 190
 * Assign a monotonically increasing serial number to csses.  It guarantees
 * cgroups with bigger numbers are newer than those with smaller numbers.
 * Also, as csses are always appended to the parent's ->children list, it
 * guarantees that sibling csses are always sorted in the ascending serial
 * number order on the list.  Protected by cgroup_mutex.
191
 */
192
static u64 css_serial_nr_next = 1;
193

194
/*
T
Tejun Heo 已提交
195 196
 * These bitmasks identify subsystems with specific features to avoid
 * having to do iterative checks repeatedly.
197
 */
198 199
static u16 have_fork_callback __read_mostly;
static u16 have_exit_callback __read_mostly;
200
static u16 have_release_callback __read_mostly;
T
Tejun Heo 已提交
201
static u16 have_canfork_callback __read_mostly;
202

203 204
/* cgroup namespace for init task */
struct cgroup_namespace init_cgroup_ns = {
205
	.count		= REFCOUNT_INIT(2),
206 207 208 209 210 211
	.user_ns	= &init_user_ns,
	.ns.ops		= &cgroupns_operations,
	.ns.inum	= PROC_CGROUP_INIT_INO,
	.root_cset	= &init_css_set,
};

212
static struct file_system_type cgroup2_fs_type;
213
static struct cftype cgroup_base_files[];
214

215 216
static int cgroup_apply_control(struct cgroup *cgrp);
static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
217 218
static void css_task_iter_skip(struct css_task_iter *it,
			       struct task_struct *task);
219
static int cgroup_destroy_locked(struct cgroup *cgrp);
220 221
static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
					      struct cgroup_subsys *ss);
222
static void css_release(struct percpu_ref *ref);
223
static void kill_css(struct cgroup_subsys_state *css);
224 225
static int cgroup_addrm_files(struct cgroup_subsys_state *css,
			      struct cgroup *cgrp, struct cftype cfts[],
226
			      bool is_add);
227

228 229 230 231 232 233 234 235
/**
 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
 * @ssid: subsys ID of interest
 *
 * cgroup_subsys_enabled() can only be used with literal subsys names which
 * is fine for individual subsystems but unsuitable for cgroup core.  This
 * is slower static_key_enabled() based test indexed by @ssid.
 */
236
bool cgroup_ssid_enabled(int ssid)
237
{
238 239 240
	if (CGROUP_SUBSYS_COUNT == 0)
		return false;

241 242 243
	return static_key_enabled(cgroup_subsys_enabled_key[ssid]);
}

244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
/**
 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
 * @cgrp: the cgroup of interest
 *
 * The default hierarchy is the v2 interface of cgroup and this function
 * can be used to test whether a cgroup is on the default hierarchy for
 * cases where a subsystem should behave differnetly depending on the
 * interface version.
 *
 * The set of behaviors which change on the default hierarchy are still
 * being determined and the mount option is prefixed with __DEVEL__.
 *
 * List of changed behaviors:
 *
 * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
 *   and "name" are disallowed.
 *
 * - When mounting an existing superblock, mount options should match.
 *
 * - Remount is disallowed.
 *
 * - rename(2) is disallowed.
 *
 * - "tasks" is removed.  Everything should be at process granularity.  Use
 *   "cgroup.procs" instead.
 *
 * - "cgroup.procs" is not sorted.  pids will be unique unless they got
 *   recycled inbetween reads.
 *
 * - "release_agent" and "notify_on_release" are removed.  Replacement
 *   notification mechanism will be implemented.
 *
 * - "cgroup.clone_children" is removed.
 *
 * - "cgroup.subtree_populated" is available.  Its value is 0 if the cgroup
 *   and its descendants contain no task; otherwise, 1.  The file also
 *   generates kernfs notification which can be monitored through poll and
 *   [di]notify when the value of the file changes.
 *
 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
 *   take masks of ancestors with non-empty cpus/mems, instead of being
 *   moved to an ancestor.
 *
 * - cpuset: a task can be moved into an empty cpuset, and again it takes
 *   masks of ancestors.
 *
 * - memcg: use_hierarchy is on by default and the cgroup file for the flag
 *   is not created.
 *
 * - blkcg: blk-throttle becomes properly hierarchical.
 *
 * - debug: disallowed on the default hierarchy.
 */
297
bool cgroup_on_dfl(const struct cgroup *cgrp)
298 299 300 301
{
	return cgrp->root == &cgrp_dfl_root;
}

302 303 304 305 306 307 308
/* IDR wrappers which synchronize using cgroup_idr_lock */
static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
			    gfp_t gfp_mask)
{
	int ret;

	idr_preload(gfp_mask);
T
Tejun Heo 已提交
309
	spin_lock_bh(&cgroup_idr_lock);
310
	ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM);
T
Tejun Heo 已提交
311
	spin_unlock_bh(&cgroup_idr_lock);
312 313 314 315 316 317 318 319
	idr_preload_end();
	return ret;
}

static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
{
	void *ret;

T
Tejun Heo 已提交
320
	spin_lock_bh(&cgroup_idr_lock);
321
	ret = idr_replace(idr, ptr, id);
T
Tejun Heo 已提交
322
	spin_unlock_bh(&cgroup_idr_lock);
323 324 325 326 327
	return ret;
}

static void cgroup_idr_remove(struct idr *idr, int id)
{
T
Tejun Heo 已提交
328
	spin_lock_bh(&cgroup_idr_lock);
329
	idr_remove(idr, id);
T
Tejun Heo 已提交
330
	spin_unlock_bh(&cgroup_idr_lock);
331 332
}

333
static bool cgroup_has_tasks(struct cgroup *cgrp)
T
Tejun Heo 已提交
334
{
335 336
	return cgrp->nr_populated_csets;
}
T
Tejun Heo 已提交
337

338
bool cgroup_is_threaded(struct cgroup *cgrp)
339 340 341 342
{
	return cgrp->dom_cgrp != cgrp;
}

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
/* can @cgrp host both domain and threaded children? */
static bool cgroup_is_mixable(struct cgroup *cgrp)
{
	/*
	 * Root isn't under domain level resource control exempting it from
	 * the no-internal-process constraint, so it can serve as a thread
	 * root and a parent of resource domains at the same time.
	 */
	return !cgroup_parent(cgrp);
}

/* can @cgrp become a thread root? should always be true for a thread root */
static bool cgroup_can_be_thread_root(struct cgroup *cgrp)
{
	/* mixables don't care */
	if (cgroup_is_mixable(cgrp))
		return true;

	/* domain roots can't be nested under threaded */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* can only have either domain or threaded children */
	if (cgrp->nr_populated_domain_children)
		return false;

	/* and no domain controllers can be enabled */
	if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)
		return false;

	return true;
}

/* is @cgrp root of a threaded subtree? */
377
bool cgroup_is_thread_root(struct cgroup *cgrp)
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
{
	/* thread root should be a domain */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* a domain w/ threaded children is a thread root */
	if (cgrp->nr_threaded_children)
		return true;

	/*
	 * A domain which has tasks and explicit threaded controllers
	 * enabled is a thread root.
	 */
	if (cgroup_has_tasks(cgrp) &&
	    (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask))
		return true;

	return false;
}

/* a domain which isn't connected to the root w/o brekage can't be used */
static bool cgroup_is_valid_domain(struct cgroup *cgrp)
{
	/* the cgroup itself can be a thread root */
	if (cgroup_is_threaded(cgrp))
		return false;

	/* but the ancestors can't be unless mixable */
	while ((cgrp = cgroup_parent(cgrp))) {
		if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp))
			return false;
		if (cgroup_is_threaded(cgrp))
			return false;
	}

	return true;
T
Tejun Heo 已提交
414 415
}

416 417 418 419 420 421
/* subsystems visibly enabled on a cgroup */
static u16 cgroup_control(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);
	u16 root_ss_mask = cgrp->root->subsys_mask;

422 423 424 425 426 427 428 429
	if (parent) {
		u16 ss_mask = parent->subtree_control;

		/* threaded cgroups can only have threaded controllers */
		if (cgroup_is_threaded(cgrp))
			ss_mask &= cgrp_dfl_threaded_ss_mask;
		return ss_mask;
	}
430 431

	if (cgroup_on_dfl(cgrp))
432 433
		root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask |
				  cgrp_dfl_implicit_ss_mask);
434 435 436 437 438 439 440 441
	return root_ss_mask;
}

/* subsystems enabled on a cgroup */
static u16 cgroup_ss_mask(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);

442 443 444 445 446 447 448 449
	if (parent) {
		u16 ss_mask = parent->subtree_ss_mask;

		/* threaded cgroups can only have threaded controllers */
		if (cgroup_is_threaded(cgrp))
			ss_mask &= cgrp_dfl_threaded_ss_mask;
		return ss_mask;
	}
450 451 452 453

	return cgrp->root->subsys_mask;
}

T
Tejun Heo 已提交
454 455 456
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
457
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
T
Tejun Heo 已提交
458
 *
459 460 461 462 463
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
464 465
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
466
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
467
{
468
	if (ss)
469
		return rcu_dereference_check(cgrp->subsys[ss->id],
T
Tejun Heo 已提交
470
					lockdep_is_held(&cgroup_mutex));
471
	else
472
		return &cgrp->self;
T
Tejun Heo 已提交
473
}
474

475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
/**
 * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
 * @ss: the subsystem of interest
 *
 * Find and get @cgrp's css assocaited with @ss.  If the css doesn't exist
 * or is offline, %NULL is returned.
 */
static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp,
						     struct cgroup_subsys *ss)
{
	struct cgroup_subsys_state *css;

	rcu_read_lock();
	css = cgroup_css(cgrp, ss);
	if (!css || !css_tryget_online(css))
		css = NULL;
	rcu_read_unlock();

	return css;
}

497 498 499
/**
 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
500
 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
501
 *
C
Chen Hanxiao 已提交
502
 * Similar to cgroup_css() but returns the effective css, which is defined
503 504 505 506 507 508 509 510 511 512
 * as the matching css of the nearest ancestor including self which has @ss
 * enabled.  If @ss is associated with the hierarchy @cgrp is on, this
 * function is guaranteed to return non-NULL css.
 */
static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
						struct cgroup_subsys *ss)
{
	lockdep_assert_held(&cgroup_mutex);

	if (!ss)
513
		return &cgrp->self;
514

T
Tejun Heo 已提交
515 516
	/*
	 * This function is used while updating css associations and thus
517
	 * can't test the csses directly.  Test ss_mask.
T
Tejun Heo 已提交
518
	 */
519
	while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) {
T
Tejun Heo 已提交
520
		cgrp = cgroup_parent(cgrp);
521 522 523
		if (!cgrp)
			return NULL;
	}
524 525

	return cgroup_css(cgrp, ss);
T
Tejun Heo 已提交
526
}
527

T
Tejun Heo 已提交
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
/**
 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
 * @cgrp: the cgroup of interest
 * @ss: the subsystem of interest
 *
 * Find and get the effective css of @cgrp for @ss.  The effective css is
 * defined as the matching css of the nearest ancestor including self which
 * has @ss enabled.  If @ss is not mounted on the hierarchy @cgrp is on,
 * the root css is returned, so this function always returns a valid css.
 * The returned css must be put using css_put().
 */
struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp,
					     struct cgroup_subsys *ss)
{
	struct cgroup_subsys_state *css;

	rcu_read_lock();

	do {
		css = cgroup_css(cgrp, ss);

		if (css && css_tryget_online(css))
			goto out_unlock;
		cgrp = cgroup_parent(cgrp);
	} while (cgrp);

	css = init_css_set.subsys[ss->id];
	css_get(css);
out_unlock:
	rcu_read_unlock();
	return css;
}

561
static void cgroup_get_live(struct cgroup *cgrp)
T
Tejun Heo 已提交
562 563 564 565 566
{
	WARN_ON_ONCE(cgroup_is_dead(cgrp));
	css_get(&cgrp->self);
}

T
Tejun Heo 已提交
567
struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
568
{
T
Tejun Heo 已提交
569
	struct cgroup *cgrp = of->kn->parent->priv;
T
Tejun Heo 已提交
570
	struct cftype *cft = of_cft(of);
T
Tejun Heo 已提交
571 572 573 574 575 576 577 578 579 580 581 582

	/*
	 * This is open and unprotected implementation of cgroup_css().
	 * seq_css() is only called from a kernfs file operation which has
	 * an active reference on the file.  Because all the subsystem
	 * files are drained before a css is disassociated with a cgroup,
	 * the matching css from the cgroup's subsys table is guaranteed to
	 * be and stay valid until the enclosing operation is complete.
	 */
	if (cft->ss)
		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
	else
583
		return &cgrp->self;
584
}
T
Tejun Heo 已提交
585
EXPORT_SYMBOL_GPL(of_css);
586

T
Tejun Heo 已提交
587 588 589 590 591 592
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
593
 * Should be called under cgroup_[tree_]mutex.
T
Tejun Heo 已提交
594 595 596 597 598 599 600 601
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

602 603 604 605 606 607 608 609 610 611 612 613 614 615
/**
 * for_each_e_css - iterate all effective css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_[tree_]mutex.
 */
#define for_each_e_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
			;						\
		else

616
/**
617
 * do_each_subsys_mask - filter for_each_subsys with a bitmask
618 619
 * @ss: the iteration cursor
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
620
 * @ss_mask: the bitmask
621 622
 *
 * The block will only run for cases where the ssid-th bit (1 << ssid) of
623
 * @ss_mask is set.
624
 */
625 626 627
#define do_each_subsys_mask(ss, ssid, ss_mask) do {			\
	unsigned long __ss_mask = (ss_mask);				\
	if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */	\
628
		(ssid) = 0;						\
629 630 631 632 633 634 635 636 637 638
		break;							\
	}								\
	for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) {	\
		(ss) = cgroup_subsys[ssid];				\
		{

#define while_each_subsys_mask()					\
		}							\
	}								\
} while (false)
639

640 641
/* iterate over child cgrps, lock should be held throughout iteration */
#define cgroup_for_each_live_child(child, cgrp)				\
642
	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
T
Tejun Heo 已提交
643
		if (({ lockdep_assert_held(&cgroup_mutex);		\
644 645 646
		       cgroup_is_dead(child); }))			\
			;						\
		else
647

648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
/* walk live descendants in preorder */
#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp)		\
	css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL))	\
		if (({ lockdep_assert_held(&cgroup_mutex);		\
		       (dsct) = (d_css)->cgroup;			\
		       cgroup_is_dead(dsct); }))			\
			;						\
		else

/* walk live descendants in postorder */
#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp)		\
	css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL))	\
		if (({ lockdep_assert_held(&cgroup_mutex);		\
		       (dsct) = (d_css)->cgroup;			\
		       cgroup_is_dead(dsct); }))			\
			;						\
		else

666 667
/*
 * The default css_set - used by init and its children prior to any
668 669 670 671 672
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */
T
Tejun Heo 已提交
673
struct css_set init_css_set = {
674
	.refcount		= REFCOUNT_INIT(1),
675
	.dom_cset		= &init_css_set,
676 677
	.tasks			= LIST_HEAD_INIT(init_css_set.tasks),
	.mg_tasks		= LIST_HEAD_INIT(init_css_set.mg_tasks),
678
	.dying_tasks		= LIST_HEAD_INIT(init_css_set.dying_tasks),
T
Tejun Heo 已提交
679
	.task_iters		= LIST_HEAD_INIT(init_css_set.task_iters),
680
	.threaded_csets		= LIST_HEAD_INIT(init_css_set.threaded_csets),
T
Tejun Heo 已提交
681
	.cgrp_links		= LIST_HEAD_INIT(init_css_set.cgrp_links),
682 683
	.mg_preload_node	= LIST_HEAD_INIT(init_css_set.mg_preload_node),
	.mg_node		= LIST_HEAD_INIT(init_css_set.mg_node),
684 685 686 687 688 689 690 691

	/*
	 * The following field is re-initialized when this cset gets linked
	 * in cgroup_init().  However, let's initialize the field
	 * statically too so that the default cgroup can be accessed safely
	 * early during boot.
	 */
	.dfl_cgrp		= &cgrp_dfl_root.cgrp,
692
};
693

694
static int css_set_count	= 1;	/* 1 for init_css_set */
695

696 697 698 699 700
static bool css_set_threaded(struct css_set *cset)
{
	return cset->dom_cset != cset;
}

701 702 703
/**
 * css_set_populated - does a css_set contain any tasks?
 * @cset: target css_set
704 705 706 707 708
 *
 * css_set_populated() should be the same as !!cset->nr_tasks at steady
 * state. However, css_set_populated() can be called while a task is being
 * added to or removed from the linked list before the nr_tasks is
 * properly updated. Hence, we can't just look at ->nr_tasks here.
709 710 711
 */
static bool css_set_populated(struct css_set *cset)
{
712
	lockdep_assert_held(&css_set_lock);
713 714 715 716

	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);
}

717
/**
718
 * cgroup_update_populated - update the populated count of a cgroup
719 720 721
 * @cgrp: the target cgroup
 * @populated: inc or dec populated count
 *
722
 * One of the css_sets associated with @cgrp is either getting its first
723 724 725 726
 * task or losing the last.  Update @cgrp->nr_populated_* accordingly.  The
 * count is propagated towards root so that a given cgroup's
 * nr_populated_children is zero iff none of its descendants contain any
 * tasks.
727
 *
728 729 730 731 732
 * @cgrp's interface file "cgroup.populated" is zero if both
 * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
 * 1 otherwise.  When the sum changes from or to zero, userland is notified
 * that the content of the interface file has changed.  This can be used to
 * detect when @cgrp and its descendants become populated or empty.
733 734 735
 */
static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
{
736 737 738
	struct cgroup *child = NULL;
	int adj = populated ? 1 : -1;

739
	lockdep_assert_held(&css_set_lock);
740 741

	do {
742
		bool was_populated = cgroup_is_populated(cgrp);
743

744
		if (!child) {
745
			cgrp->nr_populated_csets += adj;
746 747 748 749 750 751
		} else {
			if (cgroup_is_threaded(child))
				cgrp->nr_populated_threaded_children += adj;
			else
				cgrp->nr_populated_domain_children += adj;
		}
752

753
		if (was_populated == cgroup_is_populated(cgrp))
754 755
			break;

756
		cgroup1_check_for_release(cgrp);
757 758
		cgroup_file_notify(&cgrp->events_file);

759
		child = cgrp;
T
Tejun Heo 已提交
760
		cgrp = cgroup_parent(cgrp);
761 762 763
	} while (cgrp);
}

764 765 766 767 768 769
/**
 * css_set_update_populated - update populated state of a css_set
 * @cset: target css_set
 * @populated: whether @cset is populated or depopulated
 *
 * @cset is either getting the first task or losing the last.  Update the
770
 * populated counters of all associated cgroups accordingly.
771 772 773 774 775
 */
static void css_set_update_populated(struct css_set *cset, bool populated)
{
	struct cgrp_cset_link *link;

776
	lockdep_assert_held(&css_set_lock);
777 778 779 780 781

	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
		cgroup_update_populated(link->cgrp, populated);
}

782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
/*
 * @task is leaving, advance task iterators which are pointing to it so
 * that they can resume at the next position.  Advancing an iterator might
 * remove it from the list, use safe walk.  See css_task_iter_skip() for
 * details.
 */
static void css_set_skip_task_iters(struct css_set *cset,
				    struct task_struct *task)
{
	struct css_task_iter *it, *pos;

	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)
		css_task_iter_skip(it, task);
}

T
Tejun Heo 已提交
797 798 799 800 801 802 803 804 805 806 807
/**
 * css_set_move_task - move a task from one css_set to another
 * @task: task being moved
 * @from_cset: css_set @task currently belongs to (may be NULL)
 * @to_cset: new css_set @task is being moved to (may be NULL)
 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
 *
 * Move @task from @from_cset to @to_cset.  If @task didn't belong to any
 * css_set, @from_cset can be NULL.  If @task is being disassociated
 * instead of moved, @to_cset can be NULL.
 *
808
 * This function automatically handles populated counter updates and
809 810
 * css_task_iter adjustments but the caller is responsible for managing
 * @from_cset and @to_cset's reference counts.
T
Tejun Heo 已提交
811 812 813 814 815
 */
static void css_set_move_task(struct task_struct *task,
			      struct css_set *from_cset, struct css_set *to_cset,
			      bool use_mg_tasks)
{
816
	lockdep_assert_held(&css_set_lock);
T
Tejun Heo 已提交
817

818 819 820
	if (to_cset && !css_set_populated(to_cset))
		css_set_update_populated(to_cset, true);

T
Tejun Heo 已提交
821 822
	if (from_cset) {
		WARN_ON_ONCE(list_empty(&task->cg_list));
823

824
		css_set_skip_task_iters(from_cset, task);
T
Tejun Heo 已提交
825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
		list_del_init(&task->cg_list);
		if (!css_set_populated(from_cset))
			css_set_update_populated(from_cset, false);
	} else {
		WARN_ON_ONCE(!list_empty(&task->cg_list));
	}

	if (to_cset) {
		/*
		 * We are synchronized through cgroup_threadgroup_rwsem
		 * against PF_EXITING setting such that we can't race
		 * against cgroup_exit() changing the css_set to
		 * init_css_set and dropping the old one.
		 */
		WARN_ON_ONCE(task->flags & PF_EXITING);

J
Johannes Weiner 已提交
841
		cgroup_move_task(task, to_cset);
T
Tejun Heo 已提交
842 843 844 845 846
		list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :
							     &to_cset->tasks);
	}
}

847 848 849 850 851
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
852
#define CSS_SET_HASH_BITS	7
853
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
854

855
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
856
{
857
	unsigned long key = 0UL;
858 859
	struct cgroup_subsys *ss;
	int i;
860

861
	for_each_subsys(ss, i)
862 863
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
864

865
	return key;
866 867
}

868
void put_css_set_locked(struct css_set *cset)
869
{
870
	struct cgrp_cset_link *link, *tmp_link;
T
Tejun Heo 已提交
871 872
	struct cgroup_subsys *ss;
	int ssid;
873

874
	lockdep_assert_held(&css_set_lock);
875

876
	if (!refcount_dec_and_test(&cset->refcount))
877
		return;
878

879 880
	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));

881 882
	/* This css_set is dead. unlink it and release cgroup and css refs */
	for_each_subsys(ss, ssid) {
T
Tejun Heo 已提交
883
		list_del(&cset->e_cset_node[ssid]);
884 885
		css_put(cset->subsys[ssid]);
	}
886
	hash_del(&cset->hlist);
887 888
	css_set_count--;

889 890 891
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
892 893
		if (cgroup_parent(link->cgrp))
			cgroup_put(link->cgrp);
894
		kfree(link);
895
	}
896

897 898 899 900 901
	if (css_set_threaded(cset)) {
		list_del(&cset->threaded_csets_node);
		put_css_set_locked(cset->dom_cset);
	}

902
	kfree_rcu(cset, rcu_head);
903 904
}

905
/**
906
 * compare_css_sets - helper function for find_existing_css_set().
907 908
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
909 910 911
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
912
 * Returns true if "cset" matches "old_cset" except for the hierarchy
913 914
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
915 916
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
917 918 919
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
920
	struct cgroup *new_dfl_cgrp;
921 922
	struct list_head *l1, *l2;

923 924 925 926 927 928
	/*
	 * On the default hierarchy, there can be csets which are
	 * associated with the same set of cgroups but different csses.
	 * Let's first ensure that csses match.
	 */
	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
929 930
		return false;

931 932 933 934 935 936 937 938 939 940

	/* @cset's domain should match the default cgroup's */
	if (cgroup_on_dfl(new_cgrp))
		new_dfl_cgrp = new_cgrp;
	else
		new_dfl_cgrp = old_cset->dfl_cgrp;

	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)
		return false;

941 942
	/*
	 * Compare cgroup pointers in order to distinguish between
943 944 945
	 * different cgroups in hierarchies.  As different cgroups may
	 * share the same effective css, this comparison is always
	 * necessary.
946
	 */
947 948
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
949
	while (1) {
950
		struct cgrp_cset_link *link1, *link2;
951
		struct cgroup *cgrp1, *cgrp2;
952 953 954 955

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
956 957
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
958 959
			break;
		} else {
960
			BUG_ON(l2 == &old_cset->cgrp_links);
961 962
		}
		/* Locate the cgroups associated with these links. */
963 964 965 966
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
967
		/* Hierarchies should be linked in the same order. */
968
		BUG_ON(cgrp1->root != cgrp2->root);
969 970 971 972 973 974 975 976

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
977 978
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
979 980
				return false;
		} else {
981
			if (cgrp1 != cgrp2)
982 983 984 985 986 987
				return false;
		}
	}
	return true;
}

988 989 990 991 992
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
993
 */
994 995 996
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
997
{
998
	struct cgroup_root *root = cgrp->root;
999
	struct cgroup_subsys *ss;
1000
	struct css_set *cset;
1001
	unsigned long key;
1002
	int i;
1003

B
Ben Blum 已提交
1004 1005 1006 1007 1008
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
1009
	for_each_subsys(ss, i) {
1010
		if (root->subsys_mask & (1UL << i)) {
1011 1012 1013 1014 1015
			/*
			 * @ss is in this hierarchy, so we want the
			 * effective css from @cgrp.
			 */
			template[i] = cgroup_e_css(cgrp, ss);
1016
		} else {
1017 1018 1019 1020
			/*
			 * @ss is not in this hierarchy, so we don't want
			 * to change the css.
			 */
1021
			template[i] = old_cset->subsys[i];
1022 1023 1024
		}
	}

1025
	key = css_set_hash(template);
1026 1027
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
1028 1029 1030
			continue;

		/* This css_set matches what we need */
1031
		return cset;
1032
	}
1033 1034 1035 1036 1037

	/* No existing cgroup group matched */
	return NULL;
}

1038
static void free_cgrp_cset_links(struct list_head *links_to_free)
1039
{
1040
	struct cgrp_cset_link *link, *tmp_link;
1041

1042 1043
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
1044 1045 1046 1047
		kfree(link);
	}
}

1048 1049 1050 1051 1052 1053 1054
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
1055
 */
1056
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
1057
{
1058
	struct cgrp_cset_link *link;
1059
	int i;
1060 1061 1062

	INIT_LIST_HEAD(tmp_links);

1063
	for (i = 0; i < count; i++) {
1064
		link = kzalloc(sizeof(*link), GFP_KERNEL);
1065
		if (!link) {
1066
			free_cgrp_cset_links(tmp_links);
1067 1068
			return -ENOMEM;
		}
1069
		list_add(&link->cset_link, tmp_links);
1070 1071 1072 1073
	}
	return 0;
}

1074 1075
/**
 * link_css_set - a helper function to link a css_set to a cgroup
1076
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
1077
 * @cset: the css_set to be linked
1078 1079
 * @cgrp: the destination cgroup
 */
1080 1081
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
1082
{
1083
	struct cgrp_cset_link *link;
1084

1085
	BUG_ON(list_empty(tmp_links));
T
Tejun Heo 已提交
1086 1087 1088 1089

	if (cgroup_on_dfl(cgrp))
		cset->dfl_cgrp = cgrp;

1090 1091
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
1092
	link->cgrp = cgrp;
1093

1094
	/*
1095 1096
	 * Always add links to the tail of the lists so that the lists are
	 * in choronological order.
1097
	 */
1098
	list_move_tail(&link->cset_link, &cgrp->cset_links);
1099
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
1100 1101

	if (cgroup_parent(cgrp))
1102
		cgroup_get_live(cgrp);
1103 1104
}

1105 1106 1107 1108 1109 1110 1111
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
1112
 */
1113 1114
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
1115
{
1116
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
1117
	struct css_set *cset;
1118 1119
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
1120
	struct cgroup_subsys *ss;
1121
	unsigned long key;
T
Tejun Heo 已提交
1122
	int ssid;
1123

1124 1125
	lockdep_assert_held(&cgroup_mutex);

1126 1127
	/* First see if we already have a cgroup group that matches
	 * the desired set */
1128
	spin_lock_irq(&css_set_lock);
1129 1130 1131
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
1132
	spin_unlock_irq(&css_set_lock);
1133

1134 1135
	if (cset)
		return cset;
1136

1137
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
1138
	if (!cset)
1139 1140
		return NULL;

1141
	/* Allocate all the cgrp_cset_link objects that we'll need */
1142
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
1143
		kfree(cset);
1144 1145 1146
		return NULL;
	}

1147
	refcount_set(&cset->refcount, 1);
1148
	cset->dom_cset = cset;
1149
	INIT_LIST_HEAD(&cset->tasks);
T
Tejun Heo 已提交
1150
	INIT_LIST_HEAD(&cset->mg_tasks);
1151
	INIT_LIST_HEAD(&cset->dying_tasks);
1152
	INIT_LIST_HEAD(&cset->task_iters);
1153
	INIT_LIST_HEAD(&cset->threaded_csets);
1154
	INIT_HLIST_NODE(&cset->hlist);
T
Tejun Heo 已提交
1155 1156 1157
	INIT_LIST_HEAD(&cset->cgrp_links);
	INIT_LIST_HEAD(&cset->mg_preload_node);
	INIT_LIST_HEAD(&cset->mg_node);
1158 1159 1160

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
1161
	memcpy(cset->subsys, template, sizeof(cset->subsys));
1162

1163
	spin_lock_irq(&css_set_lock);
1164
	/* Add reference counts and links from the new css_set. */
1165
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
1166
		struct cgroup *c = link->cgrp;
1167

1168 1169
		if (c->root == cgrp->root)
			c = cgrp;
1170
		link_css_set(&tmp_links, cset, c);
1171
	}
1172

1173
	BUG_ON(!list_empty(&tmp_links));
1174 1175

	css_set_count++;
1176

T
Tejun Heo 已提交
1177
	/* Add @cset to the hash table */
1178 1179
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
1180

1181 1182 1183
	for_each_subsys(ss, ssid) {
		struct cgroup_subsys_state *css = cset->subsys[ssid];

T
Tejun Heo 已提交
1184
		list_add_tail(&cset->e_cset_node[ssid],
1185 1186 1187
			      &css->cgroup->e_csets[ssid]);
		css_get(css);
	}
T
Tejun Heo 已提交
1188

1189
	spin_unlock_irq(&css_set_lock);
1190

1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
	/*
	 * If @cset should be threaded, look up the matching dom_cset and
	 * link them up.  We first fully initialize @cset then look for the
	 * dom_cset.  It's simpler this way and safe as @cset is guaranteed
	 * to stay empty until we return.
	 */
	if (cgroup_is_threaded(cset->dfl_cgrp)) {
		struct css_set *dcset;

		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);
		if (!dcset) {
			put_css_set(cset);
			return NULL;
		}

		spin_lock_irq(&css_set_lock);
		cset->dom_cset = dcset;
		list_add_tail(&cset->threaded_csets_node,
			      &dcset->threaded_csets);
		spin_unlock_irq(&css_set_lock);
	}

1213
	return cset;
1214 1215
}

1216
struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
1217
{
1218
	struct cgroup *root_cgrp = kf_root->kn->priv;
T
Tejun Heo 已提交
1219

1220
	return root_cgrp->root;
T
Tejun Heo 已提交
1221 1222
}

1223
static int cgroup_init_root_id(struct cgroup_root *root)
1224 1225 1226 1227 1228
{
	int id;

	lockdep_assert_held(&cgroup_mutex);

1229
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
1230 1231 1232 1233 1234 1235 1236
	if (id < 0)
		return id;

	root->hierarchy_id = id;
	return 0;
}

1237
static void cgroup_exit_root_id(struct cgroup_root *root)
1238 1239 1240
{
	lockdep_assert_held(&cgroup_mutex);

1241
	idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1242 1243
}

1244
void cgroup_free_root(struct cgroup_root *root)
1245 1246 1247 1248 1249 1250 1251
{
	if (root) {
		idr_destroy(&root->cgroup_idr);
		kfree(root);
	}
}

1252
static void cgroup_destroy_root(struct cgroup_root *root)
1253
{
1254
	struct cgroup *cgrp = &root->cgrp;
1255 1256
	struct cgrp_cset_link *link, *tmp_link;

1257 1258
	trace_cgroup_destroy_root(root);

1259
	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
1260

T
Tejun Heo 已提交
1261
	BUG_ON(atomic_read(&root->nr_cgrps));
1262
	BUG_ON(!list_empty(&cgrp->self.children));
1263 1264

	/* Rebind all subsystems back to the default hierarchy */
1265
	WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask));
1266 1267

	/*
1268 1269
	 * Release all the links from cset_links to this hierarchy's
	 * root cgroup
1270
	 */
1271
	spin_lock_irq(&css_set_lock);
1272 1273 1274 1275 1276 1277

	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
		kfree(link);
	}
1278

1279
	spin_unlock_irq(&css_set_lock);
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289

	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
		cgroup_root_count--;
	}

	cgroup_exit_root_id(root);

	mutex_unlock(&cgroup_mutex);

T
Tejun Heo 已提交
1290
	kernfs_destroy_root(root->kf_root);
1291 1292 1293
	cgroup_free_root(root);
}

1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
/*
 * look up cgroup associated with current task's cgroup namespace on the
 * specified hierarchy
 */
static struct cgroup *
current_cgns_cgroup_from_root(struct cgroup_root *root)
{
	struct cgroup *res = NULL;
	struct css_set *cset;

	lockdep_assert_held(&css_set_lock);

	rcu_read_lock();

	cset = current->nsproxy->cgroup_ns->root_cset;
	if (cset == &init_css_set) {
		res = &root->cgrp;
	} else {
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
			struct cgroup *c = link->cgrp;

			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
	rcu_read_unlock();

	BUG_ON(!res);
	return res;
}

1329 1330
/* look up cgroup associated with given css_set on the specified hierarchy */
static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
1331
					    struct cgroup_root *root)
1332 1333 1334
{
	struct cgroup *res = NULL;

1335
	lockdep_assert_held(&cgroup_mutex);
1336
	lockdep_assert_held(&css_set_lock);
1337

1338
	if (cset == &init_css_set) {
1339
		res = &root->cgrp;
1340 1341
	} else if (root == &cgrp_dfl_root) {
		res = cset->dfl_cgrp;
1342
	} else {
1343 1344 1345
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
1346
			struct cgroup *c = link->cgrp;
1347

1348 1349 1350 1351 1352 1353
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
1354

1355 1356 1357 1358
	BUG_ON(!res);
	return res;
}

1359
/*
1360
 * Return the cgroup for "task" from the given hierarchy. Must be
1361
 * called with cgroup_mutex and css_set_lock held.
1362
 */
1363 1364
struct cgroup *task_cgroup_from_root(struct task_struct *task,
				     struct cgroup_root *root)
1365 1366 1367 1368 1369 1370 1371 1372 1373
{
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
	return cset_cgroup_from_root(task_css_set(task), root);
}

1374 1375 1376 1377 1378 1379
/*
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
1380
 * cgroup_attach_task() can increment it again.  Because a count of zero
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
1392
 * least one task in the system (init, pid == 1), therefore, root cgroup
1393
 * always has either children cgroups and/or using tasks.  So we don't
1394
 * need a special hack to ensure that root cgroup cannot be deleted.
1395 1396
 *
 * P.S.  One more locking exception.  RCU is used to guard the
1397
 * update of a tasks cgroup pointer by cgroup_attach_task()
1398 1399
 */

T
Tejun Heo 已提交
1400
static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
1401

T
Tejun Heo 已提交
1402 1403
static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
			      char *buf)
1404
{
1405 1406
	struct cgroup_subsys *ss = cft->ss;

T
Tejun Heo 已提交
1407 1408 1409
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
		snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
1410 1411
			 cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,
			 cft->name);
T
Tejun Heo 已提交
1412
	else
1413
		strscpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
T
Tejun Heo 已提交
1414
	return buf;
1415 1416
}

1417 1418 1419 1420
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
1421
 * S_IRUGO for read, S_IWUSR for write.
1422 1423
 */
static umode_t cgroup_file_mode(const struct cftype *cft)
1424
{
1425
	umode_t mode = 0;
1426

1427 1428 1429
	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
		mode |= S_IRUGO;

1430 1431 1432 1433 1434 1435
	if (cft->write_u64 || cft->write_s64 || cft->write) {
		if (cft->flags & CFTYPE_WORLD_WRITABLE)
			mode |= S_IWUGO;
		else
			mode |= S_IWUSR;
	}
1436 1437

	return mode;
1438 1439
}

1440
/**
1441
 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1442
 * @subtree_control: the new subtree_control mask to consider
1443
 * @this_ss_mask: available subsystems
1444 1445 1446 1447 1448
 *
 * On the default hierarchy, a subsystem may request other subsystems to be
 * enabled together through its ->depends_on mask.  In such cases, more
 * subsystems than specified in "cgroup.subtree_control" may be enabled.
 *
1449
 * This function calculates which subsystems need to be enabled if
1450
 * @subtree_control is to be applied while restricted to @this_ss_mask.
1451
 */
1452
static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask)
1453
{
1454
	u16 cur_ss_mask = subtree_control;
1455 1456 1457 1458 1459
	struct cgroup_subsys *ss;
	int ssid;

	lockdep_assert_held(&cgroup_mutex);

1460 1461
	cur_ss_mask |= cgrp_dfl_implicit_ss_mask;

1462
	while (true) {
1463
		u16 new_ss_mask = cur_ss_mask;
1464

1465
		do_each_subsys_mask(ss, ssid, cur_ss_mask) {
1466
			new_ss_mask |= ss->depends_on;
1467
		} while_each_subsys_mask();
1468 1469 1470 1471 1472 1473

		/*
		 * Mask out subsystems which aren't available.  This can
		 * happen only if some depended-upon subsystems were bound
		 * to non-default hierarchies.
		 */
1474
		new_ss_mask &= this_ss_mask;
1475 1476 1477 1478 1479 1480

		if (new_ss_mask == cur_ss_mask)
			break;
		cur_ss_mask = new_ss_mask;
	}

1481 1482 1483
	return cur_ss_mask;
}

1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
/**
 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
 *
 * This helper undoes cgroup_kn_lock_live() and should be invoked before
 * the method finishes if locking succeeded.  Note that once this function
 * returns the cgroup returned by cgroup_kn_lock_live() may become
 * inaccessible any time.  If the caller intends to continue to access the
 * cgroup, it should pin it before invoking this function.
 */
1494
void cgroup_kn_unlock(struct kernfs_node *kn)
1495
{
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;

	mutex_unlock(&cgroup_mutex);

	kernfs_unbreak_active_protection(kn);
	cgroup_put(cgrp);
1507 1508
}

1509 1510 1511
/**
 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
 * @kn: the kernfs_node being serviced
1512
 * @drain_offline: perform offline draining on the cgroup
1513 1514 1515 1516 1517
 *
 * This helper is to be used by a cgroup kernfs method currently servicing
 * @kn.  It breaks the active protection, performs cgroup locking and
 * verifies that the associated cgroup is alive.  Returns the cgroup if
 * alive; otherwise, %NULL.  A successful return should be undone by a
1518 1519
 * matching cgroup_kn_unlock() invocation.  If @drain_offline is %true, the
 * cgroup is drained of offlining csses before return.
1520 1521 1522 1523 1524 1525
 *
 * Any cgroup kernfs method implementation which requires locking the
 * associated cgroup should use this helper.  It avoids nesting cgroup
 * locking under kernfs active protection and allows all kernfs operations
 * including self-removal.
 */
1526
struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline)
T
Tejun Heo 已提交
1527
{
1528 1529 1530 1531 1532 1533
	struct cgroup *cgrp;

	if (kernfs_type(kn) == KERNFS_DIR)
		cgrp = kn->priv;
	else
		cgrp = kn->parent->priv;
T
Tejun Heo 已提交
1534

1535
	/*
1536
	 * We're gonna grab cgroup_mutex which nests outside kernfs
1537 1538 1539
	 * active_ref.  cgroup liveliness check alone provides enough
	 * protection against removal.  Ensure @cgrp stays accessible and
	 * break the active_ref protection.
1540
	 */
1541 1542
	if (!cgroup_tryget(cgrp))
		return NULL;
1543 1544
	kernfs_break_active_protection(kn);

1545 1546 1547 1548
	if (drain_offline)
		cgroup_lock_and_drain_offline(cgrp);
	else
		mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1549

1550 1551 1552 1553 1554
	if (!cgroup_is_dead(cgrp))
		return cgrp;

	cgroup_kn_unlock(kn);
	return NULL;
1555
}
T
Tejun Heo 已提交
1556

1557
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
1558
{
T
Tejun Heo 已提交
1559
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
1560

1561
	lockdep_assert_held(&cgroup_mutex);
1562 1563 1564 1565 1566 1567 1568 1569

	if (cft->file_offset) {
		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
		struct cgroup_file *cfile = (void *)css + cft->file_offset;

		spin_lock_irq(&cgroup_file_kn_lock);
		cfile->kn = NULL;
		spin_unlock_irq(&cgroup_file_kn_lock);
1570 1571

		del_timer_sync(&cfile->notify_timer);
1572 1573
	}

T
Tejun Heo 已提交
1574
	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
T
Tejun Heo 已提交
1575 1576
}

1577
/**
1578 1579
 * css_clear_dir - remove subsys files in a cgroup directory
 * @css: taget css
1580
 */
1581
static void css_clear_dir(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
1582
{
1583
	struct cgroup *cgrp = css->cgroup;
1584
	struct cftype *cfts;
T
Tejun Heo 已提交
1585

1586 1587 1588 1589 1590
	if (!(css->flags & CSS_VISIBLE))
		return;

	css->flags &= ~CSS_VISIBLE;

1591 1592 1593 1594 1595 1596
	if (!css->ss) {
		if (cgroup_on_dfl(cgrp))
			cfts = cgroup_base_files;
		else
			cfts = cgroup1_base_files;

1597
		cgroup_addrm_files(css, cgrp, cfts, false);
1598 1599 1600 1601
	} else {
		list_for_each_entry(cfts, &css->ss->cfts, node)
			cgroup_addrm_files(css, cgrp, cfts, false);
	}
1602 1603
}

1604
/**
1605 1606
 * css_populate_dir - create subsys files in a cgroup directory
 * @css: target css
1607 1608 1609
 *
 * On failure, no file is added.
 */
1610
static int css_populate_dir(struct cgroup_subsys_state *css)
1611
{
1612
	struct cgroup *cgrp = css->cgroup;
1613 1614
	struct cftype *cfts, *failed_cfts;
	int ret;
1615

1616
	if ((css->flags & CSS_VISIBLE) || !cgrp->kn)
1617 1618
		return 0;

1619 1620
	if (!css->ss) {
		if (cgroup_on_dfl(cgrp))
1621
			cfts = cgroup_base_files;
1622
		else
1623
			cfts = cgroup1_base_files;
1624

1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
		ret = cgroup_addrm_files(&cgrp->self, cgrp, cfts, true);
		if (ret < 0)
			return ret;
	} else {
		list_for_each_entry(cfts, &css->ss->cfts, node) {
			ret = cgroup_addrm_files(css, cgrp, cfts, true);
			if (ret < 0) {
				failed_cfts = cfts;
				goto err;
			}
1635 1636
		}
	}
1637 1638 1639

	css->flags |= CSS_VISIBLE;

1640 1641
	return 0;
err:
1642 1643 1644 1645 1646
	list_for_each_entry(cfts, &css->ss->cfts, node) {
		if (cfts == failed_cfts)
			break;
		cgroup_addrm_files(css, cgrp, cfts, false);
	}
1647 1648 1649
	return ret;
}

1650
int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
1651
{
1652
	struct cgroup *dcgrp = &dst_root->cgrp;
1653
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1654
	int ssid, i, ret;
1655

T
Tejun Heo 已提交
1656
	lockdep_assert_held(&cgroup_mutex);
1657

1658
	do_each_subsys_mask(ss, ssid, ss_mask) {
1659 1660 1661 1662 1663 1664 1665
		/*
		 * If @ss has non-root csses attached to it, can't move.
		 * If @ss is an implicit controller, it is exempt from this
		 * rule and can be stolen.
		 */
		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) &&
		    !ss->implicit_on_dfl)
T
Tejun Heo 已提交
1666
			return -EBUSY;
1667

1668
		/* can't move between two non-dummy roots either */
1669
		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
1670
			return -EBUSY;
1671
	} while_each_subsys_mask();
1672

1673
	do_each_subsys_mask(ss, ssid, ss_mask) {
1674 1675 1676
		struct cgroup_root *src_root = ss->root;
		struct cgroup *scgrp = &src_root->cgrp;
		struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);
T
Tejun Heo 已提交
1677
		struct css_set *cset;
1678

1679
		WARN_ON(!css || cgroup_css(dcgrp, ss));
1680

1681 1682 1683 1684
		/* disable from the source */
		src_root->subsys_mask &= ~(1 << ssid);
		WARN_ON(cgroup_apply_control(scgrp));
		cgroup_finalize_control(scgrp, 0);
1685

1686
		/* rebind */
1687 1688
		RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
		rcu_assign_pointer(dcgrp->subsys[ssid], css);
1689
		ss->root = dst_root;
1690
		css->cgroup = dcgrp;
1691

1692
		spin_lock_irq(&css_set_lock);
T
Tejun Heo 已提交
1693 1694
		hash_for_each(css_set_table, i, cset, hlist)
			list_move_tail(&cset->e_cset_node[ss->id],
1695
				       &dcgrp->e_csets[ss->id]);
1696
		spin_unlock_irq(&css_set_lock);
T
Tejun Heo 已提交
1697

1698
		/* default hierarchy doesn't enable controllers by default */
1699
		dst_root->subsys_mask |= 1 << ssid;
1700 1701 1702
		if (dst_root == &cgrp_dfl_root) {
			static_branch_enable(cgroup_subsys_on_dfl_key[ssid]);
		} else {
1703
			dcgrp->subtree_control |= 1 << ssid;
1704
			static_branch_disable(cgroup_subsys_on_dfl_key[ssid]);
1705
		}
1706

1707 1708 1709 1710 1711
		ret = cgroup_apply_control(dcgrp);
		if (ret)
			pr_warn("partial failure to rebind %s controller (err=%d)\n",
				ss->name, ret);

1712 1713
		if (ss->bind)
			ss->bind(css);
1714
	} while_each_subsys_mask();
1715

1716
	kernfs_activate(dcgrp->kn);
1717 1718 1719
	return 0;
}

1720 1721
int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node,
		     struct kernfs_root *kf_root)
1722
{
F
Felipe Balbi 已提交
1723
	int len = 0;
1724 1725 1726 1727 1728 1729 1730 1731
	char *buf = NULL;
	struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root);
	struct cgroup *ns_cgroup;

	buf = kmalloc(PATH_MAX, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

1732
	spin_lock_irq(&css_set_lock);
1733 1734
	ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot);
	len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX);
1735
	spin_unlock_irq(&css_set_lock);
1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746

	if (len >= PATH_MAX)
		len = -ERANGE;
	else if (len > 0) {
		seq_escape(sf, buf, " \t\n\\");
		len = 0;
	}
	kfree(buf);
	return len;
}

1747 1748 1749 1750 1751 1752
static int parse_cgroup_root_flags(char *data, unsigned int *root_flags)
{
	char *token;

	*root_flags = 0;

1753
	if (!data || *data == '\0')
1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
		return 0;

	while ((token = strsep(&data, ",")) != NULL) {
		if (!strcmp(token, "nsdelegate")) {
			*root_flags |= CGRP_ROOT_NS_DELEGATE;
			continue;
		}

		pr_err("cgroup2: unknown option \"%s\"\n", token);
		return -EINVAL;
	}

	return 0;
}

static void apply_cgroup_root_flags(unsigned int root_flags)
{
	if (current->nsproxy->cgroup_ns == &init_cgroup_ns) {
		if (root_flags & CGRP_ROOT_NS_DELEGATE)
			cgrp_dfl_root.flags |= CGRP_ROOT_NS_DELEGATE;
		else
			cgrp_dfl_root.flags &= ~CGRP_ROOT_NS_DELEGATE;
	}
}

static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root)
{
	if (cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE)
		seq_puts(seq, ",nsdelegate");
	return 0;
}

T
Tejun Heo 已提交
1786
static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
1787
{
1788 1789 1790 1791 1792 1793 1794 1795 1796
	unsigned int root_flags;
	int ret;

	ret = parse_cgroup_root_flags(data, &root_flags);
	if (ret)
		return ret;

	apply_cgroup_root_flags(root_flags);
	return 0;
1797 1798
}

1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
/*
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
 * words after the first mount.
 */
static bool use_task_css_set_links __read_mostly;

static void cgroup_enable_task_cg_lists(void)
{
	struct task_struct *p, *g;

	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
1819 1820 1821 1822 1823 1824 1825
	spin_lock_irq(&css_set_lock);

	if (use_task_css_set_links)
		goto out_unlock;

	use_task_css_set_links = true;

1826 1827 1828 1829 1830 1831 1832 1833
	do_each_thread(g, p) {
		WARN_ON_ONCE(!list_empty(&p->cg_list) ||
			     task_css_set(p) != &init_css_set);

		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
1834 1835
		 * Do it while holding siglock so that we don't end up
		 * racing against cgroup_exit().
1836 1837 1838 1839
		 *
		 * Interrupts were already disabled while acquiring
		 * the css_set_lock, so we do not need to disable it
		 * again when acquiring the sighand->siglock here.
1840
		 */
1841
		spin_lock(&p->sighand->siglock);
1842 1843 1844
		if (!(p->flags & PF_EXITING)) {
			struct css_set *cset = task_css_set(p);

1845 1846
			if (!css_set_populated(cset))
				css_set_update_populated(cset, true);
1847
			list_add_tail(&p->cg_list, &cset->tasks);
1848
			get_css_set(cset);
1849
			cset->nr_tasks++;
1850
		}
1851
		spin_unlock(&p->sighand->siglock);
1852 1853
	} while_each_thread(g, p);
out_unlock:
1854
	spin_unlock_irq(&css_set_lock);
1855
	read_unlock(&tasklist_lock);
1856
}
1857

1858 1859
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
T
Tejun Heo 已提交
1860 1861 1862
	struct cgroup_subsys *ss;
	int ssid;

1863 1864
	INIT_LIST_HEAD(&cgrp->self.sibling);
	INIT_LIST_HEAD(&cgrp->self.children);
1865
	INIT_LIST_HEAD(&cgrp->cset_links);
1866 1867
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
1868
	cgrp->self.cgroup = cgrp;
1869
	cgrp->self.flags |= CSS_ONLINE;
1870
	cgrp->dom_cgrp = cgrp;
1871 1872
	cgrp->max_descendants = INT_MAX;
	cgrp->max_depth = INT_MAX;
1873
	INIT_LIST_HEAD(&cgrp->rstat_css_list);
1874
	prev_cputime_init(&cgrp->prev_cputime);
T
Tejun Heo 已提交
1875 1876 1877

	for_each_subsys(ss, ssid)
		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
1878 1879

	init_waitqueue_head(&cgrp->offline_waitq);
1880
	INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);
1881
}
1882

1883
void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts)
1884
{
1885
	struct cgroup *cgrp = &root->cgrp;
1886

1887
	INIT_LIST_HEAD(&root->root_list);
1888
	atomic_set(&root->nr_cgrps, 1);
1889
	cgrp->root = root;
1890
	init_cgroup_housekeeping(cgrp);
1891
	idr_init(&root->cgroup_idr);
1892 1893 1894

	root->flags = opts->flags;
	if (opts->release_agent)
1895
		strscpy(root->release_agent_path, opts->release_agent, PATH_MAX);
1896
	if (opts->name)
1897
		strscpy(root->name, opts->name, MAX_CGROUP_ROOT_NAMELEN);
1898
	if (opts->cpuset_clone_children)
1899
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
1900 1901
}

1902
int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags)
1903
{
1904
	LIST_HEAD(tmp_links);
1905
	struct cgroup *root_cgrp = &root->cgrp;
1906
	struct kernfs_syscall_ops *kf_sops;
1907 1908
	struct css_set *cset;
	int i, ret;
1909

1910
	lockdep_assert_held(&cgroup_mutex);
1911

V
Vladimir Davydov 已提交
1912
	ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL);
1913
	if (ret < 0)
T
Tejun Heo 已提交
1914
		goto out;
1915
	root_cgrp->id = ret;
1916
	root_cgrp->ancestor_ids[0] = ret;
1917

1918 1919
	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release,
			      ref_flags, GFP_KERNEL);
1920 1921 1922
	if (ret)
		goto out;

1923
	/*
1924
	 * We're accessing css_set_count without locking css_set_lock here,
1925
	 * but that's OK - it can only be increased by someone holding
1926 1927 1928
	 * cgroup_lock, and that's us.  Later rebinding may disable
	 * controllers on the default hierarchy and thus create new csets,
	 * which can't be more than the existing ones.  Allocate 2x.
1929
	 */
1930
	ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links);
1931
	if (ret)
1932
		goto cancel_ref;
1933

1934
	ret = cgroup_init_root_id(root);
1935
	if (ret)
1936
		goto cancel_ref;
1937

1938 1939 1940 1941
	kf_sops = root == &cgrp_dfl_root ?
		&cgroup_kf_syscall_ops : &cgroup1_kf_syscall_ops;

	root->kf_root = kernfs_create_root(kf_sops,
S
Shaohua Li 已提交
1942 1943
					   KERNFS_ROOT_CREATE_DEACTIVATED |
					   KERNFS_ROOT_SUPPORT_EXPORTOP,
T
Tejun Heo 已提交
1944 1945 1946 1947 1948 1949
					   root_cgrp);
	if (IS_ERR(root->kf_root)) {
		ret = PTR_ERR(root->kf_root);
		goto exit_root_id;
	}
	root_cgrp->kn = root->kf_root->kn;
1950

1951
	ret = css_populate_dir(&root_cgrp->self);
1952
	if (ret)
T
Tejun Heo 已提交
1953
		goto destroy_root;
1954

1955
	ret = rebind_subsystems(root, ss_mask);
1956
	if (ret)
T
Tejun Heo 已提交
1957
		goto destroy_root;
1958

1959 1960 1961
	ret = cgroup_bpf_inherit(root_cgrp);
	WARN_ON_ONCE(ret);

1962 1963
	trace_cgroup_setup_root(root);

1964 1965 1966 1967 1968 1969 1970
	/*
	 * There must be no failure case after here, since rebinding takes
	 * care of subsystems' refcounts, which are explicitly dropped in
	 * the failure exit path.
	 */
	list_add(&root->root_list, &cgroup_roots);
	cgroup_root_count++;
A
Al Viro 已提交
1971

1972
	/*
1973
	 * Link the root cgroup in this hierarchy into all the css_set
1974 1975
	 * objects.
	 */
1976
	spin_lock_irq(&css_set_lock);
1977
	hash_for_each(css_set_table, i, cset, hlist) {
1978
		link_css_set(&tmp_links, cset, root_cgrp);
1979 1980 1981
		if (css_set_populated(cset))
			cgroup_update_populated(root_cgrp, true);
	}
1982
	spin_unlock_irq(&css_set_lock);
1983

1984
	BUG_ON(!list_empty(&root_cgrp->self.children));
1985
	BUG_ON(atomic_read(&root->nr_cgrps) != 1);
1986

T
Tejun Heo 已提交
1987
	kernfs_activate(root_cgrp->kn);
1988
	ret = 0;
T
Tejun Heo 已提交
1989
	goto out;
1990

T
Tejun Heo 已提交
1991 1992 1993 1994
destroy_root:
	kernfs_destroy_root(root->kf_root);
	root->kf_root = NULL;
exit_root_id:
1995
	cgroup_exit_root_id(root);
1996
cancel_ref:
1997
	percpu_ref_exit(&root_cgrp->self.refcnt);
T
Tejun Heo 已提交
1998
out:
1999 2000
	free_cgrp_cset_links(&tmp_links);
	return ret;
2001 2002
}

2003 2004 2005
struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags,
			       struct cgroup_root *root, unsigned long magic,
			       struct cgroup_namespace *ns)
2006
{
T
Tejun Heo 已提交
2007
	struct dentry *dentry;
2008
	bool new_sb = false;
2009

2010
	dentry = kernfs_mount(fs_type, flags, root->kf_root, magic, &new_sb);
2011

2012
	/*
2013 2014
	 * In non-init cgroup namespace, instead of root cgroup's dentry,
	 * we return the dentry corresponding to the cgroupns->root_cgrp.
2015
	 */
2016 2017
	if (!IS_ERR(dentry) && ns != &init_cgroup_ns) {
		struct dentry *nsdentry;
2018
		struct super_block *sb = dentry->d_sb;
2019
		struct cgroup *cgrp;
2020

2021 2022 2023 2024 2025 2026 2027 2028
		mutex_lock(&cgroup_mutex);
		spin_lock_irq(&css_set_lock);

		cgrp = cset_cgroup_from_root(ns->root_cset, root);

		spin_unlock_irq(&css_set_lock);
		mutex_unlock(&cgroup_mutex);

2029
		nsdentry = kernfs_node_dentry(cgrp->kn, sb);
2030
		dput(dentry);
2031 2032
		if (IS_ERR(nsdentry))
			deactivate_locked_super(sb);
2033
		dentry = nsdentry;
2034 2035
	}

2036
	if (!new_sb)
2037 2038 2039 2040 2041
		cgroup_put(&root->cgrp);

	return dentry;
}

A
Al Viro 已提交
2042
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
2043
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
2044
			 void *data)
2045
{
2046
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
T
Tejun Heo 已提交
2047
	struct dentry *dentry;
2048
	int ret;
2049

2050 2051 2052 2053 2054 2055 2056 2057
	get_cgroup_ns(ns);

	/* Check if the caller has permission to mount. */
	if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) {
		put_cgroup_ns(ns);
		return ERR_PTR(-EPERM);
	}

2058 2059 2060 2061 2062 2063
	/*
	 * The first time anyone tries to mount a cgroup, enable the list
	 * linking each css_set to its tasks and fix up all existing tasks.
	 */
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();
2064

2065
	if (fs_type == &cgroup2_fs_type) {
2066 2067 2068 2069
		unsigned int root_flags;

		ret = parse_cgroup_root_flags(data, &root_flags);
		if (ret) {
2070
			put_cgroup_ns(ns);
2071
			return ERR_PTR(ret);
2072
		}
2073

T
Tejun Heo 已提交
2074
		cgrp_dfl_visible = true;
2075
		cgroup_get_live(&cgrp_dfl_root.cgrp);
2076 2077 2078

		dentry = cgroup_do_mount(&cgroup2_fs_type, flags, &cgrp_dfl_root,
					 CGROUP2_SUPER_MAGIC, ns);
2079 2080
		if (!IS_ERR(dentry))
			apply_cgroup_root_flags(root_flags);
2081 2082 2083
	} else {
		dentry = cgroup1_mount(&cgroup_fs_type, flags, data,
				       CGROUP_SUPER_MAGIC, ns);
2084 2085
	}

2086
	put_cgroup_ns(ns);
T
Tejun Heo 已提交
2087 2088
	return dentry;
}
2089

T
Tejun Heo 已提交
2090 2091 2092
static void cgroup_kill_sb(struct super_block *sb)
{
	struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
2093
	struct cgroup_root *root = cgroup_root_from_kf(kf_root);
2094

2095
	/*
2096 2097 2098
	 * If @root doesn't have any mounts or children, start killing it.
	 * This prevents new mounts by disabling percpu_ref_tryget_live().
	 * cgroup_mount() may wait for @root's release.
2099 2100
	 *
	 * And don't kill the default root.
2101
	 */
2102
	if (!list_empty(&root->cgrp.self.children) ||
2103
	    root == &cgrp_dfl_root)
2104 2105 2106
		cgroup_put(&root->cgrp);
	else
		percpu_ref_kill(&root->cgrp.self.refcnt);
2107

T
Tejun Heo 已提交
2108
	kernfs_kill_sb(sb);
2109
}
2110

2111
struct file_system_type cgroup_fs_type = {
2112
	.name = "cgroup",
A
Al Viro 已提交
2113
	.mount = cgroup_mount,
2114
	.kill_sb = cgroup_kill_sb,
S
Serge Hallyn 已提交
2115
	.fs_flags = FS_USERNS_MOUNT,
2116
};
2117

2118 2119 2120 2121
static struct file_system_type cgroup2_fs_type = {
	.name = "cgroup2",
	.mount = cgroup_mount,
	.kill_sb = cgroup_kill_sb,
S
Serge Hallyn 已提交
2122
	.fs_flags = FS_USERNS_MOUNT,
2123
};
2124

2125 2126
int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen,
			  struct cgroup_namespace *ns)
2127 2128 2129
{
	struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root);

2130
	return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen);
2131 2132
}

2133 2134
int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen,
		   struct cgroup_namespace *ns)
2135
{
2136
	int ret;
2137 2138

	mutex_lock(&cgroup_mutex);
2139
	spin_lock_irq(&css_set_lock);
2140 2141 2142

	ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns);

2143
	spin_unlock_irq(&css_set_lock);
2144 2145 2146 2147 2148 2149
	mutex_unlock(&cgroup_mutex);

	return ret;
}
EXPORT_SYMBOL_GPL(cgroup_path_ns);

2150
/**
2151
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2152 2153 2154 2155
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
2156 2157 2158 2159 2160
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
T
Tejun Heo 已提交
2161
 * Return value is the same as kernfs_path().
2162
 */
2163
int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
2164
{
2165
	struct cgroup_root *root;
2166
	struct cgroup *cgrp;
T
Tejun Heo 已提交
2167
	int hierarchy_id = 1;
2168
	int ret;
2169 2170

	mutex_lock(&cgroup_mutex);
2171
	spin_lock_irq(&css_set_lock);
2172

2173 2174
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

2175 2176
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
2177
		ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns);
2178 2179
	} else {
		/* if no hierarchy exists, everyone is in "/" */
2180
		ret = strlcpy(buf, "/", buflen);
2181 2182
	}

2183
	spin_unlock_irq(&css_set_lock);
2184
	mutex_unlock(&cgroup_mutex);
2185
	return ret;
2186
}
2187
EXPORT_SYMBOL_GPL(task_cgroup_path);
2188

2189
/**
2190
 * cgroup_migrate_add_task - add a migration target task to a migration context
2191
 * @task: target task
2192
 * @mgctx: target migration context
2193
 *
2194 2195 2196 2197
 * Add @task, which is a migration target, to @mgctx->tset.  This function
 * becomes noop if @task doesn't need to be migrated.  @task's css_set
 * should have been added as a migration source and @task->cg_list will be
 * moved from the css_set's tasks list to mg_tasks one.
2198
 */
2199 2200
static void cgroup_migrate_add_task(struct task_struct *task,
				    struct cgroup_mgctx *mgctx)
2201 2202 2203
{
	struct css_set *cset;

2204
	lockdep_assert_held(&css_set_lock);
2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217

	/* @task either already exited or can't exit until the end */
	if (task->flags & PF_EXITING)
		return;

	/* leave @task alone if post_fork() hasn't linked it yet */
	if (list_empty(&task->cg_list))
		return;

	cset = task_css_set(task);
	if (!cset->mg_src_cgrp)
		return;

2218 2219
	mgctx->tset.nr_tasks++;

2220 2221
	list_move_tail(&task->cg_list, &cset->mg_tasks);
	if (list_empty(&cset->mg_node))
2222 2223
		list_add_tail(&cset->mg_node,
			      &mgctx->tset.src_csets);
2224
	if (list_empty(&cset->mg_dst_cset->mg_node)) {
2225
		list_add_tail(&cset->mg_dst_cset->mg_node,
2226
			      &mgctx->tset.dst_csets);
2227 2228
		mgctx->tset.dst_count++;
	}
2229 2230
}

2231 2232 2233
/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
2234
 * @dst_cssp: output variable for the destination css
2235 2236 2237
 *
 * @tset iteration is initialized and the first task is returned.
 */
2238 2239
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
					 struct cgroup_subsys_state **dst_cssp)
2240
{
2241 2242 2243
	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
	tset->cur_task = NULL;

2244
	return cgroup_taskset_next(tset, dst_cssp);
2245 2246 2247 2248 2249
}

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
2250
 * @dst_cssp: output variable for the destination css
2251 2252 2253 2254
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
2255 2256
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
					struct cgroup_subsys_state **dst_cssp)
2257
{
2258 2259
	struct css_set *cset = tset->cur_cset;
	struct task_struct *task = tset->cur_task;
2260

2261 2262 2263 2264 2265 2266
	while (&cset->mg_node != tset->csets) {
		if (!task)
			task = list_first_entry(&cset->mg_tasks,
						struct task_struct, cg_list);
		else
			task = list_next_entry(task, cg_list);
2267

2268 2269 2270
		if (&task->cg_list != &cset->mg_tasks) {
			tset->cur_cset = cset;
			tset->cur_task = task;
2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282

			/*
			 * This function may be called both before and
			 * after cgroup_taskset_migrate().  The two cases
			 * can be distinguished by looking at whether @cset
			 * has its ->mg_dst_cset set.
			 */
			if (cset->mg_dst_cset)
				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];
			else
				*dst_cssp = cset->subsys[tset->ssid];

2283 2284
			return task;
		}
2285

2286 2287 2288
		cset = list_next_entry(cset, mg_node);
		task = NULL;
	}
2289

2290
	return NULL;
2291 2292
}

2293
/**
2294
 * cgroup_taskset_migrate - migrate a taskset
2295
 * @mgctx: migration context
2296
 *
2297
 * Migrate tasks in @mgctx as setup by migration preparation functions.
2298
 * This function fails iff one of the ->can_attach callbacks fails and
2299 2300
 * guarantees that either all or none of the tasks in @mgctx are migrated.
 * @mgctx is consumed regardless of success.
2301
 */
2302
static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
2303
{
2304
	struct cgroup_taskset *tset = &mgctx->tset;
2305
	struct cgroup_subsys *ss;
2306 2307
	struct task_struct *task, *tmp_task;
	struct css_set *cset, *tmp_cset;
2308
	int ssid, failed_ssid, ret;
2309
	LIST_HEAD(tmp_links);
2310 2311

	/* check that we can legitimately attach to the cgroup */
2312
	if (tset->nr_tasks) {
2313 2314 2315 2316
		ret = allocate_memcg_blkcg_links(tset->dst_count*2, &tmp_links);
		if (ret)
			goto out_free_list;

2317 2318 2319 2320 2321 2322 2323 2324
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ss->can_attach) {
				tset->ssid = ssid;
				ret = ss->can_attach(tset);
				if (ret) {
					failed_ssid = ssid;
					goto out_cancel_attach;
				}
2325
			}
2326 2327
		} while_each_subsys_mask();
	}
2328 2329 2330 2331 2332 2333

	/*
	 * Now that we're guaranteed success, proceed to move all tasks to
	 * the new cgroup.  There are no failure cases after here, so this
	 * is the commit point.
	 */
2334
	spin_lock_irq(&css_set_lock);
2335
	list_for_each_entry(cset, &tset->src_csets, mg_node) {
T
Tejun Heo 已提交
2336 2337 2338 2339 2340
		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {
			struct css_set *from_cset = task_css_set(task);
			struct css_set *to_cset = cset->mg_dst_cset;

			get_css_set(to_cset);
2341
			to_cset->nr_tasks++;
T
Tejun Heo 已提交
2342 2343
			css_set_move_task(task, from_cset, to_cset, true);
			put_css_set_locked(from_cset);
2344
			from_cset->nr_tasks--;
T
Tejun Heo 已提交
2345
		}
2346
	}
2347
	spin_unlock_irq(&css_set_lock);
2348 2349 2350 2351 2352 2353 2354 2355

	/*
	 * Migration is committed, all target tasks are now on dst_csets.
	 * Nothing is sensitive to fork() after this point.  Notify
	 * controllers that migration is complete.
	 */
	tset->csets = &tset->dst_csets;

2356 2357 2358 2359 2360 2361
	if (tset->nr_tasks) {
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ss->attach) {
				tset->ssid = ssid;
				ss->attach(tset);
			}
2362 2363
			list_for_each_entry(cset, &tset->dst_csets, mg_node)
				insert_memcg_blkcg_link(ss, &tmp_links, cset);
2364 2365
		} while_each_subsys_mask();
	}
2366 2367 2368 2369 2370

	ret = 0;
	goto out_release_tset;

out_cancel_attach:
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380
	if (tset->nr_tasks) {
		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
			if (ssid == failed_ssid)
				break;
			if (ss->cancel_attach) {
				tset->ssid = ssid;
				ss->cancel_attach(tset);
			}
		} while_each_subsys_mask();
	}
2381
out_release_tset:
2382
	spin_lock_irq(&css_set_lock);
2383 2384 2385 2386 2387
	list_splice_init(&tset->dst_csets, &tset->src_csets);
	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {
		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
		list_del_init(&cset->mg_node);
	}
2388
	spin_unlock_irq(&css_set_lock);
2389

2390 2391 2392
out_free_list:
	free_memcg_blkcg_links(&tmp_links);

2393 2394 2395 2396 2397 2398 2399
	/*
	 * Re-initialize the cgroup_taskset structure in case it is reused
	 * again in another cgroup_migrate_add_task()/cgroup_migrate_execute()
	 * iteration.
	 */
	tset->nr_tasks = 0;
	tset->csets    = &tset->src_csets;
2400 2401 2402
	return ret;
}

2403
/**
2404
 * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2405 2406
 * @dst_cgrp: destination cgroup to test
 *
2407 2408 2409 2410
 * On the default hierarchy, except for the mixable, (possible) thread root
 * and threaded cgroups, subtree_control must be zero for migration
 * destination cgroups with tasks so that child cgroups don't compete
 * against tasks.
2411
 */
2412
int cgroup_migrate_vet_dst(struct cgroup *dst_cgrp)
2413
{
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437
	/* v1 doesn't have any restriction */
	if (!cgroup_on_dfl(dst_cgrp))
		return 0;

	/* verify @dst_cgrp can host resources */
	if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp))
		return -EOPNOTSUPP;

	/* mixables don't care */
	if (cgroup_is_mixable(dst_cgrp))
		return 0;

	/*
	 * If @dst_cgrp is already or can become a thread root or is
	 * threaded, it doesn't matter.
	 */
	if (cgroup_can_be_thread_root(dst_cgrp) || cgroup_is_threaded(dst_cgrp))
		return 0;

	/* apply no-internal-process constraint */
	if (dst_cgrp->subtree_control)
		return -EBUSY;

	return 0;
2438 2439
}

L
Li Zefan 已提交
2440
/**
2441
 * cgroup_migrate_finish - cleanup after attach
2442
 * @mgctx: migration context
B
Ben Blum 已提交
2443
 *
2444 2445
 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst().  See
 * those functions for details.
B
Ben Blum 已提交
2446
 */
2447
void cgroup_migrate_finish(struct cgroup_mgctx *mgctx)
B
Ben Blum 已提交
2448
{
2449
	LIST_HEAD(preloaded);
2450
	struct css_set *cset, *tmp_cset;
B
Ben Blum 已提交
2451

2452 2453
	lockdep_assert_held(&cgroup_mutex);

2454
	spin_lock_irq(&css_set_lock);
2455 2456 2457 2458 2459

	list_splice_tail_init(&mgctx->preloaded_src_csets, &preloaded);
	list_splice_tail_init(&mgctx->preloaded_dst_csets, &preloaded);

	list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) {
2460
		cset->mg_src_cgrp = NULL;
2461
		cset->mg_dst_cgrp = NULL;
2462 2463
		cset->mg_dst_cset = NULL;
		list_del_init(&cset->mg_preload_node);
Z
Zefan Li 已提交
2464
		put_css_set_locked(cset);
2465
	}
2466

2467
	spin_unlock_irq(&css_set_lock);
2468 2469 2470 2471 2472 2473
}

/**
 * cgroup_migrate_add_src - add a migration source css_set
 * @src_cset: the source css_set to add
 * @dst_cgrp: the destination cgroup
2474
 * @mgctx: migration context
2475 2476
 *
 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp.  Pin
2477
 * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2478 2479
 * up by cgroup_migrate_finish().
 *
2480 2481 2482 2483 2484
 * This function may be called without holding cgroup_threadgroup_rwsem
 * even if the target is a process.  Threads may be created and destroyed
 * but as long as cgroup_mutex is not dropped, no new css_set can be put
 * into play and the preloaded css_sets are guaranteed to cover all
 * migrations.
2485
 */
2486 2487
void cgroup_migrate_add_src(struct css_set *src_cset,
			    struct cgroup *dst_cgrp,
2488
			    struct cgroup_mgctx *mgctx)
2489 2490 2491 2492
{
	struct cgroup *src_cgrp;

	lockdep_assert_held(&cgroup_mutex);
2493
	lockdep_assert_held(&css_set_lock);
2494

2495 2496 2497 2498 2499 2500 2501 2502
	/*
	 * If ->dead, @src_set is associated with one or more dead cgroups
	 * and doesn't contain any migratable tasks.  Ignore it early so
	 * that the rest of migration path doesn't get confused by it.
	 */
	if (src_cset->dead)
		return;

2503 2504 2505 2506 2507 2508
	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);

	if (!list_empty(&src_cset->mg_preload_node))
		return;

	WARN_ON(src_cset->mg_src_cgrp);
2509
	WARN_ON(src_cset->mg_dst_cgrp);
2510 2511 2512 2513
	WARN_ON(!list_empty(&src_cset->mg_tasks));
	WARN_ON(!list_empty(&src_cset->mg_node));

	src_cset->mg_src_cgrp = src_cgrp;
2514
	src_cset->mg_dst_cgrp = dst_cgrp;
2515
	get_css_set(src_cset);
2516
	list_add_tail(&src_cset->mg_preload_node, &mgctx->preloaded_src_csets);
2517 2518 2519 2520
}

/**
 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2521
 * @mgctx: migration context
2522
 *
2523
 * Tasks are about to be moved and all the source css_sets have been
2524 2525 2526
 * preloaded to @mgctx->preloaded_src_csets.  This function looks up and
 * pins all destination css_sets, links each to its source, and append them
 * to @mgctx->preloaded_dst_csets.
2527 2528 2529 2530
 *
 * This function must be called after cgroup_migrate_add_src() has been
 * called on each migration source css_set.  After migration is performed
 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2531
 * @mgctx.
2532
 */
2533
int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx)
2534
{
2535
	struct css_set *src_cset, *tmp_cset;
2536 2537 2538 2539

	lockdep_assert_held(&cgroup_mutex);

	/* look up the dst cset for each src cset and link it to src */
2540 2541
	list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets,
				 mg_preload_node) {
2542
		struct css_set *dst_cset;
2543 2544
		struct cgroup_subsys *ss;
		int ssid;
2545

2546
		dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);
2547 2548 2549 2550
		if (!dst_cset)
			goto err;

		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
2551 2552 2553 2554 2555 2556 2557 2558

		/*
		 * If src cset equals dst, it's noop.  Drop the src.
		 * cgroup_migrate() will skip the cset too.  Note that we
		 * can't handle src == dst as some nodes are used by both.
		 */
		if (src_cset == dst_cset) {
			src_cset->mg_src_cgrp = NULL;
2559
			src_cset->mg_dst_cgrp = NULL;
2560
			list_del_init(&src_cset->mg_preload_node);
Z
Zefan Li 已提交
2561 2562
			put_css_set(src_cset);
			put_css_set(dst_cset);
2563 2564 2565
			continue;
		}

2566 2567 2568
		src_cset->mg_dst_cset = dst_cset;

		if (list_empty(&dst_cset->mg_preload_node))
2569 2570
			list_add_tail(&dst_cset->mg_preload_node,
				      &mgctx->preloaded_dst_csets);
2571
		else
Z
Zefan Li 已提交
2572
			put_css_set(dst_cset);
2573 2574 2575 2576

		for_each_subsys(ss, ssid)
			if (src_cset->subsys[ssid] != dst_cset->subsys[ssid])
				mgctx->ss_mask |= 1 << ssid;
2577 2578 2579 2580
	}

	return 0;
err:
2581
	cgroup_migrate_finish(mgctx);
2582 2583 2584 2585 2586 2587 2588
	return -ENOMEM;
}

/**
 * cgroup_migrate - migrate a process or task to a cgroup
 * @leader: the leader of the process or the task to migrate
 * @threadgroup: whether @leader points to the whole process or a single task
2589
 * @mgctx: migration context
2590
 *
2591 2592 2593
 * Migrate a process or task denoted by @leader.  If migrating a process,
 * the caller must be holding cgroup_threadgroup_rwsem.  The caller is also
 * responsible for invoking cgroup_migrate_add_src() and
2594 2595 2596 2597 2598 2599 2600 2601 2602
 * cgroup_migrate_prepare_dst() on the targets before invoking this
 * function and following up with cgroup_migrate_finish().
 *
 * As long as a controller's ->can_attach() doesn't fail, this function is
 * guaranteed to succeed.  This means that, excluding ->can_attach()
 * failure, when migrating multiple targets, the success or failure can be
 * decided for all targets by invoking group_migrate_prepare_dst() before
 * actually starting migrating.
 */
2603
int cgroup_migrate(struct task_struct *leader, bool threadgroup,
2604
		   struct cgroup_mgctx *mgctx)
B
Ben Blum 已提交
2605
{
2606
	struct task_struct *task;
B
Ben Blum 已提交
2607

2608 2609 2610 2611 2612
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
2613
	spin_lock_irq(&css_set_lock);
2614
	rcu_read_lock();
2615
	task = leader;
B
Ben Blum 已提交
2616
	do {
2617
		cgroup_migrate_add_task(task, mgctx);
2618 2619
		if (!threadgroup)
			break;
2620
	} while_each_thread(leader, task);
2621
	rcu_read_unlock();
2622
	spin_unlock_irq(&css_set_lock);
B
Ben Blum 已提交
2623

2624
	return cgroup_migrate_execute(mgctx);
B
Ben Blum 已提交
2625 2626
}

2627 2628 2629 2630 2631 2632
/**
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
 * @dst_cgrp: the cgroup to attach to
 * @leader: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
 *
2633
 * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
2634
 */
2635 2636
int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,
		       bool threadgroup)
2637
{
2638
	DEFINE_CGROUP_MGCTX(mgctx);
2639 2640 2641
	struct task_struct *task;
	int ret;

2642 2643 2644
	ret = cgroup_migrate_vet_dst(dst_cgrp);
	if (ret)
		return ret;
2645

2646
	/* look up all src csets */
2647
	spin_lock_irq(&css_set_lock);
2648 2649 2650
	rcu_read_lock();
	task = leader;
	do {
2651
		cgroup_migrate_add_src(task_css_set(task), dst_cgrp, &mgctx);
2652 2653 2654 2655
		if (!threadgroup)
			break;
	} while_each_thread(leader, task);
	rcu_read_unlock();
2656
	spin_unlock_irq(&css_set_lock);
2657 2658

	/* prepare dst csets and commit */
2659
	ret = cgroup_migrate_prepare_dst(&mgctx);
2660
	if (!ret)
2661
		ret = cgroup_migrate(leader, threadgroup, &mgctx);
2662

2663
	cgroup_migrate_finish(&mgctx);
2664 2665

	if (!ret)
2666
		TRACE_CGROUP_PATH(attach_task, dst_cgrp, leader, threadgroup);
2667

2668
	return ret;
B
Ben Blum 已提交
2669 2670
}

2671 2672
struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup)
	__acquires(&cgroup_threadgroup_rwsem)
2673 2674
{
	struct task_struct *tsk;
2675
	pid_t pid;
2676

2677
	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2678
		return ERR_PTR(-EINVAL);
B
Ben Blum 已提交
2679

T
Tejun Heo 已提交
2680
	percpu_down_write(&cgroup_threadgroup_rwsem);
2681

2682
	rcu_read_lock();
2683
	if (pid) {
2684
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2685
		if (!tsk) {
2686 2687
			tsk = ERR_PTR(-ESRCH);
			goto out_unlock_threadgroup;
2688
		}
2689
	} else {
2690
		tsk = current;
2691
	}
2692 2693

	if (threadgroup)
2694
		tsk = tsk->group_leader;
2695 2696

	/*
2697 2698 2699 2700
	 * kthreads may acquire PF_NO_SETAFFINITY during initialization.
	 * If userland migrates such a kthread to a non-root cgroup, it can
	 * become trapped in a cpuset, or RT kthread may be born in a
	 * cgroup with no rt_runtime allocated.  Just say no.
2701
	 */
2702
	if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) {
2703 2704
		tsk = ERR_PTR(-EINVAL);
		goto out_unlock_threadgroup;
2705 2706
	}

2707
	get_task_struct(tsk);
2708 2709 2710 2711 2712
	goto out_unlock_rcu;

out_unlock_threadgroup:
	percpu_up_write(&cgroup_threadgroup_rwsem);
out_unlock_rcu:
2713
	rcu_read_unlock();
2714 2715
	return tsk;
}
2716

2717 2718 2719 2720 2721
void cgroup_procs_write_finish(struct task_struct *task)
	__releases(&cgroup_threadgroup_rwsem)
{
	struct cgroup_subsys *ss;
	int ssid;
2722

2723 2724
	/* release reference from cgroup_procs_write_start() */
	put_task_struct(task);
T
Tejun Heo 已提交
2725 2726

	percpu_up_write(&cgroup_threadgroup_rwsem);
2727 2728 2729
	for_each_subsys(ss, ssid)
		if (ss->post_attach)
			ss->post_attach();
2730 2731
}

2732
static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask)
2733
{
2734 2735 2736
	struct cgroup_subsys *ss;
	bool printed = false;
	int ssid;
2737

2738
	do_each_subsys_mask(ss, ssid, ss_mask) {
2739 2740 2741 2742
		if (printed)
			seq_putc(seq, ' ');
		seq_printf(seq, "%s", ss->name);
		printed = true;
2743
	} while_each_subsys_mask();
2744 2745
	if (printed)
		seq_putc(seq, '\n');
2746 2747
}

2748 2749
/* show controllers which are enabled from the parent */
static int cgroup_controllers_show(struct seq_file *seq, void *v)
2750
{
2751 2752
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2753
	cgroup_print_ss_mask(seq, cgroup_control(cgrp));
2754
	return 0;
2755 2756
}

2757 2758
/* show controllers which are enabled for a given cgroup's children */
static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
2759
{
2760 2761
	struct cgroup *cgrp = seq_css(seq)->cgroup;

2762
	cgroup_print_ss_mask(seq, cgrp->subtree_control);
2763 2764 2765 2766 2767 2768 2769
	return 0;
}

/**
 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
 * @cgrp: root of the subtree to update csses for
 *
2770 2771 2772 2773
 * @cgrp's control masks have changed and its subtree's css associations
 * need to be updated accordingly.  This function looks up all css_sets
 * which are attached to the subtree, creates the matching updated css_sets
 * and migrates the tasks to the new ones.
2774 2775 2776
 */
static int cgroup_update_dfl_csses(struct cgroup *cgrp)
{
2777
	DEFINE_CGROUP_MGCTX(mgctx);
2778 2779
	struct cgroup_subsys_state *d_css;
	struct cgroup *dsct;
2780 2781 2782 2783 2784
	struct css_set *src_cset;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

T
Tejun Heo 已提交
2785 2786
	percpu_down_write(&cgroup_threadgroup_rwsem);

2787
	/* look up all csses currently attached to @cgrp's subtree */
2788
	spin_lock_irq(&css_set_lock);
2789
	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
2790 2791
		struct cgrp_cset_link *link;

2792
		list_for_each_entry(link, &dsct->cset_links, cset_link)
2793
			cgroup_migrate_add_src(link->cset, dsct, &mgctx);
2794
	}
2795
	spin_unlock_irq(&css_set_lock);
2796 2797

	/* NULL dst indicates self on default hierarchy */
2798
	ret = cgroup_migrate_prepare_dst(&mgctx);
2799 2800 2801
	if (ret)
		goto out_finish;

2802
	spin_lock_irq(&css_set_lock);
2803
	list_for_each_entry(src_cset, &mgctx.preloaded_src_csets, mg_preload_node) {
2804
		struct task_struct *task, *ntask;
2805

2806 2807
		/* all tasks in src_csets need to be migrated */
		list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)
2808
			cgroup_migrate_add_task(task, &mgctx);
2809
	}
2810
	spin_unlock_irq(&css_set_lock);
2811

2812
	ret = cgroup_migrate_execute(&mgctx);
2813
out_finish:
2814
	cgroup_migrate_finish(&mgctx);
T
Tejun Heo 已提交
2815
	percpu_up_write(&cgroup_threadgroup_rwsem);
2816 2817 2818
	return ret;
}

2819
/**
2820
 * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
2821
 * @cgrp: root of the target subtree
2822 2823
 *
 * Because css offlining is asynchronous, userland may try to re-enable a
2824 2825
 * controller while the previous css is still around.  This function grabs
 * cgroup_mutex and drains the previous css instances of @cgrp's subtree.
2826
 */
2827
void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
2828
	__acquires(&cgroup_mutex)
2829 2830
{
	struct cgroup *dsct;
2831
	struct cgroup_subsys_state *d_css;
2832 2833 2834
	struct cgroup_subsys *ss;
	int ssid;

2835 2836
restart:
	mutex_lock(&cgroup_mutex);
2837

2838
	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
2839 2840 2841 2842
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
			DEFINE_WAIT(wait);

2843
			if (!css || !percpu_ref_is_dying(&css->refcnt))
2844 2845
				continue;

2846
			cgroup_get_live(dsct);
2847 2848 2849 2850 2851 2852 2853 2854
			prepare_to_wait(&dsct->offline_waitq, &wait,
					TASK_UNINTERRUPTIBLE);

			mutex_unlock(&cgroup_mutex);
			schedule();
			finish_wait(&dsct->offline_waitq, &wait);

			cgroup_put(dsct);
2855
			goto restart;
2856 2857 2858 2859
		}
	}
}

2860
/**
2861
 * cgroup_save_control - save control masks and dom_cgrp of a subtree
2862 2863
 * @cgrp: root of the target subtree
 *
2864 2865 2866
 * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the
 * respective old_ prefixed fields for @cgrp's subtree including @cgrp
 * itself.
2867 2868 2869 2870 2871 2872 2873 2874 2875
 */
static void cgroup_save_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
		dsct->old_subtree_control = dsct->subtree_control;
		dsct->old_subtree_ss_mask = dsct->subtree_ss_mask;
2876
		dsct->old_dom_cgrp = dsct->dom_cgrp;
2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
	}
}

/**
 * cgroup_propagate_control - refresh control masks of a subtree
 * @cgrp: root of the target subtree
 *
 * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
 * ->subtree_control and propagate controller availability through the
 * subtree so that descendants don't have unavailable controllers enabled.
 */
static void cgroup_propagate_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
		dsct->subtree_control &= cgroup_control(dsct);
2895 2896 2897
		dsct->subtree_ss_mask =
			cgroup_calc_subtree_ss_mask(dsct->subtree_control,
						    cgroup_ss_mask(dsct));
2898 2899 2900 2901
	}
}

/**
2902
 * cgroup_restore_control - restore control masks and dom_cgrp of a subtree
2903 2904
 * @cgrp: root of the target subtree
 *
2905 2906 2907
 * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the
 * respective old_ prefixed fields for @cgrp's subtree including @cgrp
 * itself.
2908 2909 2910 2911 2912 2913 2914 2915 2916
 */
static void cgroup_restore_control(struct cgroup *cgrp)
{
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;

	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
		dsct->subtree_control = dsct->old_subtree_control;
		dsct->subtree_ss_mask = dsct->old_subtree_ss_mask;
2917
		dsct->dom_cgrp = dsct->old_dom_cgrp;
2918 2919 2920
	}
}

2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
static bool css_visible(struct cgroup_subsys_state *css)
{
	struct cgroup_subsys *ss = css->ss;
	struct cgroup *cgrp = css->cgroup;

	if (cgroup_control(cgrp) & (1 << ss->id))
		return true;
	if (!(cgroup_ss_mask(cgrp) & (1 << ss->id)))
		return false;
	return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl;
}

2933 2934
/**
 * cgroup_apply_control_enable - enable or show csses according to control
2935
 * @cgrp: root of the target subtree
2936
 *
2937
 * Walk @cgrp's subtree and create new csses or make the existing ones
2938 2939 2940 2941 2942 2943
 * visible.  A css is created invisible if it's being implicitly enabled
 * through dependency.  An invisible css is made visible when the userland
 * explicitly enables it.
 *
 * Returns 0 on success, -errno on failure.  On failure, csses which have
 * been processed already aren't cleaned up.  The caller is responsible for
2944
 * cleaning up with cgroup_apply_control_disable().
2945 2946 2947 2948
 */
static int cgroup_apply_control_enable(struct cgroup *cgrp)
{
	struct cgroup *dsct;
2949
	struct cgroup_subsys_state *d_css;
2950 2951 2952
	struct cgroup_subsys *ss;
	int ssid, ret;

2953
	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
2954 2955 2956
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);

2957 2958
			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));

2959 2960 2961 2962 2963 2964 2965 2966 2967
			if (!(cgroup_ss_mask(dsct) & (1 << ss->id)))
				continue;

			if (!css) {
				css = css_create(dsct, ss);
				if (IS_ERR(css))
					return PTR_ERR(css);
			}

2968
			if (css_visible(css)) {
2969
				ret = css_populate_dir(css);
2970 2971 2972 2973 2974 2975 2976 2977 2978
				if (ret)
					return ret;
			}
		}
	}

	return 0;
}

2979 2980
/**
 * cgroup_apply_control_disable - kill or hide csses according to control
2981
 * @cgrp: root of the target subtree
2982
 *
2983
 * Walk @cgrp's subtree and kill and hide csses so that they match
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994
 * cgroup_ss_mask() and cgroup_visible_mask().
 *
 * A css is hidden when the userland requests it to be disabled while other
 * subsystems are still depending on it.  The css must not actively control
 * resources and be in the vanilla state if it's made visible again later.
 * Controllers which may be depended upon should provide ->css_reset() for
 * this purpose.
 */
static void cgroup_apply_control_disable(struct cgroup *cgrp)
{
	struct cgroup *dsct;
2995
	struct cgroup_subsys_state *d_css;
2996 2997 2998
	struct cgroup_subsys *ss;
	int ssid;

2999
	cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) {
3000 3001 3002
		for_each_subsys(ss, ssid) {
			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);

3003 3004
			WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt));

3005 3006 3007
			if (!css)
				continue;

3008 3009
			if (css->parent &&
			    !(cgroup_ss_mask(dsct) & (1 << ss->id))) {
3010
				kill_css(css);
3011
			} else if (!css_visible(css)) {
3012
				css_clear_dir(css);
3013 3014 3015 3016 3017 3018 3019
				if (ss->css_reset)
					ss->css_reset(css);
			}
		}
	}
}

3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075
/**
 * cgroup_apply_control - apply control mask updates to the subtree
 * @cgrp: root of the target subtree
 *
 * subsystems can be enabled and disabled in a subtree using the following
 * steps.
 *
 * 1. Call cgroup_save_control() to stash the current state.
 * 2. Update ->subtree_control masks in the subtree as desired.
 * 3. Call cgroup_apply_control() to apply the changes.
 * 4. Optionally perform other related operations.
 * 5. Call cgroup_finalize_control() to finish up.
 *
 * This function implements step 3 and propagates the mask changes
 * throughout @cgrp's subtree, updates csses accordingly and perform
 * process migrations.
 */
static int cgroup_apply_control(struct cgroup *cgrp)
{
	int ret;

	cgroup_propagate_control(cgrp);

	ret = cgroup_apply_control_enable(cgrp);
	if (ret)
		return ret;

	/*
	 * At this point, cgroup_e_css() results reflect the new csses
	 * making the following cgroup_update_dfl_csses() properly update
	 * css associations of all tasks in the subtree.
	 */
	ret = cgroup_update_dfl_csses(cgrp);
	if (ret)
		return ret;

	return 0;
}

/**
 * cgroup_finalize_control - finalize control mask update
 * @cgrp: root of the target subtree
 * @ret: the result of the update
 *
 * Finalize control mask update.  See cgroup_apply_control() for more info.
 */
static void cgroup_finalize_control(struct cgroup *cgrp, int ret)
{
	if (ret) {
		cgroup_restore_control(cgrp);
		cgroup_propagate_control(cgrp);
	}

	cgroup_apply_control_disable(cgrp);
}

3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115
static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable)
{
	u16 domain_enable = enable & ~cgrp_dfl_threaded_ss_mask;

	/* if nothing is getting enabled, nothing to worry about */
	if (!enable)
		return 0;

	/* can @cgrp host any resources? */
	if (!cgroup_is_valid_domain(cgrp->dom_cgrp))
		return -EOPNOTSUPP;

	/* mixables don't care */
	if (cgroup_is_mixable(cgrp))
		return 0;

	if (domain_enable) {
		/* can't enable domain controllers inside a thread subtree */
		if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp))
			return -EOPNOTSUPP;
	} else {
		/*
		 * Threaded controllers can handle internal competitions
		 * and are always allowed inside a (prospective) thread
		 * subtree.
		 */
		if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp))
			return 0;
	}

	/*
	 * Controllers can't be enabled for a cgroup with tasks to avoid
	 * child cgroups competing against tasks.
	 */
	if (cgroup_has_tasks(cgrp))
		return -EBUSY;

	return 0;
}

3116
/* change the enabled child controllers for a cgroup in the default hierarchy */
3117 3118 3119
static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
3120
{
3121
	u16 enable = 0, disable = 0;
3122
	struct cgroup *cgrp, *child;
3123
	struct cgroup_subsys *ss;
3124
	char *tok;
3125 3126 3127
	int ssid, ret;

	/*
3128 3129
	 * Parse input - space separated list of subsystem names prefixed
	 * with either + or -.
3130
	 */
3131 3132
	buf = strstrip(buf);
	while ((tok = strsep(&buf, " "))) {
3133 3134
		if (tok[0] == '\0')
			continue;
T
Tejun Heo 已提交
3135
		do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) {
3136 3137
			if (!cgroup_ssid_enabled(ssid) ||
			    strcmp(tok + 1, ss->name))
3138 3139 3140
				continue;

			if (*tok == '+') {
3141 3142
				enable |= 1 << ssid;
				disable &= ~(1 << ssid);
3143
			} else if (*tok == '-') {
3144 3145
				disable |= 1 << ssid;
				enable &= ~(1 << ssid);
3146 3147 3148 3149
			} else {
				return -EINVAL;
			}
			break;
3150
		} while_each_subsys_mask();
3151 3152 3153 3154
		if (ssid == CGROUP_SUBSYS_COUNT)
			return -EINVAL;
	}

3155
	cgrp = cgroup_kn_lock_live(of->kn, true);
3156 3157
	if (!cgrp)
		return -ENODEV;
3158 3159 3160

	for_each_subsys(ss, ssid) {
		if (enable & (1 << ssid)) {
3161
			if (cgrp->subtree_control & (1 << ssid)) {
3162 3163 3164 3165
				enable &= ~(1 << ssid);
				continue;
			}

3166
			if (!(cgroup_control(cgrp) & (1 << ssid))) {
3167 3168 3169
				ret = -ENOENT;
				goto out_unlock;
			}
3170
		} else if (disable & (1 << ssid)) {
3171
			if (!(cgrp->subtree_control & (1 << ssid))) {
3172 3173 3174 3175 3176 3177
				disable &= ~(1 << ssid);
				continue;
			}

			/* a child has it enabled? */
			cgroup_for_each_live_child(child, cgrp) {
3178
				if (child->subtree_control & (1 << ssid)) {
3179
					ret = -EBUSY;
3180
					goto out_unlock;
3181 3182 3183 3184 3185 3186 3187
				}
			}
		}
	}

	if (!enable && !disable) {
		ret = 0;
3188
		goto out_unlock;
3189 3190
	}

3191 3192
	ret = cgroup_vet_subtree_control_enable(cgrp, enable);
	if (ret)
3193
		goto out_unlock;
3194

3195 3196
	/* save and update control masks and prepare csses */
	cgroup_save_control(cgrp);
3197

3198 3199
	cgrp->subtree_control |= enable;
	cgrp->subtree_control &= ~disable;
3200

3201 3202
	ret = cgroup_apply_control(cgrp);
	cgroup_finalize_control(cgrp, ret);
3203 3204
	if (ret)
		goto out_unlock;
3205 3206 3207

	kernfs_activate(cgrp->kn);
out_unlock:
3208
	cgroup_kn_unlock(of->kn);
3209
	return ret ?: nbytes;
3210 3211
}

3212 3213 3214 3215 3216 3217 3218 3219 3220
/**
 * cgroup_enable_threaded - make @cgrp threaded
 * @cgrp: the target cgroup
 *
 * Called when "threaded" is written to the cgroup.type interface file and
 * tries to make @cgrp threaded and join the parent's resource domain.
 * This function is never called on the root cgroup as cgroup.type doesn't
 * exist on it.
 */
3221 3222 3223 3224
static int cgroup_enable_threaded(struct cgroup *cgrp)
{
	struct cgroup *parent = cgroup_parent(cgrp);
	struct cgroup *dom_cgrp = parent->dom_cgrp;
3225 3226
	struct cgroup *dsct;
	struct cgroup_subsys_state *d_css;
3227 3228 3229 3230 3231 3232 3233 3234
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* noop if already threaded */
	if (cgroup_is_threaded(cgrp))
		return 0;

3235 3236 3237 3238 3239 3240 3241 3242 3243 3244
	/*
	 * If @cgroup is populated or has domain controllers enabled, it
	 * can't be switched.  While the below cgroup_can_be_thread_root()
	 * test can catch the same conditions, that's only when @parent is
	 * not mixable, so let's check it explicitly.
	 */
	if (cgroup_is_populated(cgrp) ||
	    cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)
		return -EOPNOTSUPP;

3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255
	/* we're joining the parent's domain, ensure its validity */
	if (!cgroup_is_valid_domain(dom_cgrp) ||
	    !cgroup_can_be_thread_root(dom_cgrp))
		return -EOPNOTSUPP;

	/*
	 * The following shouldn't cause actual migrations and should
	 * always succeed.
	 */
	cgroup_save_control(cgrp);

3256 3257 3258 3259
	cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp)
		if (dsct == cgrp || cgroup_is_threaded(dsct))
			dsct->dom_cgrp = dom_cgrp;

3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304
	ret = cgroup_apply_control(cgrp);
	if (!ret)
		parent->nr_threaded_children++;

	cgroup_finalize_control(cgrp, ret);
	return ret;
}

static int cgroup_type_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;

	if (cgroup_is_threaded(cgrp))
		seq_puts(seq, "threaded\n");
	else if (!cgroup_is_valid_domain(cgrp))
		seq_puts(seq, "domain invalid\n");
	else if (cgroup_is_thread_root(cgrp))
		seq_puts(seq, "domain threaded\n");
	else
		seq_puts(seq, "domain\n");

	return 0;
}

static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	int ret;

	/* only switching to threaded mode is supported */
	if (strcmp(strstrip(buf), "threaded"))
		return -EINVAL;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	/* threaded can only be enabled */
	ret = cgroup_enable_threaded(cgrp);

	cgroup_kn_unlock(of->kn);
	return ret ?: nbytes;
}

3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333
static int cgroup_max_descendants_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;
	int descendants = READ_ONCE(cgrp->max_descendants);

	if (descendants == INT_MAX)
		seq_puts(seq, "max\n");
	else
		seq_printf(seq, "%d\n", descendants);

	return 0;
}

static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of,
					   char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	int descendants;
	ssize_t ret;

	buf = strstrip(buf);
	if (!strcmp(buf, "max")) {
		descendants = INT_MAX;
	} else {
		ret = kstrtoint(buf, 0, &descendants);
		if (ret)
			return ret;
	}

D
Dan Carpenter 已提交
3334
	if (descendants < 0)
3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376
		return -ERANGE;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	cgrp->max_descendants = descendants;

	cgroup_kn_unlock(of->kn);

	return nbytes;
}

static int cgroup_max_depth_show(struct seq_file *seq, void *v)
{
	struct cgroup *cgrp = seq_css(seq)->cgroup;
	int depth = READ_ONCE(cgrp->max_depth);

	if (depth == INT_MAX)
		seq_puts(seq, "max\n");
	else
		seq_printf(seq, "%d\n", depth);

	return 0;
}

static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of,
				      char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *cgrp;
	ssize_t ret;
	int depth;

	buf = strstrip(buf);
	if (!strcmp(buf, "max")) {
		depth = INT_MAX;
	} else {
		ret = kstrtoint(buf, 0, &depth);
		if (ret)
			return ret;
	}

D
Dan Carpenter 已提交
3377
	if (depth < 0)
3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390
		return -ERANGE;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENOENT;

	cgrp->max_depth = depth;

	cgroup_kn_unlock(of->kn);

	return nbytes;
}

3391
static int cgroup_events_show(struct seq_file *seq, void *v)
3392
{
3393
	seq_printf(seq, "populated %d\n",
3394
		   cgroup_is_populated(seq_css(seq)->cgroup));
3395 3396 3397
	return 0;
}

T
Tejun Heo 已提交
3398
static int cgroup_stat_show(struct seq_file *seq, void *v)
3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409
{
	struct cgroup *cgroup = seq_css(seq)->cgroup;

	seq_printf(seq, "nr_descendants %d\n",
		   cgroup->nr_descendants);
	seq_printf(seq, "nr_dying_descendants %d\n",
		   cgroup->nr_dying_descendants);

	return 0;
}

3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430
static int __maybe_unused cgroup_extra_stat_show(struct seq_file *seq,
						 struct cgroup *cgrp, int ssid)
{
	struct cgroup_subsys *ss = cgroup_subsys[ssid];
	struct cgroup_subsys_state *css;
	int ret;

	if (!ss->css_extra_stat_show)
		return 0;

	css = cgroup_tryget_css(cgrp, ss);
	if (!css)
		return 0;

	ret = ss->css_extra_stat_show(seq, css);
	css_put(css);
	return ret;
}

static int cpu_stat_show(struct seq_file *seq, void *v)
{
3431
	struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup;
3432 3433
	int ret = 0;

3434
	cgroup_base_stat_cputime_show(seq);
3435 3436 3437 3438 3439 3440
#ifdef CONFIG_CGROUP_SCHED
	ret = cgroup_extra_stat_show(seq, cgrp, cpu_cgrp_id);
#endif
	return ret;
}

J
Johannes Weiner 已提交
3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453
#ifdef CONFIG_PSI
static int cgroup_io_pressure_show(struct seq_file *seq, void *v)
{
	return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_IO);
}
static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)
{
	return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_MEM);
}
static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
{
	return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_CPU);
}
S
Suren Baghdasaryan 已提交
3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512

static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,
					  size_t nbytes, enum psi_res res)
{
	struct psi_trigger *new;
	struct cgroup *cgrp;

	cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!cgrp)
		return -ENODEV;

	cgroup_get(cgrp);
	cgroup_kn_unlock(of->kn);

	new = psi_trigger_create(&cgrp->psi, buf, nbytes, res);
	if (IS_ERR(new)) {
		cgroup_put(cgrp);
		return PTR_ERR(new);
	}

	psi_trigger_replace(&of->priv, new);

	cgroup_put(cgrp);

	return nbytes;
}

static ssize_t cgroup_io_pressure_write(struct kernfs_open_file *of,
					  char *buf, size_t nbytes,
					  loff_t off)
{
	return cgroup_pressure_write(of, buf, nbytes, PSI_IO);
}

static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,
					  char *buf, size_t nbytes,
					  loff_t off)
{
	return cgroup_pressure_write(of, buf, nbytes, PSI_MEM);
}

static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
					  char *buf, size_t nbytes,
					  loff_t off)
{
	return cgroup_pressure_write(of, buf, nbytes, PSI_CPU);
}

static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
					  poll_table *pt)
{
	return psi_trigger_poll(&of->priv, of->file, pt);
}

static void cgroup_pressure_release(struct kernfs_open_file *of)
{
	psi_trigger_replace(&of->priv, NULL);
}
#endif /* CONFIG_PSI */
J
Johannes Weiner 已提交
3513

3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530
static int cgroup_file_open(struct kernfs_open_file *of)
{
	struct cftype *cft = of->kn->priv;

	if (cft->open)
		return cft->open(of);
	return 0;
}

static void cgroup_file_release(struct kernfs_open_file *of)
{
	struct cftype *cft = of->kn->priv;

	if (cft->release)
		cft->release(of);
}

T
Tejun Heo 已提交
3531 3532
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
				 size_t nbytes, loff_t off)
3533
{
3534
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
T
Tejun Heo 已提交
3535 3536 3537
	struct cgroup *cgrp = of->kn->parent->priv;
	struct cftype *cft = of->kn->priv;
	struct cgroup_subsys_state *css;
3538
	int ret;
3539

3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550
	/*
	 * If namespaces are delegation boundaries, disallow writes to
	 * files in an non-init namespace root from inside the namespace
	 * except for the files explicitly marked delegatable -
	 * cgroup.procs and cgroup.subtree_control.
	 */
	if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) &&
	    !(cft->flags & CFTYPE_NS_DELEGATABLE) &&
	    ns != &init_cgroup_ns && ns->root_cset->dfl_cgrp == cgrp)
		return -EPERM;

T
Tejun Heo 已提交
3551 3552 3553
	if (cft->write)
		return cft->write(of, buf, nbytes, off);

T
Tejun Heo 已提交
3554 3555 3556 3557 3558 3559 3560 3561 3562
	/*
	 * kernfs guarantees that a file isn't deleted with operations in
	 * flight, which means that the matching css is and stays alive and
	 * doesn't need to be pinned.  The RCU locking is not necessary
	 * either.  It's just for the convenience of using cgroup_css().
	 */
	rcu_read_lock();
	css = cgroup_css(cgrp, cft->ss);
	rcu_read_unlock();
3563

3564
	if (cft->write_u64) {
3565 3566 3567 3568 3569 3570 3571 3572 3573
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
3574
	} else {
3575
		ret = -EINVAL;
3576
	}
T
Tejun Heo 已提交
3577

3578
	return ret ?: nbytes;
3579 3580
}

3581 3582 3583 3584 3585 3586 3587 3588 3589 3590
static __poll_t cgroup_file_poll(struct kernfs_open_file *of, poll_table *pt)
{
	struct cftype *cft = of->kn->priv;

	if (cft->poll)
		return cft->poll(of, pt);

	return kernfs_generic_poll(of, pt);
}

3591
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
3592
{
T
Tejun Heo 已提交
3593
	return seq_cft(seq)->seq_start(seq, ppos);
3594 3595
}

3596
static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
3597
{
T
Tejun Heo 已提交
3598
	return seq_cft(seq)->seq_next(seq, v, ppos);
3599 3600
}

3601
static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
3602
{
3603 3604
	if (seq_cft(seq)->seq_stop)
		seq_cft(seq)->seq_stop(seq, v);
3605 3606
}

3607
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
3608
{
3609 3610
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
3611

3612 3613
	if (cft->seq_show)
		return cft->seq_show(m, arg);
3614

3615
	if (cft->read_u64)
3616 3617 3618 3619 3620 3621
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
3622 3623
}

T
Tejun Heo 已提交
3624 3625
static struct kernfs_ops cgroup_kf_single_ops = {
	.atomic_write_len	= PAGE_SIZE,
3626 3627
	.open			= cgroup_file_open,
	.release		= cgroup_file_release,
T
Tejun Heo 已提交
3628
	.write			= cgroup_file_write,
3629
	.poll			= cgroup_file_poll,
T
Tejun Heo 已提交
3630
	.seq_show		= cgroup_seqfile_show,
3631 3632
};

T
Tejun Heo 已提交
3633 3634
static struct kernfs_ops cgroup_kf_ops = {
	.atomic_write_len	= PAGE_SIZE,
3635 3636
	.open			= cgroup_file_open,
	.release		= cgroup_file_release,
T
Tejun Heo 已提交
3637
	.write			= cgroup_file_write,
3638
	.poll			= cgroup_file_poll,
T
Tejun Heo 已提交
3639 3640 3641 3642 3643
	.seq_start		= cgroup_seqfile_start,
	.seq_next		= cgroup_seqfile_next,
	.seq_stop		= cgroup_seqfile_stop,
	.seq_show		= cgroup_seqfile_show,
};
3644

3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658
/* set uid and gid of cgroup dirs and files to that of the creator */
static int cgroup_kn_set_ugid(struct kernfs_node *kn)
{
	struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
			       .ia_uid = current_fsuid(),
			       .ia_gid = current_fsgid(), };

	if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
	    gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
		return 0;

	return kernfs_setattr(kn, &iattr);
}

3659 3660 3661 3662 3663 3664
static void cgroup_file_notify_timer(struct timer_list *timer)
{
	cgroup_file_notify(container_of(timer, struct cgroup_file,
					notify_timer));
}

3665 3666
static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
			   struct cftype *cft)
3667
{
T
Tejun Heo 已提交
3668
	char name[CGROUP_FILE_NAME_MAX];
T
Tejun Heo 已提交
3669 3670
	struct kernfs_node *kn;
	struct lock_class_key *key = NULL;
3671
	int ret;
T
Tejun Heo 已提交
3672

T
Tejun Heo 已提交
3673 3674 3675 3676
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	key = &cft->lockdep_key;
#endif
	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
3677 3678 3679
				  cgroup_file_mode(cft),
				  GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
				  0, cft->kf_ops, cft,
T
Tejun Heo 已提交
3680
				  NULL, key);
3681 3682 3683 3684
	if (IS_ERR(kn))
		return PTR_ERR(kn);

	ret = cgroup_kn_set_ugid(kn);
3685
	if (ret) {
3686
		kernfs_remove(kn);
3687 3688 3689
		return ret;
	}

3690 3691 3692
	if (cft->file_offset) {
		struct cgroup_file *cfile = (void *)css + cft->file_offset;

3693 3694
		timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0);

3695
		spin_lock_irq(&cgroup_file_kn_lock);
3696
		cfile->kn = kn;
3697
		spin_unlock_irq(&cgroup_file_kn_lock);
3698 3699
	}

3700
	return 0;
3701 3702
}

3703 3704
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
3705 3706
 * @css: the target css
 * @cgrp: the target cgroup (usually css->cgroup)
3707 3708 3709 3710
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3711
 * For removals, this function never fails.
3712
 */
3713 3714
static int cgroup_addrm_files(struct cgroup_subsys_state *css,
			      struct cgroup *cgrp, struct cftype cfts[],
3715
			      bool is_add)
3716
{
3717
	struct cftype *cft, *cft_end = NULL;
3718
	int ret = 0;
3719

3720
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
3721

3722 3723
restart:
	for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {
3724
		/* does cft->flags tell us to skip this file on @cgrp? */
3725
		if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
T
Tejun Heo 已提交
3726
			continue;
3727
		if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
3728
			continue;
T
Tejun Heo 已提交
3729
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
3730
			continue;
T
Tejun Heo 已提交
3731
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
3732 3733
			continue;

3734
		if (is_add) {
3735
			ret = cgroup_add_file(css, cgrp, cft);
3736
			if (ret) {
3737 3738
				pr_warn("%s: failed to add %s, err=%d\n",
					__func__, cft->name, ret);
3739 3740 3741
				cft_end = cft;
				is_add = false;
				goto restart;
3742
			}
3743 3744
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
3745
		}
3746
	}
3747
	return ret;
3748 3749
}

3750
static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
3751
{
3752
	struct cgroup_subsys *ss = cfts[0].ss;
3753
	struct cgroup *root = &ss->root->cgrp;
3754
	struct cgroup_subsys_state *css;
3755
	int ret = 0;
3756

3757
	lockdep_assert_held(&cgroup_mutex);
3758 3759

	/* add/rm files for all cgroups created before */
3760
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
3761 3762
		struct cgroup *cgrp = css->cgroup;

3763
		if (!(css->flags & CSS_VISIBLE))
3764 3765
			continue;

3766
		ret = cgroup_addrm_files(css, cgrp, cfts, is_add);
3767 3768
		if (ret)
			break;
3769
	}
3770 3771 3772

	if (is_add && !ret)
		kernfs_activate(root->kn);
3773
	return ret;
3774 3775
}

3776
static void cgroup_exit_cftypes(struct cftype *cfts)
3777
{
3778
	struct cftype *cft;
3779

T
Tejun Heo 已提交
3780 3781 3782 3783 3784
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		/* free copy for custom atomic_write_len, see init_cftypes() */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
			kfree(cft->kf_ops);
		cft->kf_ops = NULL;
3785
		cft->ss = NULL;
3786 3787

		/* revert flags set by cgroup core while adding @cfts */
3788
		cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
T
Tejun Heo 已提交
3789
	}
3790 3791
}

T
Tejun Heo 已提交
3792
static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3793 3794 3795
{
	struct cftype *cft;

T
Tejun Heo 已提交
3796 3797 3798
	for (cft = cfts; cft->name[0] != '\0'; cft++) {
		struct kernfs_ops *kf_ops;

T
Tejun Heo 已提交
3799 3800
		WARN_ON(cft->ss || cft->kf_ops);

T
Tejun Heo 已提交
3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817
		if (cft->seq_start)
			kf_ops = &cgroup_kf_ops;
		else
			kf_ops = &cgroup_kf_single_ops;

		/*
		 * Ugh... if @cft wants a custom max_write_len, we need to
		 * make a copy of kf_ops to set its atomic_write_len.
		 */
		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
			kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
			if (!kf_ops) {
				cgroup_exit_cftypes(cfts);
				return -ENOMEM;
			}
			kf_ops->atomic_write_len = cft->max_write_len;
		}
3818

T
Tejun Heo 已提交
3819
		cft->kf_ops = kf_ops;
3820
		cft->ss = ss;
T
Tejun Heo 已提交
3821
	}
3822

T
Tejun Heo 已提交
3823
	return 0;
3824 3825
}

3826 3827
static int cgroup_rm_cftypes_locked(struct cftype *cfts)
{
3828
	lockdep_assert_held(&cgroup_mutex);
3829 3830 3831 3832 3833 3834 3835 3836

	if (!cfts || !cfts[0].ss)
		return -ENOENT;

	list_del(&cfts->node);
	cgroup_apply_cftypes(cfts, false);
	cgroup_exit_cftypes(cfts);
	return 0;
3837 3838
}

3839 3840 3841 3842
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
3843 3844 3845
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
3846 3847
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3848
 * registered.
3849
 */
3850
int cgroup_rm_cftypes(struct cftype *cfts)
3851
{
3852
	int ret;
3853

3854
	mutex_lock(&cgroup_mutex);
3855
	ret = cgroup_rm_cftypes_locked(cfts);
3856
	mutex_unlock(&cgroup_mutex);
3857
	return ret;
T
Tejun Heo 已提交
3858 3859
}

3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873
/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
3874
static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3875
{
3876
	int ret;
3877

3878
	if (!cgroup_ssid_enabled(ss->id))
3879 3880
		return 0;

3881 3882
	if (!cfts || cfts[0].name[0] == '\0')
		return 0;
3883

T
Tejun Heo 已提交
3884 3885 3886
	ret = cgroup_init_cftypes(ss, cfts);
	if (ret)
		return ret;
3887

3888
	mutex_lock(&cgroup_mutex);
3889

T
Tejun Heo 已提交
3890
	list_add_tail(&cfts->node, &ss->cfts);
3891
	ret = cgroup_apply_cftypes(cfts, true);
3892
	if (ret)
3893
		cgroup_rm_cftypes_locked(cfts);
3894

3895
	mutex_unlock(&cgroup_mutex);
3896
	return ret;
3897 3898
}

3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911
/**
 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the default hierarchy.
 */
int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
	struct cftype *cft;

	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3912
		cft->flags |= __CFTYPE_ONLY_ON_DFL;
3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
	return cgroup_add_cftypes(ss, cfts);
}

/**
 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Similar to cgroup_add_cftypes() but the added files are only used for
 * the legacy hierarchies.
 */
3924 3925
int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
{
3926 3927
	struct cftype *cft;

3928 3929
	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
		cft->flags |= __CFTYPE_NOT_ON_DFL;
3930 3931 3932
	return cgroup_add_cftypes(ss, cfts);
}

3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
/**
 * cgroup_file_notify - generate a file modified event for a cgroup_file
 * @cfile: target cgroup_file
 *
 * @cfile must have been obtained by setting cftype->file_offset.
 */
void cgroup_file_notify(struct cgroup_file *cfile)
{
	unsigned long flags;

	spin_lock_irqsave(&cgroup_file_kn_lock, flags);
3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954
	if (cfile->kn) {
		unsigned long last = cfile->notified_at;
		unsigned long next = last + CGROUP_FILE_NOTIFY_MIN_INTV;

		if (time_in_range(jiffies, last, next)) {
			timer_reduce(&cfile->notify_timer, next);
		} else {
			kernfs_notify(cfile->kn);
			cfile->notified_at = jiffies;
		}
	}
3955 3956 3957
	spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
}

3958
/**
3959
 * css_next_child - find the next child of a given css
3960 3961
 * @pos: the current position (%NULL to initiate traversal)
 * @parent: css whose children to walk
3962
 *
3963
 * This function returns the next child of @parent and should be called
3964
 * under either cgroup_mutex or RCU read lock.  The only requirement is
3965 3966 3967 3968 3969 3970 3971 3972 3973
 * that @parent and @pos are accessible.  The next sibling is guaranteed to
 * be returned regardless of their states.
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
3974
 */
3975 3976
struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
					   struct cgroup_subsys_state *parent)
3977
{
3978
	struct cgroup_subsys_state *next;
3979

T
Tejun Heo 已提交
3980
	cgroup_assert_mutex_or_rcu_locked();
3981 3982

	/*
3983 3984 3985 3986 3987 3988 3989 3990 3991 3992
	 * @pos could already have been unlinked from the sibling list.
	 * Once a cgroup is removed, its ->sibling.next is no longer
	 * updated when its next sibling changes.  CSS_RELEASED is set when
	 * @pos is taken off list, at which time its next pointer is valid,
	 * and, as releases are serialized, the one pointed to by the next
	 * pointer is guaranteed to not have started release yet.  This
	 * implies that if we observe !CSS_RELEASED on @pos in this RCU
	 * critical section, the one pointed to by its next pointer is
	 * guaranteed to not have finished its RCU grace period even if we
	 * have dropped rcu_read_lock() inbetween iterations.
3993
	 *
3994 3995 3996 3997 3998 3999 4000
	 * If @pos has CSS_RELEASED set, its next pointer can't be
	 * dereferenced; however, as each css is given a monotonically
	 * increasing unique serial number and always appended to the
	 * sibling list, the next one can be found by walking the parent's
	 * children until the first css with higher serial number than
	 * @pos's.  While this path can be slower, it happens iff iteration
	 * races against release and the race window is very small.
4001
	 */
4002
	if (!pos) {
4003 4004 4005
		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
	} else if (likely(!(pos->flags & CSS_RELEASED))) {
		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
4006
	} else {
4007
		list_for_each_entry_rcu(next, &parent->children, sibling)
4008 4009
			if (next->serial_nr > pos->serial_nr)
				break;
4010 4011
	}

4012 4013
	/*
	 * @next, if not pointing to the head, can be dereferenced and is
4014
	 * the next sibling.
4015
	 */
4016 4017
	if (&next->sibling != &parent->children)
		return next;
4018
	return NULL;
4019 4020
}

4021
/**
4022
 * css_next_descendant_pre - find the next descendant for pre-order walk
4023
 * @pos: the current position (%NULL to initiate traversal)
4024
 * @root: css whose descendants to walk
4025
 *
4026
 * To be used by css_for_each_descendant_pre().  Find the next descendant
4027 4028
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
4029
 *
4030 4031 4032 4033
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
4034 4035 4036 4037 4038 4039 4040
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
4041
 */
4042 4043 4044
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
4045
{
4046
	struct cgroup_subsys_state *next;
4047

T
Tejun Heo 已提交
4048
	cgroup_assert_mutex_or_rcu_locked();
4049

4050
	/* if first iteration, visit @root */
4051
	if (!pos)
4052
		return root;
4053 4054

	/* visit the first child if exists */
4055
	next = css_next_child(NULL, pos);
4056 4057 4058 4059
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
4060
	while (pos != root) {
T
Tejun Heo 已提交
4061
		next = css_next_child(pos, pos->parent);
4062
		if (next)
4063
			return next;
T
Tejun Heo 已提交
4064
		pos = pos->parent;
4065
	}
4066 4067 4068 4069

	return NULL;
}

4070
/**
4071 4072
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
4073
 *
4074 4075
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
4076
 * subtree of @pos.
4077
 *
4078 4079 4080 4081
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
4082
 */
4083 4084
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
4085
{
4086
	struct cgroup_subsys_state *last, *tmp;
4087

T
Tejun Heo 已提交
4088
	cgroup_assert_mutex_or_rcu_locked();
4089 4090 4091 4092 4093

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
4094
		css_for_each_child(tmp, last)
4095 4096 4097 4098 4099 4100
			pos = tmp;
	} while (pos);

	return last;
}

4101 4102
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
4103
{
4104
	struct cgroup_subsys_state *last;
4105 4106 4107

	do {
		last = pos;
4108
		pos = css_next_child(NULL, pos);
4109 4110 4111 4112 4113 4114
	} while (pos);

	return last;
}

/**
4115
 * css_next_descendant_post - find the next descendant for post-order walk
4116
 * @pos: the current position (%NULL to initiate traversal)
4117
 * @root: css whose descendants to walk
4118
 *
4119
 * To be used by css_for_each_descendant_post().  Find the next descendant
4120 4121
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
4122
 *
4123 4124 4125 4126 4127
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
4128 4129 4130 4131 4132 4133 4134
 *
 * If a subsystem synchronizes ->css_online() and the start of iteration, a
 * css which finished ->css_online() is guaranteed to be visible in the
 * future iterations and will stay visible until the last reference is put.
 * A css which hasn't finished ->css_online() or already finished
 * ->css_offline() may show up during traversal.  It's each subsystem's
 * responsibility to synchronize against on/offlining.
4135
 */
4136 4137 4138
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
4139
{
4140
	struct cgroup_subsys_state *next;
4141

T
Tejun Heo 已提交
4142
	cgroup_assert_mutex_or_rcu_locked();
4143

4144 4145 4146
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
4147

4148 4149 4150 4151
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

4152
	/* if there's an unvisited sibling, visit its leftmost descendant */
T
Tejun Heo 已提交
4153
	next = css_next_child(pos, pos->parent);
4154
	if (next)
4155
		return css_leftmost_descendant(next);
4156 4157

	/* no sibling left, visit parent */
T
Tejun Heo 已提交
4158
	return pos->parent;
4159 4160
}

4161 4162 4163 4164 4165 4166 4167 4168 4169
/**
 * css_has_online_children - does a css have online children
 * @css: the target css
 *
 * Returns %true if @css has any online children; otherwise, %false.  This
 * function can be called from any context but the caller is responsible
 * for synchronizing against on/offlining as necessary.
 */
bool css_has_online_children(struct cgroup_subsys_state *css)
4170
{
4171 4172
	struct cgroup_subsys_state *child;
	bool ret = false;
4173 4174

	rcu_read_lock();
4175
	css_for_each_child(child, css) {
4176
		if (child->flags & CSS_ONLINE) {
4177 4178
			ret = true;
			break;
4179 4180 4181
		}
	}
	rcu_read_unlock();
4182
	return ret;
4183 4184
}

4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
static struct css_set *css_task_iter_next_css_set(struct css_task_iter *it)
{
	struct list_head *l;
	struct cgrp_cset_link *link;
	struct css_set *cset;

	lockdep_assert_held(&css_set_lock);

	/* find the next threaded cset */
	if (it->tcset_pos) {
		l = it->tcset_pos->next;

		if (l != it->tcset_head) {
			it->tcset_pos = l;
			return container_of(l, struct css_set,
					    threaded_csets_node);
		}

		it->tcset_pos = NULL;
	}

	/* find the next cset */
	l = it->cset_pos;
	l = l->next;
	if (l == it->cset_head) {
		it->cset_pos = NULL;
		return NULL;
	}

	if (it->ss) {
		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);
	} else {
		link = list_entry(l, struct cgrp_cset_link, cset_link);
		cset = link->cset;
	}

	it->cset_pos = l;

	/* initialize threaded css_set walking */
	if (it->flags & CSS_TASK_ITER_THREADED) {
		if (it->cur_dcset)
			put_css_set_locked(it->cur_dcset);
		it->cur_dcset = cset;
		get_css_set(cset);

		it->tcset_head = &cset->threaded_csets;
		it->tcset_pos = &cset->threaded_csets;
	}

	return cset;
}

4237
/**
4238
 * css_task_iter_advance_css_set - advance a task itererator to the next css_set
4239 4240 4241
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
4242
 */
4243
static void css_task_iter_advance_css_set(struct css_task_iter *it)
4244 4245 4246
{
	struct css_set *cset;

4247
	lockdep_assert_held(&css_set_lock);
4248

4249 4250
	/* Advance to the next non-empty css_set */
	do {
4251 4252
		cset = css_task_iter_next_css_set(it);
		if (!cset) {
4253
			it->task_pos = NULL;
4254 4255
			return;
		}
4256
	} while (!css_set_populated(cset) && list_empty(&cset->dying_tasks));
T
Tejun Heo 已提交
4257 4258

	if (!list_empty(&cset->tasks))
T
Tejun Heo 已提交
4259
		it->task_pos = cset->tasks.next;
4260
	else if (!list_empty(&cset->mg_tasks))
T
Tejun Heo 已提交
4261
		it->task_pos = cset->mg_tasks.next;
4262 4263
	else
		it->task_pos = cset->dying_tasks.next;
T
Tejun Heo 已提交
4264 4265 4266

	it->tasks_head = &cset->tasks;
	it->mg_tasks_head = &cset->mg_tasks;
4267
	it->dying_tasks_head = &cset->dying_tasks;
4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290

	/*
	 * We don't keep css_sets locked across iteration steps and thus
	 * need to take steps to ensure that iteration can be resumed after
	 * the lock is re-acquired.  Iteration is performed at two levels -
	 * css_sets and tasks in them.
	 *
	 * Once created, a css_set never leaves its cgroup lists, so a
	 * pinned css_set is guaranteed to stay put and we can resume
	 * iteration afterwards.
	 *
	 * Tasks may leave @cset across iteration steps.  This is resolved
	 * by registering each iterator with the css_set currently being
	 * walked and making css_set_move_task() advance iterators whose
	 * next task is leaving.
	 */
	if (it->cur_cset) {
		list_del(&it->iters_node);
		put_css_set_locked(it->cur_cset);
	}
	get_css_set(cset);
	it->cur_cset = cset;
	list_add(&it->iters_node, &cset->task_iters);
4291 4292
}

4293 4294
static void css_task_iter_skip(struct css_task_iter *it,
			       struct task_struct *task)
4295
{
4296 4297 4298 4299 4300 4301 4302
	lockdep_assert_held(&css_set_lock);

	if (it->task_pos == &task->cg_list) {
		it->task_pos = it->task_pos->next;
		it->flags |= CSS_TASK_ITER_SKIPPED;
	}
}
4303

4304 4305
static void css_task_iter_advance(struct css_task_iter *it)
{
4306 4307
	struct task_struct *task;

4308
	lockdep_assert_held(&css_set_lock);
4309
repeat:
T
Tejun Heo 已提交
4310 4311 4312 4313 4314 4315
	if (it->task_pos) {
		/*
		 * Advance iterator to find next entry.  cset->tasks is
		 * consumed first and then ->mg_tasks.  After ->mg_tasks,
		 * we move onto the next cset.
		 */
4316 4317 4318 4319
		if (it->flags & CSS_TASK_ITER_SKIPPED)
			it->flags &= ~CSS_TASK_ITER_SKIPPED;
		else
			it->task_pos = it->task_pos->next;
4320

4321 4322 4323
		if (it->task_pos == it->tasks_head)
			it->task_pos = it->mg_tasks_head->next;
		if (it->task_pos == it->mg_tasks_head)
4324 4325
			it->task_pos = it->dying_tasks_head->next;
		if (it->task_pos == it->dying_tasks_head)
T
Tejun Heo 已提交
4326 4327 4328
			css_task_iter_advance_css_set(it);
	} else {
		/* called from start, proceed to the first cset */
4329
		css_task_iter_advance_css_set(it);
T
Tejun Heo 已提交
4330
	}
4331

4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349
	if (!it->task_pos)
		return;

	task = list_entry(it->task_pos, struct task_struct, cg_list);

	if (it->flags & CSS_TASK_ITER_PROCS) {
		/* if PROCS, skip over tasks which aren't group leaders */
		if (!thread_group_leader(task))
			goto repeat;

		/* and dying leaders w/o live member threads */
		if (!atomic_read(&task->signal->live))
			goto repeat;
	} else {
		/* skip all dying ones */
		if (task->flags & PF_EXITING)
			goto repeat;
	}
4350 4351
}

4352
/**
4353 4354
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
4355
 * @flags: CSS_TASK_ITER_* flags
4356 4357
 * @it: the task iterator to use
 *
4358 4359 4360 4361
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
4362
 */
4363
void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags,
4364
			 struct css_task_iter *it)
4365
{
4366 4367
	/* no one should try to iterate before mounting cgroups */
	WARN_ON_ONCE(!use_task_css_set_links);
4368

4369 4370
	memset(it, 0, sizeof(*it));

4371
	spin_lock_irq(&css_set_lock);
4372

4373
	it->ss = css->ss;
4374
	it->flags = flags;
4375 4376 4377 4378 4379 4380

	if (it->ss)
		it->cset_pos = &css->cgroup->e_csets[css->ss->id];
	else
		it->cset_pos = &css->cgroup->cset_links;

T
Tejun Heo 已提交
4381
	it->cset_head = it->cset_pos;
4382

T
Tejun Heo 已提交
4383
	css_task_iter_advance(it);
4384

4385
	spin_unlock_irq(&css_set_lock);
4386 4387
}

4388
/**
4389
 * css_task_iter_next - return the next task for the iterator
4390 4391 4392
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
4393 4394
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
4395
 */
4396
struct task_struct *css_task_iter_next(struct css_task_iter *it)
4397
{
4398
	if (it->cur_task) {
4399
		put_task_struct(it->cur_task);
4400 4401
		it->cur_task = NULL;
	}
4402

4403
	spin_lock_irq(&css_set_lock);
4404

4405 4406 4407 4408
	/* @it may be half-advanced by skips, finish advancing */
	if (it->flags & CSS_TASK_ITER_SKIPPED)
		css_task_iter_advance(it);

4409 4410 4411 4412 4413 4414
	if (it->task_pos) {
		it->cur_task = list_entry(it->task_pos, struct task_struct,
					  cg_list);
		get_task_struct(it->cur_task);
		css_task_iter_advance(it);
	}
4415

4416
	spin_unlock_irq(&css_set_lock);
4417 4418

	return it->cur_task;
4419 4420
}

4421
/**
4422
 * css_task_iter_end - finish task iteration
4423 4424
 * @it: the task iterator to finish
 *
4425
 * Finish task iteration started by css_task_iter_start().
4426
 */
4427
void css_task_iter_end(struct css_task_iter *it)
4428
{
4429
	if (it->cur_cset) {
4430
		spin_lock_irq(&css_set_lock);
4431 4432
		list_del(&it->iters_node);
		put_css_set_locked(it->cur_cset);
4433
		spin_unlock_irq(&css_set_lock);
4434 4435
	}

4436 4437 4438
	if (it->cur_dcset)
		put_css_set(it->cur_dcset);

4439 4440
	if (it->cur_task)
		put_task_struct(it->cur_task);
4441 4442
}

4443
static void cgroup_procs_release(struct kernfs_open_file *of)
4444
{
4445 4446 4447 4448 4449
	if (of->priv) {
		css_task_iter_end(of->priv);
		kfree(of->priv);
	}
}
4450

4451 4452 4453 4454
static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos)
{
	struct kernfs_open_file *of = s->private;
	struct css_task_iter *it = of->priv;
4455

4456
	return css_task_iter_next(it);
4457
}
4458

4459 4460
static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos,
				  unsigned int iter_flags)
4461 4462 4463 4464
{
	struct kernfs_open_file *of = s->private;
	struct cgroup *cgrp = seq_css(s)->cgroup;
	struct css_task_iter *it = of->priv;
4465

4466
	/*
4467 4468
	 * When a seq_file is seeked, it's always traversed sequentially
	 * from position 0, so we can simply keep iterating on !0 *pos.
4469
	 */
4470 4471 4472
	if (!it) {
		if (WARN_ON_ONCE((*pos)++))
			return ERR_PTR(-EINVAL);
4473

4474 4475 4476 4477
		it = kzalloc(sizeof(*it), GFP_KERNEL);
		if (!it)
			return ERR_PTR(-ENOMEM);
		of->priv = it;
4478
		css_task_iter_start(&cgrp->self, iter_flags, it);
4479 4480
	} else if (!(*pos)++) {
		css_task_iter_end(it);
4481
		css_task_iter_start(&cgrp->self, iter_flags, it);
4482
	}
4483

4484 4485
	return cgroup_procs_next(s, NULL, NULL);
}
4486

4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503
static void *cgroup_procs_start(struct seq_file *s, loff_t *pos)
{
	struct cgroup *cgrp = seq_css(s)->cgroup;

	/*
	 * All processes of a threaded subtree belong to the domain cgroup
	 * of the subtree.  Only threads can be distributed across the
	 * subtree.  Reject reads on cgroup.procs in the subtree proper.
	 * They're always empty anyway.
	 */
	if (cgroup_is_threaded(cgrp))
		return ERR_PTR(-EOPNOTSUPP);

	return __cgroup_procs_start(s, pos, CSS_TASK_ITER_PROCS |
					    CSS_TASK_ITER_THREADED);
}

4504
static int cgroup_procs_show(struct seq_file *s, void *v)
4505
{
4506
	seq_printf(s, "%d\n", task_pid_vnr(v));
4507 4508 4509
	return 0;
}

4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582
static int cgroup_procs_write_permission(struct cgroup *src_cgrp,
					 struct cgroup *dst_cgrp,
					 struct super_block *sb)
{
	struct cgroup_namespace *ns = current->nsproxy->cgroup_ns;
	struct cgroup *com_cgrp = src_cgrp;
	struct inode *inode;
	int ret;

	lockdep_assert_held(&cgroup_mutex);

	/* find the common ancestor */
	while (!cgroup_is_descendant(dst_cgrp, com_cgrp))
		com_cgrp = cgroup_parent(com_cgrp);

	/* %current should be authorized to migrate to the common ancestor */
	inode = kernfs_get_inode(sb, com_cgrp->procs_file.kn);
	if (!inode)
		return -ENOMEM;

	ret = inode_permission(inode, MAY_WRITE);
	iput(inode);
	if (ret)
		return ret;

	/*
	 * If namespaces are delegation boundaries, %current must be able
	 * to see both source and destination cgroups from its namespace.
	 */
	if ((cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE) &&
	    (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) ||
	     !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp)))
		return -ENOENT;

	return 0;
}

static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
				  char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *src_cgrp, *dst_cgrp;
	struct task_struct *task;
	ssize_t ret;

	dst_cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!dst_cgrp)
		return -ENODEV;

	task = cgroup_procs_write_start(buf, true);
	ret = PTR_ERR_OR_ZERO(task);
	if (ret)
		goto out_unlock;

	/* find the source cgroup */
	spin_lock_irq(&css_set_lock);
	src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
	spin_unlock_irq(&css_set_lock);

	ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp,
					    of->file->f_path.dentry->d_sb);
	if (ret)
		goto out_finish;

	ret = cgroup_attach_task(dst_cgrp, task, true);

out_finish:
	cgroup_procs_write_finish(task);
out_unlock:
	cgroup_kn_unlock(of->kn);

	return ret ?: nbytes;
}

4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631
static void *cgroup_threads_start(struct seq_file *s, loff_t *pos)
{
	return __cgroup_procs_start(s, pos, 0);
}

static ssize_t cgroup_threads_write(struct kernfs_open_file *of,
				    char *buf, size_t nbytes, loff_t off)
{
	struct cgroup *src_cgrp, *dst_cgrp;
	struct task_struct *task;
	ssize_t ret;

	buf = strstrip(buf);

	dst_cgrp = cgroup_kn_lock_live(of->kn, false);
	if (!dst_cgrp)
		return -ENODEV;

	task = cgroup_procs_write_start(buf, false);
	ret = PTR_ERR_OR_ZERO(task);
	if (ret)
		goto out_unlock;

	/* find the source cgroup */
	spin_lock_irq(&css_set_lock);
	src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
	spin_unlock_irq(&css_set_lock);

	/* thread migrations follow the cgroup.procs delegation rule */
	ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp,
					    of->file->f_path.dentry->d_sb);
	if (ret)
		goto out_finish;

	/* and must be contained in the same domain */
	ret = -EOPNOTSUPP;
	if (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp)
		goto out_finish;

	ret = cgroup_attach_task(dst_cgrp, task, false);

out_finish:
	cgroup_procs_write_finish(task);
out_unlock:
	cgroup_kn_unlock(of->kn);

	return ret ?: nbytes;
}

4632
/* cgroup core interface files for the default hierarchy */
4633
static struct cftype cgroup_base_files[] = {
4634 4635 4636 4637 4638 4639
	{
		.name = "cgroup.type",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_type_show,
		.write = cgroup_type_write,
	},
4640
	{
4641
		.name = "cgroup.procs",
4642
		.flags = CFTYPE_NS_DELEGATABLE,
4643
		.file_offset = offsetof(struct cgroup, procs_file),
4644 4645 4646 4647
		.release = cgroup_procs_release,
		.seq_start = cgroup_procs_start,
		.seq_next = cgroup_procs_next,
		.seq_show = cgroup_procs_show,
4648
		.write = cgroup_procs_write,
4649
	},
4650 4651
	{
		.name = "cgroup.threads",
4652
		.flags = CFTYPE_NS_DELEGATABLE,
4653 4654 4655 4656 4657 4658
		.release = cgroup_procs_release,
		.seq_start = cgroup_threads_start,
		.seq_next = cgroup_procs_next,
		.seq_show = cgroup_procs_show,
		.write = cgroup_threads_write,
	},
4659 4660 4661 4662 4663 4664
	{
		.name = "cgroup.controllers",
		.seq_show = cgroup_controllers_show,
	},
	{
		.name = "cgroup.subtree_control",
4665
		.flags = CFTYPE_NS_DELEGATABLE,
4666
		.seq_show = cgroup_subtree_control_show,
4667
		.write = cgroup_subtree_control_write,
4668
	},
4669
	{
4670
		.name = "cgroup.events",
4671
		.flags = CFTYPE_NOT_ON_ROOT,
4672
		.file_offset = offsetof(struct cgroup, events_file),
4673
		.seq_show = cgroup_events_show,
4674
	},
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684
	{
		.name = "cgroup.max.descendants",
		.seq_show = cgroup_max_descendants_show,
		.write = cgroup_max_descendants_write,
	},
	{
		.name = "cgroup.max.depth",
		.seq_show = cgroup_max_depth_show,
		.write = cgroup_max_depth_write,
	},
4685 4686
	{
		.name = "cgroup.stat",
T
Tejun Heo 已提交
4687
		.seq_show = cgroup_stat_show,
4688
	},
4689 4690 4691 4692 4693
	{
		.name = "cpu.stat",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cpu_stat_show,
	},
J
Johannes Weiner 已提交
4694 4695 4696 4697 4698
#ifdef CONFIG_PSI
	{
		.name = "io.pressure",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_io_pressure_show,
S
Suren Baghdasaryan 已提交
4699 4700 4701
		.write = cgroup_io_pressure_write,
		.poll = cgroup_pressure_poll,
		.release = cgroup_pressure_release,
J
Johannes Weiner 已提交
4702 4703 4704 4705 4706
	},
	{
		.name = "memory.pressure",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_memory_pressure_show,
S
Suren Baghdasaryan 已提交
4707 4708 4709
		.write = cgroup_memory_pressure_write,
		.poll = cgroup_pressure_poll,
		.release = cgroup_pressure_release,
J
Johannes Weiner 已提交
4710 4711 4712 4713 4714
	},
	{
		.name = "cpu.pressure",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = cgroup_cpu_pressure_show,
S
Suren Baghdasaryan 已提交
4715 4716 4717
		.write = cgroup_cpu_pressure_write,
		.poll = cgroup_pressure_poll,
		.release = cgroup_pressure_release,
J
Johannes Weiner 已提交
4718
	},
S
Suren Baghdasaryan 已提交
4719
#endif /* CONFIG_PSI */
4720 4721
	{ }	/* terminate */
};
4722

4723 4724 4725 4726 4727 4728 4729
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4730 4731 4732
 *    and thus css_tryget_online() is guaranteed to fail, the css can be
 *    offlined by invoking offline_css().  After offlining, the base ref is
 *    put.  Implemented in css_killed_work_fn().
4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
4745
static void css_free_rwork_fn(struct work_struct *work)
4746
{
4747 4748
	struct cgroup_subsys_state *css = container_of(to_rcu_work(work),
				struct cgroup_subsys_state, destroy_rwork);
4749
	struct cgroup_subsys *ss = css->ss;
4750
	struct cgroup *cgrp = css->cgroup;
4751

4752 4753
	percpu_ref_exit(&css->refcnt);

4754
	if (ss) {
4755
		/* css free path */
4756
		struct cgroup_subsys_state *parent = css->parent;
4757 4758
		int id = css->id;

4759 4760
		delete_memcg_blkcg_link(ss, css);

4761 4762
		ss->css_free(css);
		cgroup_idr_remove(&ss->css_idr, id);
4763
		cgroup_put(cgrp);
4764 4765 4766

		if (parent)
			css_put(parent);
4767 4768 4769
	} else {
		/* cgroup free path */
		atomic_dec(&cgrp->root->nr_cgrps);
4770
		cgroup1_pidlist_destroy_all(cgrp);
4771
		cancel_work_sync(&cgrp->release_agent_work);
4772

T
Tejun Heo 已提交
4773
		if (cgroup_parent(cgrp)) {
4774 4775 4776 4777 4778 4779
			/*
			 * We get a ref to the parent, and put the ref when
			 * this cgroup is being freed, so it's guaranteed
			 * that the parent won't be destroyed before its
			 * children.
			 */
T
Tejun Heo 已提交
4780
			cgroup_put(cgroup_parent(cgrp));
4781
			kernfs_put(cgrp->kn);
J
Johannes Weiner 已提交
4782
			psi_cgroup_free(cgrp);
4783
			if (cgroup_on_dfl(cgrp))
T
Tejun Heo 已提交
4784
				cgroup_rstat_exit(cgrp);
4785 4786 4787 4788 4789 4790 4791 4792 4793 4794
			kfree(cgrp);
		} else {
			/*
			 * This is root cgroup's refcnt reaching zero,
			 * which indicates that the root should be
			 * released.
			 */
			cgroup_destroy_root(cgrp->root);
		}
	}
4795 4796
}

4797
static void css_release_work_fn(struct work_struct *work)
4798 4799
{
	struct cgroup_subsys_state *css =
4800
		container_of(work, struct cgroup_subsys_state, destroy_work);
4801
	struct cgroup_subsys *ss = css->ss;
4802
	struct cgroup *cgrp = css->cgroup;
4803

4804 4805
	mutex_lock(&cgroup_mutex);

4806
	css->flags |= CSS_RELEASED;
4807 4808
	list_del_rcu(&css->sibling);

4809 4810
	if (ss) {
		/* css release path */
4811 4812 4813 4814 4815
		if (!list_empty(&css->rstat_css_node)) {
			cgroup_rstat_flush(cgrp);
			list_del_rcu(&css->rstat_css_node);
		}

4816
		cgroup_idr_replace(&ss->css_idr, NULL, css->id);
4817 4818
		if (ss->css_released)
			ss->css_released(css);
4819
	} else {
4820 4821
		struct cgroup *tcgrp;

4822
		/* cgroup release path */
4823
		TRACE_CGROUP_PATH(release, cgrp);
4824

4825
		if (cgroup_on_dfl(cgrp))
T
Tejun Heo 已提交
4826
			cgroup_rstat_flush(cgrp);
4827

4828
		spin_lock_irq(&css_set_lock);
4829 4830 4831
		for (tcgrp = cgroup_parent(cgrp); tcgrp;
		     tcgrp = cgroup_parent(tcgrp))
			tcgrp->nr_dying_descendants--;
4832
		spin_unlock_irq(&css_set_lock);
4833

4834 4835
		cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
		cgrp->id = -1;
4836 4837 4838 4839 4840 4841 4842 4843

		/*
		 * There are two control paths which try to determine
		 * cgroup from dentry without going through kernfs -
		 * cgroupstats_build() and css_tryget_online_from_dir().
		 * Those are supported by RCU protecting clearing of
		 * cgrp->kn->priv backpointer.
		 */
4844 4845 4846
		if (cgrp->kn)
			RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv,
					 NULL);
4847 4848

		cgroup_bpf_put(cgrp);
4849
	}
4850

4851 4852
	mutex_unlock(&cgroup_mutex);

4853 4854
	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);
4855 4856 4857 4858 4859 4860 4861
}

static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4862 4863
	INIT_WORK(&css->destroy_work, css_release_work_fn);
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4864 4865
}

4866 4867
static void init_and_link_css(struct cgroup_subsys_state *css,
			      struct cgroup_subsys *ss, struct cgroup *cgrp)
4868
{
4869 4870
	lockdep_assert_held(&cgroup_mutex);

4871
	cgroup_get_live(cgrp);
4872

4873
	memset(css, 0, sizeof(*css));
4874
	css->cgroup = cgrp;
4875
	css->ss = ss;
4876
	css->id = -1;
4877 4878
	INIT_LIST_HEAD(&css->sibling);
	INIT_LIST_HEAD(&css->children);
4879
	INIT_LIST_HEAD(&css->rstat_css_node);
4880
	css->serial_nr = css_serial_nr_next++;
4881
	atomic_set(&css->online_cnt, 0);
4882

T
Tejun Heo 已提交
4883 4884
	if (cgroup_parent(cgrp)) {
		css->parent = cgroup_css(cgroup_parent(cgrp), ss);
4885 4886
		css_get(css->parent);
	}
4887

4888 4889 4890
	if (cgroup_on_dfl(cgrp) && ss->css_rstat_flush)
		list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list);

4891
	BUG_ON(cgroup_css(cgrp, ss));
4892 4893
}

4894
/* invoke ->css_online() on a new CSS and mark it online if successful */
4895
static int online_css(struct cgroup_subsys_state *css)
4896
{
4897
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
4898 4899
	int ret = 0;

4900 4901
	lockdep_assert_held(&cgroup_mutex);

4902
	if (ss->css_online)
4903
		ret = ss->css_online(css);
4904
	if (!ret) {
4905
		css->flags |= CSS_ONLINE;
4906
		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
4907 4908 4909 4910

		atomic_inc(&css->online_cnt);
		if (css->parent)
			atomic_inc(&css->parent->online_cnt);
4911
	}
T
Tejun Heo 已提交
4912
	return ret;
4913 4914
}

4915
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4916
static void offline_css(struct cgroup_subsys_state *css)
4917
{
4918
	struct cgroup_subsys *ss = css->ss;
4919 4920 4921 4922 4923 4924

	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

4925
	if (ss->css_offline)
4926
		ss->css_offline(css);
4927

4928
	css->flags &= ~CSS_ONLINE;
4929
	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
4930 4931

	wake_up_all(&css->cgroup->offline_waitq);
4932 4933
}

4934
/**
4935
 * css_create - create a cgroup_subsys_state
4936 4937 4938 4939
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
4940 4941
 * css is online and installed in @cgrp.  This function doesn't create the
 * interface files.  Returns 0 on success, -errno on failure.
4942
 */
4943 4944
static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
					      struct cgroup_subsys *ss)
4945
{
T
Tejun Heo 已提交
4946
	struct cgroup *parent = cgroup_parent(cgrp);
4947
	struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
4948 4949 4950 4951 4952
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgroup_mutex);

4953
	css = ss->css_alloc(parent_css);
4954 4955
	if (!css)
		css = ERR_PTR(-ENOMEM);
4956
	if (IS_ERR(css))
4957
		return css;
4958

4959
	init_and_link_css(css, ss, cgrp);
4960

4961
	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
4962
	if (err)
4963
		goto err_free_css;
4964

V
Vladimir Davydov 已提交
4965
	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);
4966
	if (err < 0)
4967
		goto err_free_css;
4968
	css->id = err;
4969

4970
	/* @css is ready to be brought online now, make it visible */
4971
	list_add_tail_rcu(&css->sibling, &parent_css->children);
4972
	cgroup_idr_replace(&ss->css_idr, css, css->id);
4973 4974 4975

	err = online_css(css);
	if (err)
4976
		goto err_list_del;
4977

4978
	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
T
Tejun Heo 已提交
4979
	    cgroup_parent(parent)) {
4980
		pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
4981
			current->comm, current->pid, ss->name);
4982
		if (!strcmp(ss->name, "memory"))
4983
			pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
4984 4985 4986
		ss->warned_broken_hierarchy = true;
	}

4987
	return css;
4988

4989 4990
err_list_del:
	list_del_rcu(&css->sibling);
4991
err_free_css:
4992
	list_del_rcu(&css->rstat_css_node);
4993 4994
	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);
4995
	return ERR_PTR(err);
4996 4997
}

4998 4999 5000 5001 5002
/*
 * The returned cgroup is fully initialized including its control mask, but
 * it isn't associated with its kernfs_node and doesn't have the control
 * mask applied.
 */
5003
static struct cgroup *cgroup_create(struct cgroup *parent)
5004
{
5005 5006 5007
	struct cgroup_root *root = parent->root;
	struct cgroup *cgrp, *tcgrp;
	int level = parent->level + 1;
5008
	int ret;
5009

T
Tejun Heo 已提交
5010
	/* allocate the cgroup and its ID, 0 is reserved for the root */
5011 5012
	cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)),
		       GFP_KERNEL);
5013 5014
	if (!cgrp)
		return ERR_PTR(-ENOMEM);
5015

5016
	ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
5017 5018 5019
	if (ret)
		goto out_free_cgrp;

5020
	if (cgroup_on_dfl(parent)) {
T
Tejun Heo 已提交
5021
		ret = cgroup_rstat_init(cgrp);
5022 5023 5024 5025
		if (ret)
			goto out_cancel_ref;
	}

5026 5027 5028 5029
	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
V
Vladimir Davydov 已提交
5030
	cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL);
5031
	if (cgrp->id < 0) {
T
Tejun Heo 已提交
5032
		ret = -ENOMEM;
5033
		goto out_stat_exit;
5034 5035
	}

5036
	init_cgroup_housekeeping(cgrp);
5037

5038
	cgrp->self.parent = &parent->self;
T
Tejun Heo 已提交
5039
	cgrp->root = root;
5040
	cgrp->level = level;
J
Johannes Weiner 已提交
5041 5042

	ret = psi_cgroup_alloc(cgrp);
5043 5044
	if (ret)
		goto out_idr_free;
5045

J
Johannes Weiner 已提交
5046 5047 5048 5049
	ret = cgroup_bpf_inherit(cgrp);
	if (ret)
		goto out_psi_free;

5050
	spin_lock_irq(&css_set_lock);
5051
	for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) {
5052
		cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
5053

5054 5055 5056
		if (tcgrp != cgrp)
			tcgrp->nr_descendants++;
	}
5057
	spin_unlock_irq(&css_set_lock);
5058

5059 5060 5061
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

5062 5063
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
5064

5065
	cgrp->self.serial_nr = css_serial_nr_next++;
5066

5067
	/* allocation complete, commit to creation */
5068
	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
5069
	atomic_inc(&root->nr_cgrps);
5070
	cgroup_get_live(parent);
5071

5072 5073 5074 5075
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
5076
	cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
5077

5078 5079
	/*
	 * On the default hierarchy, a child doesn't automatically inherit
5080
	 * subtree_control from the parent.  Each is configured manually.
5081
	 */
5082
	if (!cgroup_on_dfl(cgrp))
5083
		cgrp->subtree_control = cgroup_control(cgrp);
5084 5085 5086

	cgroup_propagate_control(cgrp);

5087 5088
	return cgrp;

J
Johannes Weiner 已提交
5089 5090
out_psi_free:
	psi_cgroup_free(cgrp);
5091 5092
out_idr_free:
	cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
5093 5094
out_stat_exit:
	if (cgroup_on_dfl(parent))
T
Tejun Heo 已提交
5095
		cgroup_rstat_exit(cgrp);
5096 5097 5098 5099 5100 5101 5102
out_cancel_ref:
	percpu_ref_exit(&cgrp->self.refcnt);
out_free_cgrp:
	kfree(cgrp);
	return ERR_PTR(ret);
}

5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125
static bool cgroup_check_hierarchy_limits(struct cgroup *parent)
{
	struct cgroup *cgroup;
	int ret = false;
	int level = 1;

	lockdep_assert_held(&cgroup_mutex);

	for (cgroup = parent; cgroup; cgroup = cgroup_parent(cgroup)) {
		if (cgroup->nr_descendants >= cgroup->max_descendants)
			goto fail;

		if (level > cgroup->max_depth)
			goto fail;

		level++;
	}

	ret = true;
fail:
	return ret;
}

5126
int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
5127 5128 5129
{
	struct cgroup *parent, *cgrp;
	struct kernfs_node *kn;
5130
	int ret;
5131 5132 5133 5134 5135

	/* do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable */
	if (strchr(name, '\n'))
		return -EINVAL;

5136
	parent = cgroup_kn_lock_live(parent_kn, false);
5137 5138 5139
	if (!parent)
		return -ENODEV;

5140 5141 5142 5143 5144
	if (!cgroup_check_hierarchy_limits(parent)) {
		ret = -EAGAIN;
		goto out_unlock;
	}

5145 5146 5147 5148 5149 5150
	cgrp = cgroup_create(parent);
	if (IS_ERR(cgrp)) {
		ret = PTR_ERR(cgrp);
		goto out_unlock;
	}

5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168
	/* create the directory */
	kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
	if (IS_ERR(kn)) {
		ret = PTR_ERR(kn);
		goto out_destroy;
	}
	cgrp->kn = kn;

	/*
	 * This extra ref will be put in cgroup_free_fn() and guarantees
	 * that @cgrp->kn is always accessible.
	 */
	kernfs_get(kn);

	ret = cgroup_kn_set_ugid(kn);
	if (ret)
		goto out_destroy;

5169
	ret = css_populate_dir(&cgrp->self);
5170 5171 5172
	if (ret)
		goto out_destroy;

5173 5174 5175
	ret = cgroup_apply_control_enable(cgrp);
	if (ret)
		goto out_destroy;
5176

5177
	TRACE_CGROUP_PATH(mkdir, cgrp);
5178

5179
	/* let's create and online css's */
T
Tejun Heo 已提交
5180
	kernfs_activate(kn);
5181

T
Tejun Heo 已提交
5182 5183
	ret = 0;
	goto out_unlock;
5184

5185 5186
out_destroy:
	cgroup_destroy_locked(cgrp);
T
Tejun Heo 已提交
5187
out_unlock:
5188
	cgroup_kn_unlock(parent_kn);
T
Tejun Heo 已提交
5189
	return ret;
5190 5191
}

5192 5193
/*
 * This is called when the refcnt of a css is confirmed to be killed.
5194 5195
 * css_tryget_online() is now guaranteed to fail.  Tell the subsystem to
 * initate destruction and put the css ref from kill_css().
5196 5197
 */
static void css_killed_work_fn(struct work_struct *work)
5198
{
5199 5200
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
5201

5202
	mutex_lock(&cgroup_mutex);
5203

5204 5205 5206 5207 5208 5209 5210 5211
	do {
		offline_css(css);
		css_put(css);
		/* @css can't go away while we're holding cgroup_mutex */
		css = css->parent;
	} while (css && atomic_dec_and_test(&css->online_cnt));

	mutex_unlock(&cgroup_mutex);
5212 5213
}

5214 5215
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
5216 5217 5218 5219
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

5220 5221 5222 5223
	if (atomic_dec_and_test(&css->online_cnt)) {
		INIT_WORK(&css->destroy_work, css_killed_work_fn);
		queue_work(cgroup_destroy_wq, &css->destroy_work);
	}
5224 5225
}

5226 5227 5228 5229 5230 5231
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
5232 5233
 * asynchronously once css_tryget_online() is guaranteed to fail and when
 * the reference count reaches zero, @css will be released.
5234 5235
 */
static void kill_css(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
5236
{
5237
	lockdep_assert_held(&cgroup_mutex);
5238

5239 5240 5241 5242 5243
	if (css->flags & CSS_DYING)
		return;

	css->flags |= CSS_DYING;

T
Tejun Heo 已提交
5244 5245 5246 5247
	/*
	 * This must happen before css is disassociated with its cgroup.
	 * See seq_css() for details.
	 */
5248
	css_clear_dir(css);
5249

T
Tejun Heo 已提交
5250 5251 5252 5253 5254 5255 5256 5257 5258
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
5259
	 * css_tryget_online().  We can't simply call percpu_ref_kill() and
T
Tejun Heo 已提交
5260 5261 5262 5263 5264 5265 5266
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
5267 5268 5269 5270 5271 5272 5273 5274
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
5275 5276 5277
 * guarantee that css_tryget_online() won't succeed by the time
 * ->css_offline() is invoked.  To satisfy all the requirements,
 * destruction is implemented in the following two steps.
5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
5293 5294
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
5295
{
5296
	struct cgroup *tcgrp, *parent = cgroup_parent(cgrp);
T
Tejun Heo 已提交
5297
	struct cgroup_subsys_state *css;
5298
	struct cgrp_cset_link *link;
T
Tejun Heo 已提交
5299
	int ssid;
5300

5301 5302
	lockdep_assert_held(&cgroup_mutex);

5303 5304 5305 5306 5307
	/*
	 * Only migration can raise populated from zero and we're already
	 * holding cgroup_mutex.
	 */
	if (cgroup_is_populated(cgrp))
5308
		return -EBUSY;
L
Li Zefan 已提交
5309

5310
	/*
5311 5312 5313
	 * Make sure there's no live children.  We can't test emptiness of
	 * ->self.children as dead children linger on it while being
	 * drained; otherwise, "rmdir parent/child parent" may fail.
5314
	 */
5315
	if (css_has_online_children(&cgrp->self))
5316 5317
		return -EBUSY;

5318
	/*
5319 5320 5321 5322
	 * Mark @cgrp and the associated csets dead.  The former prevents
	 * further task migration and child creation by disabling
	 * cgroup_lock_live_group().  The latter makes the csets ignored by
	 * the migration path.
5323
	 */
5324
	cgrp->self.flags &= ~CSS_ONLINE;
5325

5326
	spin_lock_irq(&css_set_lock);
5327 5328
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		link->cset->dead = true;
5329
	spin_unlock_irq(&css_set_lock);
5330

5331
	/* initiate massacre of all css's */
T
Tejun Heo 已提交
5332 5333
	for_each_css(css, ssid, cgrp)
		kill_css(css);
5334

5335 5336
	/* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */
	css_clear_dir(&cgrp->self);
5337
	kernfs_remove(cgrp->kn);
5338

5339 5340 5341
	if (parent && cgroup_is_threaded(cgrp))
		parent->nr_threaded_children--;

5342
	spin_lock_irq(&css_set_lock);
5343 5344 5345 5346
	for (tcgrp = cgroup_parent(cgrp); tcgrp; tcgrp = cgroup_parent(tcgrp)) {
		tcgrp->nr_descendants--;
		tcgrp->nr_dying_descendants++;
	}
5347
	spin_unlock_irq(&css_set_lock);
5348

5349
	cgroup1_check_for_release(parent);
T
Tejun Heo 已提交
5350

5351
	/* put the base reference */
5352
	percpu_ref_kill(&cgrp->self.refcnt);
5353

5354 5355 5356
	return 0;
};

5357
int cgroup_rmdir(struct kernfs_node *kn)
5358
{
5359
	struct cgroup *cgrp;
T
Tejun Heo 已提交
5360
	int ret = 0;
5361

5362
	cgrp = cgroup_kn_lock_live(kn, false);
5363 5364
	if (!cgrp)
		return 0;
5365

5366
	ret = cgroup_destroy_locked(cgrp);
5367
	if (!ret)
5368
		TRACE_CGROUP_PATH(rmdir, cgrp);
5369

5370
	cgroup_kn_unlock(kn);
5371
	return ret;
5372 5373
}

T
Tejun Heo 已提交
5374
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
5375
	.show_options		= cgroup_show_options,
T
Tejun Heo 已提交
5376 5377 5378
	.remount_fs		= cgroup_remount,
	.mkdir			= cgroup_mkdir,
	.rmdir			= cgroup_rmdir,
5379
	.show_path		= cgroup_show_path,
T
Tejun Heo 已提交
5380 5381
};

5382
static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
5383 5384
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
5385

5386
	pr_debug("Initializing cgroup subsys %s\n", ss->name);
5387

5388 5389
	mutex_lock(&cgroup_mutex);

5390
	idr_init(&ss->css_idr);
T
Tejun Heo 已提交
5391
	INIT_LIST_HEAD(&ss->cfts);
5392

5393 5394 5395
	/* Create the root cgroup state for this subsystem */
	ss->root = &cgrp_dfl_root;
	css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
5396 5397
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
5398
	init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
5399 5400 5401 5402 5403 5404 5405

	/*
	 * Root csses are never destroyed and we can't initialize
	 * percpu_ref during early init.  Disable refcnting.
	 */
	css->flags |= CSS_NO_REF;

5406
	if (early) {
5407
		/* allocation can't be done safely during early init */
5408 5409 5410 5411 5412
		css->id = 1;
	} else {
		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
		BUG_ON(css->id < 0);
	}
5413

L
Li Zefan 已提交
5414
	/* Update the init_css_set to contain a subsys
5415
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
5416
	 * newly registered, all tasks and hence the
5417
	 * init_css_set is in the subsystem's root cgroup. */
5418
	init_css_set.subsys[ss->id] = css;
5419

5420 5421
	have_fork_callback |= (bool)ss->fork << ss->id;
	have_exit_callback |= (bool)ss->exit << ss->id;
5422
	have_release_callback |= (bool)ss->release << ss->id;
5423
	have_canfork_callback |= (bool)ss->can_fork << ss->id;
5424

L
Li Zefan 已提交
5425 5426 5427 5428 5429
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

5430
	BUG_ON(online_css(css));
5431

B
Ben Blum 已提交
5432 5433 5434
	mutex_unlock(&cgroup_mutex);
}

5435
/**
L
Li Zefan 已提交
5436 5437 5438 5439
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
5440 5441 5442
 */
int __init cgroup_init_early(void)
{
5443
	static struct cgroup_sb_opts __initdata opts;
5444
	struct cgroup_subsys *ss;
5445
	int i;
5446

5447
	init_cgroup_root(&cgrp_dfl_root, &opts);
5448 5449
	cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;

5450
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
5451

T
Tejun Heo 已提交
5452
	for_each_subsys(ss, i) {
5453
		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
5454
		     "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n",
5455
		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
5456
		     ss->id, ss->name);
5457 5458 5459
		WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
		     "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);

5460
		ss->id = i;
5461
		ss->name = cgroup_subsys_name[i];
5462 5463
		if (!ss->legacy_name)
			ss->legacy_name = cgroup_subsys_name[i];
5464 5465

		if (ss->early_init)
5466
			cgroup_init_subsys(ss, true);
5467 5468 5469 5470
	}
	return 0;
}

5471
static u16 cgroup_disable_mask __initdata;
5472

5473
/**
L
Li Zefan 已提交
5474 5475 5476 5477
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
5478 5479 5480
 */
int __init cgroup_init(void)
{
5481
	struct cgroup_subsys *ss;
5482
	int ssid;
5483

5484
	BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16);
5485
	BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
5486 5487
	BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files));
	BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files));
5488

T
Tejun Heo 已提交
5489
	cgroup_rstat_boot();
5490

5491 5492 5493 5494 5495 5496
	/*
	 * The latency of the synchronize_sched() is too high for cgroups,
	 * avoid it at the cost of forcing all readers into the slow path.
	 */
	rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss);

5497 5498
	get_user_ns(init_cgroup_ns.user_ns);

T
Tejun Heo 已提交
5499 5500
	mutex_lock(&cgroup_mutex);

5501 5502 5503 5504 5505 5506
	/*
	 * Add init_css_set to the hash table so that dfl_root can link to
	 * it during init.
	 */
	hash_add(css_set_table, &init_css_set.hlist,
		 css_set_hash(init_css_set.subsys));
5507

5508
	BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0, 0));
5509

T
Tejun Heo 已提交
5510 5511
	mutex_unlock(&cgroup_mutex);

5512
	for_each_subsys(ss, ssid) {
5513 5514 5515 5516 5517 5518 5519 5520 5521 5522
		if (ss->early_init) {
			struct cgroup_subsys_state *css =
				init_css_set.subsys[ss->id];

			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
						   GFP_KERNEL);
			BUG_ON(css->id < 0);
		} else {
			cgroup_init_subsys(ss, false);
		}
5523

T
Tejun Heo 已提交
5524 5525
		list_add_tail(&init_css_set.e_cset_node[ssid],
			      &cgrp_dfl_root.cgrp.e_csets[ssid]);
5526 5527

		/*
5528 5529 5530
		 * Setting dfl_root subsys_mask needs to consider the
		 * disabled flag and cftype registration needs kmalloc,
		 * both of which aren't available during early_init.
5531
		 */
5532 5533 5534 5535
		if (cgroup_disable_mask & (1 << ssid)) {
			static_branch_disable(cgroup_subsys_enabled_key[ssid]);
			printk(KERN_INFO "Disabling %s control group subsystem\n",
			       ss->name);
5536
			continue;
5537
		}
5538

5539
		if (cgroup1_ssid_disabled(ssid))
5540 5541 5542
			printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n",
			       ss->name);

5543 5544
		cgrp_dfl_root.subsys_mask |= 1 << ss->id;

5545 5546 5547
		/* implicit controllers must be threaded too */
		WARN_ON(ss->implicit_on_dfl && !ss->threaded);

5548 5549 5550
		if (ss->implicit_on_dfl)
			cgrp_dfl_implicit_ss_mask |= 1 << ss->id;
		else if (!ss->dfl_cftypes)
T
Tejun Heo 已提交
5551
			cgrp_dfl_inhibit_ss_mask |= 1 << ss->id;
5552

5553 5554 5555
		if (ss->threaded)
			cgrp_dfl_threaded_ss_mask |= 1 << ss->id;

5556 5557 5558 5559 5560
		if (ss->dfl_cftypes == ss->legacy_cftypes) {
			WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
		} else {
			WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
			WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
5561
		}
5562 5563 5564

		if (ss->bind)
			ss->bind(init_css_set.subsys[ssid]);
5565 5566 5567 5568

		mutex_lock(&cgroup_mutex);
		css_populate_dir(init_css_set.subsys[ssid]);
		mutex_unlock(&cgroup_mutex);
5569 5570
	}

5571 5572 5573 5574 5575
	/* init_css_set.subsys[] has been updated, re-hash */
	hash_del(&init_css_set.hlist);
	hash_add(css_set_table, &init_css_set.hlist,
		 css_set_hash(init_css_set.subsys));

5576 5577
	WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup"));
	WARN_ON(register_filesystem(&cgroup_fs_type));
5578
	WARN_ON(register_filesystem(&cgroup2_fs_type));
5579
	WARN_ON(!proc_create_single("cgroups", 0, NULL, proc_cgroupstats_show));
5580

T
Tejun Heo 已提交
5581
	return 0;
5582
}
5583

5584 5585 5586 5587 5588
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
5589
	 * Use 1 for @max_active.
5590 5591 5592 5593
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
5594
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
5595 5596 5597 5598 5599
	BUG_ON(!cgroup_destroy_wq);
	return 0;
}
core_initcall(cgroup_wq_init);

5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611
void cgroup_path_from_kernfs_id(const union kernfs_node_id *id,
					char *buf, size_t buflen)
{
	struct kernfs_node *kn;

	kn = kernfs_get_node_by_id(cgrp_dfl_root.kf_root, id);
	if (!kn)
		return;
	kernfs_path(kn, buf, buflen);
	kernfs_put(kn);
}

5612 5613 5614 5615 5616
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 */
Z
Zefan Li 已提交
5617 5618
int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
		     struct pid *pid, struct task_struct *tsk)
5619
{
5620
	char *buf;
5621
	int retval;
5622
	struct cgroup_root *root;
5623 5624

	retval = -ENOMEM;
T
Tejun Heo 已提交
5625
	buf = kmalloc(PATH_MAX, GFP_KERNEL);
5626 5627 5628 5629
	if (!buf)
		goto out;

	mutex_lock(&cgroup_mutex);
5630
	spin_lock_irq(&css_set_lock);
5631

5632
	for_each_root(root) {
5633
		struct cgroup_subsys *ss;
5634
		struct cgroup *cgrp;
T
Tejun Heo 已提交
5635
		int ssid, count = 0;
5636

T
Tejun Heo 已提交
5637
		if (root == &cgrp_dfl_root && !cgrp_dfl_visible)
5638 5639
			continue;

5640
		seq_printf(m, "%d:", root->hierarchy_id);
5641 5642 5643 5644
		if (root != &cgrp_dfl_root)
			for_each_subsys(ss, ssid)
				if (root->subsys_mask & (1 << ssid))
					seq_printf(m, "%s%s", count++ ? "," : "",
5645
						   ss->legacy_name);
5646 5647 5648
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
5649
		seq_putc(m, ':');
5650

5651
		cgrp = task_cgroup_from_root(tsk, root);
5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662

		/*
		 * On traditional hierarchies, all zombie tasks show up as
		 * belonging to the root cgroup.  On the default hierarchy,
		 * while a zombie doesn't show up in "cgroup.procs" and
		 * thus can't be migrated, its /proc/PID/cgroup keeps
		 * reporting the cgroup it belonged to before exiting.  If
		 * the cgroup is removed before the zombie is reaped,
		 * " (deleted)" is appended to the cgroup path.
		 */
		if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {
5663
			retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX,
5664
						current->nsproxy->cgroup_ns);
5665
			if (retval >= PATH_MAX)
5666
				retval = -ENAMETOOLONG;
5667
			if (retval < 0)
5668
				goto out_unlock;
5669 5670

			seq_puts(m, buf);
5671
		} else {
5672
			seq_puts(m, "/");
T
Tejun Heo 已提交
5673
		}
5674 5675 5676 5677 5678

		if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp))
			seq_puts(m, " (deleted)\n");
		else
			seq_putc(m, '\n');
5679 5680
	}

Z
Zefan Li 已提交
5681
	retval = 0;
5682
out_unlock:
5683
	spin_unlock_irq(&css_set_lock);
5684 5685 5686 5687 5688 5689
	mutex_unlock(&cgroup_mutex);
	kfree(buf);
out:
	return retval;
}

5690
/**
5691
 * cgroup_fork - initialize cgroup related fields during copy_process()
L
Li Zefan 已提交
5692
 * @child: pointer to task_struct of forking parent process.
5693
 *
5694 5695 5696
 * A task is associated with the init_css_set until cgroup_post_fork()
 * attaches it to the parent's css_set.  Empty cg_list indicates that
 * @child isn't holding reference to its css_set.
5697 5698 5699
 */
void cgroup_fork(struct task_struct *child)
{
5700
	RCU_INIT_POINTER(child->cgroups, &init_css_set);
5701
	INIT_LIST_HEAD(&child->cg_list);
5702 5703
}

5704 5705 5706 5707 5708 5709 5710 5711
/**
 * cgroup_can_fork - called on a new task before the process is exposed
 * @child: the task in question.
 *
 * This calls the subsystem can_fork() callbacks. If the can_fork() callback
 * returns an error, the fork aborts with that error code. This allows for
 * a cgroup subsystem to conditionally allow or deny new forks.
 */
5712
int cgroup_can_fork(struct task_struct *child)
5713 5714 5715 5716
{
	struct cgroup_subsys *ss;
	int i, j, ret;

5717
	do_each_subsys_mask(ss, i, have_canfork_callback) {
5718
		ret = ss->can_fork(child);
5719 5720
		if (ret)
			goto out_revert;
5721
	} while_each_subsys_mask();
5722 5723 5724 5725 5726 5727 5728 5729

	return 0;

out_revert:
	for_each_subsys(ss, j) {
		if (j >= i)
			break;
		if (ss->cancel_fork)
5730
			ss->cancel_fork(child);
5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742
	}

	return ret;
}

/**
 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
 * @child: the task in question
 *
 * This calls the cancel_fork() callbacks if a fork failed *after*
 * cgroup_can_fork() succeded.
 */
5743
void cgroup_cancel_fork(struct task_struct *child)
5744 5745 5746 5747 5748 5749
{
	struct cgroup_subsys *ss;
	int i;

	for_each_subsys(ss, i)
		if (ss->cancel_fork)
5750
			ss->cancel_fork(child);
5751 5752
}

5753
/**
L
Li Zefan 已提交
5754 5755 5756
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
5757 5758 5759
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
5760
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5761
 * list.
L
Li Zefan 已提交
5762
 */
5763
void cgroup_post_fork(struct task_struct *child)
5764
{
5765
	struct cgroup_subsys *ss;
5766 5767
	int i;

5768
	/*
D
Dongsheng Yang 已提交
5769
	 * This may race against cgroup_enable_task_cg_lists().  As that
5770 5771 5772 5773 5774 5775 5776
	 * function sets use_task_css_set_links before grabbing
	 * tasklist_lock and we just went through tasklist_lock to add
	 * @child, it's guaranteed that either we see the set
	 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
	 * @child during its iteration.
	 *
	 * If we won the race, @child is associated with %current's
5777
	 * css_set.  Grabbing css_set_lock guarantees both that the
5778 5779 5780 5781 5782 5783
	 * association is stable, and, on completion of the parent's
	 * migration, @child is visible in the source of migration or
	 * already in the destination cgroup.  This guarantee is necessary
	 * when implementing operations which need to migrate all tasks of
	 * a cgroup to another.
	 *
D
Dongsheng Yang 已提交
5784
	 * Note that if we lose to cgroup_enable_task_cg_lists(), @child
5785 5786 5787
	 * will remain in init_css_set.  This is safe because all tasks are
	 * in the init_css_set before cg_links is enabled and there's no
	 * operation which transfers all tasks out of init_css_set.
5788
	 */
5789
	if (use_task_css_set_links) {
5790 5791
		struct css_set *cset;

5792
		spin_lock_irq(&css_set_lock);
5793
		cset = task_css_set(current);
5794 5795
		if (list_empty(&child->cg_list)) {
			get_css_set(cset);
5796
			cset->nr_tasks++;
T
Tejun Heo 已提交
5797
			css_set_move_task(child, NULL, cset, false);
5798
		}
5799
		spin_unlock_irq(&css_set_lock);
5800
	}
5801 5802 5803 5804 5805 5806

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
5807
	do_each_subsys_mask(ss, i, have_fork_callback) {
5808
		ss->fork(child);
5809
	} while_each_subsys_mask();
5810
}
5811

5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
5824 5825 5826 5827 5828
 * We set the exiting tasks cgroup to the root cgroup (top_cgroup).  We
 * call cgroup_exit() while the task is still competent to handle
 * notify_on_release(), then leave the task attached to the root cgroup in
 * each hierarchy for the remainder of its exit.  No need to bother with
 * init_css_set refcnting.  init_css_set never goes away and we can't race
5829
 * with migration path - PF_EXITING is visible to migration path.
5830
 */
5831
void cgroup_exit(struct task_struct *tsk)
5832
{
5833
	struct cgroup_subsys *ss;
5834
	struct css_set *cset;
5835
	int i;
5836 5837

	/*
5838
	 * Unlink from @tsk from its css_set.  As migration path can't race
5839
	 * with us, we can check css_set and cg_list without synchronization.
5840
	 */
5841 5842
	cset = task_css_set(tsk);

5843
	if (!list_empty(&tsk->cg_list)) {
5844
		spin_lock_irq(&css_set_lock);
T
Tejun Heo 已提交
5845
		css_set_move_task(tsk, cset, NULL, false);
5846
		list_add_tail(&tsk->cg_list, &cset->dying_tasks);
5847
		cset->nr_tasks--;
5848
		spin_unlock_irq(&css_set_lock);
5849 5850
	} else {
		get_css_set(cset);
5851 5852
	}

5853
	/* see cgroup_post_fork() for details */
5854
	do_each_subsys_mask(ss, i, have_exit_callback) {
5855
		ss->exit(tsk);
5856
	} while_each_subsys_mask();
5857
}
5858

5859
void cgroup_release(struct task_struct *task)
5860
{
5861 5862 5863
	struct cgroup_subsys *ss;
	int ssid;

5864 5865
	do_each_subsys_mask(ss, ssid, have_release_callback) {
		ss->release(task);
5866
	} while_each_subsys_mask();
5867 5868 5869 5870 5871 5872 5873

	if (use_task_css_set_links) {
		spin_lock_irq(&css_set_lock);
		css_set_skip_task_iters(task_css_set(task), task);
		list_del_init(&task->cg_list);
		spin_unlock_irq(&css_set_lock);
	}
5874
}
5875

5876 5877 5878
void cgroup_free(struct task_struct *task)
{
	struct css_set *cset = task_css_set(task);
5879
	put_css_set(cset);
5880
}
5881

5882 5883
static int __init cgroup_disable(char *str)
{
5884
	struct cgroup_subsys *ss;
5885
	char *token;
5886
	int i;
5887 5888 5889 5890

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
5891

T
Tejun Heo 已提交
5892
		for_each_subsys(ss, i) {
5893 5894 5895
			if (strcmp(token, ss->name) &&
			    strcmp(token, ss->legacy_name))
				continue;
5896
			cgroup_disable_mask |= 1 << i;
5897 5898 5899 5900 5901
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
5902

5903
/**
5904
 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
5905 5906
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
5907
 *
5908 5909 5910
 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
 * to get the corresponding css and return it.  If such css doesn't exist
 * or can't be pinned, an ERR_PTR value is returned.
S
Stephane Eranian 已提交
5911
 */
5912 5913
struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
						       struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
5914
{
T
Tejun Heo 已提交
5915
	struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
5916
	struct file_system_type *s_type = dentry->d_sb->s_type;
T
Tejun Heo 已提交
5917
	struct cgroup_subsys_state *css = NULL;
S
Stephane Eranian 已提交
5918 5919
	struct cgroup *cgrp;

5920
	/* is @dentry a cgroup dir? */
5921 5922
	if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) ||
	    !kn || kernfs_type(kn) != KERNFS_DIR)
S
Stephane Eranian 已提交
5923 5924
		return ERR_PTR(-EBADF);

5925 5926
	rcu_read_lock();

T
Tejun Heo 已提交
5927 5928 5929
	/*
	 * This path doesn't originate from kernfs and @kn could already
	 * have been or be removed at any point.  @kn->priv is RCU
5930
	 * protected for this access.  See css_release_work_fn() for details.
T
Tejun Heo 已提交
5931
	 */
5932
	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);
T
Tejun Heo 已提交
5933 5934
	if (cgrp)
		css = cgroup_css(cgrp, ss);
5935

5936
	if (!css || !css_tryget_online(css))
5937 5938 5939 5940
		css = ERR_PTR(-ENOENT);

	rcu_read_unlock();
	return css;
S
Stephane Eranian 已提交
5941 5942
}

5943 5944 5945 5946 5947 5948 5949 5950 5951 5952
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
5953
	WARN_ON_ONCE(!rcu_read_lock_held());
5954
	return idr_find(&ss->css_idr, id);
S
Stephane Eranian 已提交
5955 5956
}

5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976
/**
 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
 * @path: path on the default hierarchy
 *
 * Find the cgroup at @path on the default hierarchy, increment its
 * reference count and return it.  Returns pointer to the found cgroup on
 * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
 * if @path points to a non-directory.
 */
struct cgroup *cgroup_get_from_path(const char *path)
{
	struct kernfs_node *kn;
	struct cgroup *cgrp;

	mutex_lock(&cgroup_mutex);

	kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path);
	if (kn) {
		if (kernfs_type(kn) == KERNFS_DIR) {
			cgrp = kn->priv;
5977
			cgroup_get_live(cgrp);
5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990
		} else {
			cgrp = ERR_PTR(-ENOTDIR);
		}
		kernfs_put(kn);
	} else {
		cgrp = ERR_PTR(-ENOENT);
	}

	mutex_unlock(&cgroup_mutex);
	return cgrp;
}
EXPORT_SYMBOL_GPL(cgroup_get_from_path);

5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024
/**
 * cgroup_get_from_fd - get a cgroup pointer from a fd
 * @fd: fd obtained by open(cgroup2_dir)
 *
 * Find the cgroup from a fd which should be obtained
 * by opening a cgroup directory.  Returns a pointer to the
 * cgroup on success. ERR_PTR is returned if the cgroup
 * cannot be found.
 */
struct cgroup *cgroup_get_from_fd(int fd)
{
	struct cgroup_subsys_state *css;
	struct cgroup *cgrp;
	struct file *f;

	f = fget_raw(fd);
	if (!f)
		return ERR_PTR(-EBADF);

	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);
	fput(f);
	if (IS_ERR(css))
		return ERR_CAST(css);

	cgrp = css->cgroup;
	if (!cgroup_on_dfl(cgrp)) {
		cgroup_put(cgrp);
		return ERR_PTR(-EBADF);
	}

	return cgrp;
}
EXPORT_SYMBOL_GPL(cgroup_get_from_fd);

T
Tejun Heo 已提交
6025 6026 6027 6028 6029 6030 6031 6032
/*
 * sock->sk_cgrp_data handling.  For more info, see sock_cgroup_data
 * definition in cgroup-defs.h.
 */
#ifdef CONFIG_SOCK_CGROUP_DATA

#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)

6033
DEFINE_SPINLOCK(cgroup_sk_update_lock);
T
Tejun Heo 已提交
6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054
static bool cgroup_sk_alloc_disabled __read_mostly;

void cgroup_sk_alloc_disable(void)
{
	if (cgroup_sk_alloc_disabled)
		return;
	pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
	cgroup_sk_alloc_disabled = true;
}

#else

#define cgroup_sk_alloc_disabled	false

#endif

void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
{
	if (cgroup_sk_alloc_disabled)
		return;

6055 6056
	/* Socket clone path */
	if (skcd->val) {
6057 6058 6059 6060 6061
		/*
		 * We might be cloning a socket which is left in an empty
		 * cgroup and the cgroup might have already been rmdir'd.
		 * Don't use cgroup_get_live().
		 */
6062 6063 6064 6065
		cgroup_get(sock_cgroup_ptr(skcd));
		return;
	}

T
Tejun Heo 已提交
6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088
	rcu_read_lock();

	while (true) {
		struct css_set *cset;

		cset = task_css_set(current);
		if (likely(cgroup_tryget(cset->dfl_cgrp))) {
			skcd->val = (unsigned long)cset->dfl_cgrp;
			break;
		}
		cpu_relax();
	}

	rcu_read_unlock();
}

void cgroup_sk_free(struct sock_cgroup_data *skcd)
{
	cgroup_put(sock_cgroup_ptr(skcd));
}

#endif	/* CONFIG_SOCK_CGROUP_DATA */

6089
#ifdef CONFIG_CGROUP_BPF
6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101
int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
		      enum bpf_attach_type type, u32 flags)
{
	int ret;

	mutex_lock(&cgroup_mutex);
	ret = __cgroup_bpf_attach(cgrp, prog, type, flags);
	mutex_unlock(&cgroup_mutex);
	return ret;
}
int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
		      enum bpf_attach_type type, u32 flags)
6102
{
6103
	int ret;
6104 6105

	mutex_lock(&cgroup_mutex);
6106
	ret = __cgroup_bpf_detach(cgrp, prog, type, flags);
6107
	mutex_unlock(&cgroup_mutex);
6108
	return ret;
6109
}
6110 6111 6112 6113 6114 6115 6116 6117 6118 6119
int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
		     union bpf_attr __user *uattr)
{
	int ret;

	mutex_lock(&cgroup_mutex);
	ret = __cgroup_bpf_query(cgrp, attr, uattr);
	mutex_unlock(&cgroup_mutex);
	return ret;
}
6120
#endif /* CONFIG_CGROUP_BPF */
6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165

#ifdef CONFIG_SYSFS
static ssize_t show_delegatable_files(struct cftype *files, char *buf,
				      ssize_t size, const char *prefix)
{
	struct cftype *cft;
	ssize_t ret = 0;

	for (cft = files; cft && cft->name[0] != '\0'; cft++) {
		if (!(cft->flags & CFTYPE_NS_DELEGATABLE))
			continue;

		if (prefix)
			ret += snprintf(buf + ret, size - ret, "%s.", prefix);

		ret += snprintf(buf + ret, size - ret, "%s\n", cft->name);

		if (unlikely(ret >= size)) {
			WARN_ON(1);
			break;
		}
	}

	return ret;
}

static ssize_t delegate_show(struct kobject *kobj, struct kobj_attribute *attr,
			      char *buf)
{
	struct cgroup_subsys *ss;
	int ssid;
	ssize_t ret = 0;

	ret = show_delegatable_files(cgroup_base_files, buf, PAGE_SIZE - ret,
				     NULL);

	for_each_subsys(ss, ssid)
		ret += show_delegatable_files(ss->dfl_cftypes, buf + ret,
					      PAGE_SIZE - ret,
					      cgroup_subsys_name[ssid]);

	return ret;
}
static struct kobj_attribute cgroup_delegate_attr = __ATTR_RO(delegate);

6166 6167 6168 6169 6170 6171 6172
static ssize_t features_show(struct kobject *kobj, struct kobj_attribute *attr,
			     char *buf)
{
	return snprintf(buf, PAGE_SIZE, "nsdelegate\n");
}
static struct kobj_attribute cgroup_features_attr = __ATTR_RO(features);

6173 6174
static struct attribute *cgroup_sysfs_attrs[] = {
	&cgroup_delegate_attr.attr,
6175
	&cgroup_features_attr.attr,
6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189
	NULL,
};

static const struct attribute_group cgroup_sysfs_attr_group = {
	.attrs = cgroup_sysfs_attrs,
	.name = "cgroup",
};

static int __init cgroup_sysfs_init(void)
{
	return sysfs_create_group(kernel_kobj, &cgroup_sysfs_attr_group);
}
subsys_initcall(cgroup_sysfs_init);
#endif /* CONFIG_SYSFS */