cgroup.c 147.0 KB
Newer Older
1 2 3 4 5 6
/*
 *  Generic process-grouping system.
 *
 *  Based originally on the cpuset system, extracted by Paul Menage
 *  Copyright (C) 2006 Google, Inc
 *
7 8 9 10
 *  Notifications support
 *  Copyright (C) 2009 Nokia Corporation
 *  Author: Kirill A. Shutemov
 *
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
 *  Copyright notices from the original cpuset code:
 *  --------------------------------------------------
 *  Copyright (C) 2003 BULL SA.
 *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
 *  2003-10-10 Written by Simon Derr.
 *  2003-10-22 Updates by Stephen Hemminger.
 *  2004 May-July Rework by Paul Jackson.
 *  ---------------------------------------------------
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

#include <linux/cgroup.h>
30
#include <linux/cred.h>
31
#include <linux/ctype.h>
32
#include <linux/errno.h>
33
#include <linux/init_task.h>
34 35 36 37 38 39
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
40
#include <linux/proc_fs.h>
41 42
#include <linux/rcupdate.h>
#include <linux/sched.h>
43
#include <linux/backing-dev.h>
44 45 46 47
#include <linux/slab.h>
#include <linux/magic.h>
#include <linux/spinlock.h>
#include <linux/string.h>
48
#include <linux/sort.h>
49
#include <linux/kmod.h>
50
#include <linux/module.h>
B
Balbir Singh 已提交
51 52
#include <linux/delayacct.h>
#include <linux/cgroupstats.h>
53
#include <linux/hashtable.h>
54
#include <linux/namei.h>
L
Li Zefan 已提交
55
#include <linux/pid_namespace.h>
56
#include <linux/idr.h>
57
#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
58
#include <linux/flex_array.h> /* used in cgroup_attach_task */
59
#include <linux/kthread.h>
B
Balbir Singh 已提交
60

A
Arun Sharma 已提交
61
#include <linux/atomic.h>
62

63 64 65 66 67 68 69 70
/*
 * pidlists linger the following amount before being destroyed.  The goal
 * is avoiding frequent destruction in the middle of consecutive read calls
 * Expiring in the middle is a performance problem not a correctness one.
 * 1 sec should be enough.
 */
#define CGROUP_PIDLIST_DESTROY_DELAY	HZ

T
Tejun Heo 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * cgroup_mutex is the master lock.  Any modification to cgroup or its
 * hierarchy must be performed while holding it.
 *
 * cgroup_root_mutex nests inside cgroup_mutex and should be held to modify
 * cgroupfs_root of any cgroup hierarchy - subsys list, flags,
 * release_agent_path and so on.  Modifying requires both cgroup_mutex and
 * cgroup_root_mutex.  Readers can acquire either of the two.  This is to
 * break the following locking order cycle.
 *
 *  A. cgroup_mutex -> cred_guard_mutex -> s_type->i_mutex_key -> namespace_sem
 *  B. namespace_sem -> cgroup_mutex
 *
 * B happens only through cgroup_show_options() and using cgroup_root_mutex
 * breaks it.
 */
T
Tejun Heo 已提交
87 88
#ifdef CONFIG_PROVE_RCU
DEFINE_MUTEX(cgroup_mutex);
89
EXPORT_SYMBOL_GPL(cgroup_mutex);	/* only for lockdep */
T
Tejun Heo 已提交
90
#else
91
static DEFINE_MUTEX(cgroup_mutex);
T
Tejun Heo 已提交
92 93
#endif

T
Tejun Heo 已提交
94
static DEFINE_MUTEX(cgroup_root_mutex);
95

96 97 98 99 100
#define cgroup_assert_mutex_or_rcu_locked()				\
	rcu_lockdep_assert(rcu_read_lock_held() ||			\
			   lockdep_is_held(&cgroup_mutex),		\
			   "cgroup_mutex or RCU read lock required");

101 102 103 104 105 106 107 108
#ifdef CONFIG_LOCKDEP
#define cgroup_assert_mutex_or_root_locked()				\
	WARN_ON_ONCE(debug_locks && (!lockdep_is_held(&cgroup_mutex) &&	\
				     !lockdep_is_held(&cgroup_root_mutex)))
#else
#define cgroup_assert_mutex_or_root_locked()	do { } while (0)
#endif

109 110 111 112 113 114 115 116
/*
 * cgroup destruction makes heavy use of work items and there can be a lot
 * of concurrent destructions.  Use a separate workqueue so that cgroup
 * destruction work items don't end up filling up max_active of system_wq
 * which may lead to deadlock.
 */
static struct workqueue_struct *cgroup_destroy_wq;

117 118 119 120 121 122
/*
 * pidlist destructions need to be flushed on cgroup destruction.  Use a
 * separate workqueue as flush domain.
 */
static struct workqueue_struct *cgroup_pidlist_destroy_wq;

B
Ben Blum 已提交
123 124
/*
 * Generate an array of cgroup subsystem pointers. At boot time, this is
125
 * populated with the built in subsystems, and modular subsystems are
B
Ben Blum 已提交
126 127 128
 * registered after that. The mutable section of this array is protected by
 * cgroup_mutex.
 */
129
#define SUBSYS(_x) [_x ## _subsys_id] = &_x ## _subsys,
130
#define IS_SUBSYS_ENABLED(option) IS_BUILTIN(option)
131
static struct cgroup_subsys *cgroup_subsys[CGROUP_SUBSYS_COUNT] = {
132 133 134 135
#include <linux/cgroup_subsys.h>
};

/*
136 137 138
 * The dummy hierarchy, reserved for the subsystems that are otherwise
 * unattached - it never has more than a single cgroup, and all tasks are
 * part of that cgroup.
139
 */
140 141 142 143
static struct cgroupfs_root cgroup_dummy_root;

/* dummy_top is a shorthand for the dummy hierarchy's top cgroup */
static struct cgroup * const cgroup_dummy_top = &cgroup_dummy_root.top_cgroup;
144 145 146

/* The list of hierarchy roots */

147 148
static LIST_HEAD(cgroup_roots);
static int cgroup_root_count;
149

T
Tejun Heo 已提交
150 151 152 153 154
/*
 * Hierarchy ID allocation and mapping.  It follows the same exclusion
 * rules as other root ops - both cgroup_mutex and cgroup_root_mutex for
 * writes, either for reads.
 */
155
static DEFINE_IDR(cgroup_hierarchy_idr);
156

157 158
static struct cgroup_name root_cgroup_name = { .name = "/" };

159 160 161 162 163
/*
 * Assign a monotonically increasing serial number to cgroups.  It
 * guarantees cgroups with bigger numbers are newer than those with smaller
 * numbers.  Also, as cgroups are always appended to the parent's
 * ->children list, it guarantees that sibling cgroups are always sorted in
164 165
 * the ascending serial number order on the list.  Protected by
 * cgroup_mutex.
166
 */
167
static u64 cgroup_serial_nr_next = 1;
168

169
/* This flag indicates whether tasks in the fork and exit paths should
L
Li Zefan 已提交
170 171 172
 * check for fork/exit handlers to call. This avoids us having to do
 * extra work in the fork/exit path if none of the subsystems need to
 * be called.
173
 */
174
static int need_forkexit_callback __read_mostly;
175

176 177
static struct cftype cgroup_base_files[];

178
static void cgroup_destroy_css_killed(struct cgroup *cgrp);
179
static int cgroup_destroy_locked(struct cgroup *cgrp);
180 181
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add);
182
static int cgroup_file_release(struct inode *inode, struct file *file);
183
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp);
184

T
Tejun Heo 已提交
185 186 187
/**
 * cgroup_css - obtain a cgroup's css for the specified subsystem
 * @cgrp: the cgroup of interest
188
 * @ss: the subsystem of interest (%NULL returns the dummy_css)
T
Tejun Heo 已提交
189
 *
190 191 192 193 194
 * Return @cgrp's css (cgroup_subsys_state) associated with @ss.  This
 * function must be called either under cgroup_mutex or rcu_read_lock() and
 * the caller is responsible for pinning the returned css if it wants to
 * keep accessing it outside the said locks.  This function may return
 * %NULL if @cgrp doesn't have @subsys_id enabled.
T
Tejun Heo 已提交
195 196
 */
static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
197
					      struct cgroup_subsys *ss)
T
Tejun Heo 已提交
198
{
199 200 201 202 203
	if (ss)
		return rcu_dereference_check(cgrp->subsys[ss->subsys_id],
					     lockdep_is_held(&cgroup_mutex));
	else
		return &cgrp->dummy_css;
T
Tejun Heo 已提交
204
}
205

206
/* convenient tests for these bits */
207
static inline bool cgroup_is_dead(const struct cgroup *cgrp)
208
{
209
	return test_bit(CGRP_DEAD, &cgrp->flags);
210 211
}

212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
/**
 * cgroup_is_descendant - test ancestry
 * @cgrp: the cgroup to be tested
 * @ancestor: possible ancestor of @cgrp
 *
 * Test whether @cgrp is a descendant of @ancestor.  It also returns %true
 * if @cgrp == @ancestor.  This function is safe to call as long as @cgrp
 * and @ancestor are accessible.
 */
bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor)
{
	while (cgrp) {
		if (cgrp == ancestor)
			return true;
		cgrp = cgrp->parent;
	}
	return false;
}
EXPORT_SYMBOL_GPL(cgroup_is_descendant);
231

232
static int cgroup_is_releasable(const struct cgroup *cgrp)
233 234
{
	const int bits =
235 236 237
		(1 << CGRP_RELEASABLE) |
		(1 << CGRP_NOTIFY_ON_RELEASE);
	return (cgrp->flags & bits) == bits;
238 239
}

240
static int notify_on_release(const struct cgroup *cgrp)
241
{
242
	return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
243 244
}

T
Tejun Heo 已提交
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
/**
 * for_each_css - iterate all css's of a cgroup
 * @css: the iteration cursor
 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
 * @cgrp: the target cgroup to iterate css's of
 *
 * Should be called under cgroup_mutex.
 */
#define for_each_css(css, ssid, cgrp)					\
	for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)	\
		if (!((css) = rcu_dereference_check(			\
				(cgrp)->subsys[(ssid)],			\
				lockdep_is_held(&cgroup_mutex)))) { }	\
		else

260 261 262
/**
 * for_each_subsys - iterate all loaded cgroup subsystems
 * @ss: the iteration cursor
263
 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
264
 *
265 266
 * Iterates through all loaded subsystems.  Should be called under
 * cgroup_mutex or cgroup_root_mutex.
267
 */
268 269 270 271
#define for_each_subsys(ss, ssid)					\
	for (({ cgroup_assert_mutex_or_root_locked(); (ssid) = 0; });	\
	     (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++)			\
		if (!((ss) = cgroup_subsys[(ssid)])) { }		\
272 273 274 275 276 277 278 279 280 281 282 283 284 285
		else

/**
 * for_each_builtin_subsys - iterate all built-in cgroup subsystems
 * @ss: the iteration cursor
 * @i: the index of @ss, CGROUP_BUILTIN_SUBSYS_COUNT after reaching the end
 *
 * Bulit-in subsystems are always present and iteration itself doesn't
 * require any synchronization.
 */
#define for_each_builtin_subsys(ss, i)					\
	for ((i) = 0; (i) < CGROUP_BUILTIN_SUBSYS_COUNT &&		\
	     (((ss) = cgroup_subsys[i]) || true); (i)++)

286 287 288
/* iterate across the active hierarchies */
#define for_each_active_root(root)					\
	list_for_each_entry((root), &cgroup_roots, root_list)
289

290 291 292 293 294
static inline struct cgroup *__d_cgrp(struct dentry *dentry)
{
	return dentry->d_fsdata;
}

T
Tejun Heo 已提交
295
static inline struct cfent *__d_cfe(struct dentry *dentry)
296 297 298 299
{
	return dentry->d_fsdata;
}

T
Tejun Heo 已提交
300 301 302 303 304
static inline struct cftype *__d_cft(struct dentry *dentry)
{
	return __d_cfe(dentry)->type;
}

305 306 307 308
/**
 * cgroup_lock_live_group - take cgroup_mutex and check that cgrp is alive.
 * @cgrp: the cgroup to be checked for liveness
 *
T
Tejun Heo 已提交
309 310
 * On success, returns true; the mutex should be later unlocked.  On
 * failure returns false with no lock held.
311
 */
312
static bool cgroup_lock_live_group(struct cgroup *cgrp)
313 314
{
	mutex_lock(&cgroup_mutex);
315
	if (cgroup_is_dead(cgrp)) {
316 317 318 319 320 321
		mutex_unlock(&cgroup_mutex);
		return false;
	}
	return true;
}

322 323 324
/* the list of cgroups eligible for automatic release. Protected by
 * release_list_lock */
static LIST_HEAD(release_list);
325
static DEFINE_RAW_SPINLOCK(release_list_lock);
326 327
static void cgroup_release_agent(struct work_struct *work);
static DECLARE_WORK(release_agent_work, cgroup_release_agent);
328
static void check_for_release(struct cgroup *cgrp);
329

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
/*
 * A cgroup can be associated with multiple css_sets as different tasks may
 * belong to different cgroups on different hierarchies.  In the other
 * direction, a css_set is naturally associated with multiple cgroups.
 * This M:N relationship is represented by the following link structure
 * which exists for each association and allows traversing the associations
 * from both sides.
 */
struct cgrp_cset_link {
	/* the cgroup and css_set this link associates */
	struct cgroup		*cgrp;
	struct css_set		*cset;

	/* list of cgrp_cset_links anchored at cgrp->cset_links */
	struct list_head	cset_link;

	/* list of cgrp_cset_links anchored at css_set->cgrp_links */
	struct list_head	cgrp_link;
348 349 350 351 352 353 354 355 356 357
};

/* The default css_set - used by init and its children prior to any
 * hierarchies being mounted. It contains a pointer to the root state
 * for each subsystem. Also used to anchor the list of css_sets. Not
 * reference-counted, to improve performance when child cgroups
 * haven't been created.
 */

static struct css_set init_css_set;
358
static struct cgrp_cset_link init_cgrp_cset_link;
359

360 361 362
/*
 * css_set_lock protects the list of css_set objects, and the chain of
 * tasks off each css_set.  Nests outside task->alloc_lock due to
363
 * css_task_iter_start().
364
 */
365 366 367
static DEFINE_RWLOCK(css_set_lock);
static int css_set_count;

368 369 370 371 372
/*
 * hash table for cgroup groups. This improves the performance to find
 * an existing css_set. This hash doesn't (currently) take into
 * account cgroups in empty hierarchies.
 */
373
#define CSS_SET_HASH_BITS	7
374
static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
375

376
static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
377
{
378
	unsigned long key = 0UL;
379 380
	struct cgroup_subsys *ss;
	int i;
381

382
	for_each_subsys(ss, i)
383 384
		key += (unsigned long)css[i];
	key = (key >> 16) ^ key;
385

386
	return key;
387 388
}

389 390
/*
 * We don't maintain the lists running through each css_set to its task
391 392 393
 * until after the first call to css_task_iter_start().  This reduces the
 * fork()/exit() overhead for people who have cgroups compiled into their
 * kernel but not actually in use.
394
 */
395
static int use_task_css_set_links __read_mostly;
396

397
static void __put_css_set(struct css_set *cset, int taskexit)
398
{
399
	struct cgrp_cset_link *link, *tmp_link;
400

401 402 403 404 405
	/*
	 * Ensure that the refcount doesn't hit zero while any readers
	 * can see it. Similar to atomic_dec_and_lock(), but for an
	 * rwlock
	 */
406
	if (atomic_add_unless(&cset->refcount, -1, 1))
407 408
		return;
	write_lock(&css_set_lock);
409
	if (!atomic_dec_and_test(&cset->refcount)) {
410 411 412
		write_unlock(&css_set_lock);
		return;
	}
413

414
	/* This css_set is dead. unlink it and release cgroup refcounts */
415
	hash_del(&cset->hlist);
416 417
	css_set_count--;

418
	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
419
		struct cgroup *cgrp = link->cgrp;
420

421 422
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
L
Li Zefan 已提交
423

424
		/* @cgrp can't go away while we're holding css_set_lock */
T
Tejun Heo 已提交
425
		if (list_empty(&cgrp->cset_links) && notify_on_release(cgrp)) {
426
			if (taskexit)
427 428
				set_bit(CGRP_RELEASABLE, &cgrp->flags);
			check_for_release(cgrp);
429
		}
430 431

		kfree(link);
432
	}
433 434

	write_unlock(&css_set_lock);
435
	kfree_rcu(cset, rcu_head);
436 437
}

438 439 440
/*
 * refcounted get/put for css_set objects
 */
441
static inline void get_css_set(struct css_set *cset)
442
{
443
	atomic_inc(&cset->refcount);
444 445
}

446
static inline void put_css_set(struct css_set *cset)
447
{
448
	__put_css_set(cset, 0);
449 450
}

451
static inline void put_css_set_taskexit(struct css_set *cset)
452
{
453
	__put_css_set(cset, 1);
454 455
}

456
/**
457
 * compare_css_sets - helper function for find_existing_css_set().
458 459
 * @cset: candidate css_set being tested
 * @old_cset: existing css_set for a task
460 461 462
 * @new_cgrp: cgroup that's being entered by the task
 * @template: desired set of css pointers in css_set (pre-calculated)
 *
L
Li Zefan 已提交
463
 * Returns true if "cset" matches "old_cset" except for the hierarchy
464 465
 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
 */
466 467
static bool compare_css_sets(struct css_set *cset,
			     struct css_set *old_cset,
468 469 470 471 472
			     struct cgroup *new_cgrp,
			     struct cgroup_subsys_state *template[])
{
	struct list_head *l1, *l2;

473
	if (memcmp(template, cset->subsys, sizeof(cset->subsys))) {
474 475 476 477 478 479 480 481 482 483 484 485 486
		/* Not all subsystems matched */
		return false;
	}

	/*
	 * Compare cgroup pointers in order to distinguish between
	 * different cgroups in heirarchies with no subsystems. We
	 * could get by with just this check alone (and skip the
	 * memcmp above) but on most setups the memcmp check will
	 * avoid the need for this more expensive check on almost all
	 * candidates.
	 */

487 488
	l1 = &cset->cgrp_links;
	l2 = &old_cset->cgrp_links;
489
	while (1) {
490
		struct cgrp_cset_link *link1, *link2;
491
		struct cgroup *cgrp1, *cgrp2;
492 493 494 495

		l1 = l1->next;
		l2 = l2->next;
		/* See if we reached the end - both lists are equal length. */
496 497
		if (l1 == &cset->cgrp_links) {
			BUG_ON(l2 != &old_cset->cgrp_links);
498 499
			break;
		} else {
500
			BUG_ON(l2 == &old_cset->cgrp_links);
501 502
		}
		/* Locate the cgroups associated with these links. */
503 504 505 506
		link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
		link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
		cgrp1 = link1->cgrp;
		cgrp2 = link2->cgrp;
507
		/* Hierarchies should be linked in the same order. */
508
		BUG_ON(cgrp1->root != cgrp2->root);
509 510 511 512 513 514 515 516

		/*
		 * If this hierarchy is the hierarchy of the cgroup
		 * that's changing, then we need to check that this
		 * css_set points to the new cgroup; if it's any other
		 * hierarchy, then this css_set should point to the
		 * same cgroup as the old css_set.
		 */
517 518
		if (cgrp1->root == new_cgrp->root) {
			if (cgrp1 != new_cgrp)
519 520
				return false;
		} else {
521
			if (cgrp1 != cgrp2)
522 523 524 525 526 527
				return false;
		}
	}
	return true;
}

528 529 530 531 532
/**
 * find_existing_css_set - init css array and find the matching css_set
 * @old_cset: the css_set that we're using before the cgroup transition
 * @cgrp: the cgroup that we're moving into
 * @template: out param for the new set of csses, should be clear on entry
533
 */
534 535 536
static struct css_set *find_existing_css_set(struct css_set *old_cset,
					struct cgroup *cgrp,
					struct cgroup_subsys_state *template[])
537
{
538
	struct cgroupfs_root *root = cgrp->root;
539
	struct cgroup_subsys *ss;
540
	struct css_set *cset;
541
	unsigned long key;
542
	int i;
543

B
Ben Blum 已提交
544 545 546 547 548
	/*
	 * Build the set of subsystem state objects that we want to see in the
	 * new css_set. while subsystems can change globally, the entries here
	 * won't change, so no need for locking.
	 */
549
	for_each_subsys(ss, i) {
550
		if (root->subsys_mask & (1UL << i)) {
551 552 553
			/* Subsystem is in this hierarchy. So we want
			 * the subsystem state from the new
			 * cgroup */
554
			template[i] = cgroup_css(cgrp, ss);
555 556 557
		} else {
			/* Subsystem is not in this hierarchy, so we
			 * don't want to change the subsystem state */
558
			template[i] = old_cset->subsys[i];
559 560 561
		}
	}

562
	key = css_set_hash(template);
563 564
	hash_for_each_possible(css_set_table, cset, hlist, key) {
		if (!compare_css_sets(cset, old_cset, cgrp, template))
565 566 567
			continue;

		/* This css_set matches what we need */
568
		return cset;
569
	}
570 571 572 573 574

	/* No existing cgroup group matched */
	return NULL;
}

575
static void free_cgrp_cset_links(struct list_head *links_to_free)
576
{
577
	struct cgrp_cset_link *link, *tmp_link;
578

579 580
	list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
		list_del(&link->cset_link);
581 582 583 584
		kfree(link);
	}
}

585 586 587 588 589 590 591
/**
 * allocate_cgrp_cset_links - allocate cgrp_cset_links
 * @count: the number of links to allocate
 * @tmp_links: list_head the allocated links are put on
 *
 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
 * through ->cset_link.  Returns 0 on success or -errno.
592
 */
593
static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
594
{
595
	struct cgrp_cset_link *link;
596
	int i;
597 598 599

	INIT_LIST_HEAD(tmp_links);

600
	for (i = 0; i < count; i++) {
601
		link = kzalloc(sizeof(*link), GFP_KERNEL);
602
		if (!link) {
603
			free_cgrp_cset_links(tmp_links);
604 605
			return -ENOMEM;
		}
606
		list_add(&link->cset_link, tmp_links);
607 608 609 610
	}
	return 0;
}

611 612
/**
 * link_css_set - a helper function to link a css_set to a cgroup
613
 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
614
 * @cset: the css_set to be linked
615 616
 * @cgrp: the destination cgroup
 */
617 618
static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
			 struct cgroup *cgrp)
619
{
620
	struct cgrp_cset_link *link;
621

622 623 624
	BUG_ON(list_empty(tmp_links));
	link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
	link->cset = cset;
625
	link->cgrp = cgrp;
626
	list_move(&link->cset_link, &cgrp->cset_links);
627 628 629 630
	/*
	 * Always add links to the tail of the list so that the list
	 * is sorted by order of hierarchy creation
	 */
631
	list_add_tail(&link->cgrp_link, &cset->cgrp_links);
632 633
}

634 635 636 637 638 639 640
/**
 * find_css_set - return a new css_set with one cgroup updated
 * @old_cset: the baseline css_set
 * @cgrp: the cgroup to be updated
 *
 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
 * substituted into the appropriate hierarchy.
641
 */
642 643
static struct css_set *find_css_set(struct css_set *old_cset,
				    struct cgroup *cgrp)
644
{
645
	struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
646
	struct css_set *cset;
647 648
	struct list_head tmp_links;
	struct cgrp_cset_link *link;
649
	unsigned long key;
650

651 652
	lockdep_assert_held(&cgroup_mutex);

653 654
	/* First see if we already have a cgroup group that matches
	 * the desired set */
655
	read_lock(&css_set_lock);
656 657 658
	cset = find_existing_css_set(old_cset, cgrp, template);
	if (cset)
		get_css_set(cset);
659
	read_unlock(&css_set_lock);
660

661 662
	if (cset)
		return cset;
663

664
	cset = kzalloc(sizeof(*cset), GFP_KERNEL);
665
	if (!cset)
666 667
		return NULL;

668
	/* Allocate all the cgrp_cset_link objects that we'll need */
669
	if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
670
		kfree(cset);
671 672 673
		return NULL;
	}

674
	atomic_set(&cset->refcount, 1);
675
	INIT_LIST_HEAD(&cset->cgrp_links);
676 677
	INIT_LIST_HEAD(&cset->tasks);
	INIT_HLIST_NODE(&cset->hlist);
678 679 680

	/* Copy the set of subsystem state objects generated in
	 * find_existing_css_set() */
681
	memcpy(cset->subsys, template, sizeof(cset->subsys));
682 683 684

	write_lock(&css_set_lock);
	/* Add reference counts and links from the new css_set. */
685
	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
686
		struct cgroup *c = link->cgrp;
687

688 689
		if (c->root == cgrp->root)
			c = cgrp;
690
		link_css_set(&tmp_links, cset, c);
691
	}
692

693
	BUG_ON(!list_empty(&tmp_links));
694 695

	css_set_count++;
696 697

	/* Add this cgroup group to the hash table */
698 699
	key = css_set_hash(cset->subsys);
	hash_add(css_set_table, &cset->hlist, key);
700

701 702
	write_unlock(&css_set_lock);

703
	return cset;
704 705
}

706 707 708 709 710 711 712
/*
 * Return the cgroup for "task" from the given hierarchy. Must be
 * called with cgroup_mutex held.
 */
static struct cgroup *task_cgroup_from_root(struct task_struct *task,
					    struct cgroupfs_root *root)
{
713
	struct css_set *cset;
714 715 716 717 718 719 720 721 722
	struct cgroup *res = NULL;

	BUG_ON(!mutex_is_locked(&cgroup_mutex));
	read_lock(&css_set_lock);
	/*
	 * No need to lock the task - since we hold cgroup_mutex the
	 * task can't change groups, so the only thing that can happen
	 * is that it exits and its css is set back to init_css_set.
	 */
723
	cset = task_css_set(task);
724
	if (cset == &init_css_set) {
725 726
		res = &root->top_cgroup;
	} else {
727 728 729
		struct cgrp_cset_link *link;

		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
730
			struct cgroup *c = link->cgrp;
731

732 733 734 735 736 737 738 739 740 741 742
			if (c->root == root) {
				res = c;
				break;
			}
		}
	}
	read_unlock(&css_set_lock);
	BUG_ON(!res);
	return res;
}

743 744 745 746 747 748 749 750 751 752
/*
 * There is one global cgroup mutex. We also require taking
 * task_lock() when dereferencing a task's cgroup subsys pointers.
 * See "The task_lock() exception", at the end of this comment.
 *
 * A task must hold cgroup_mutex to modify cgroups.
 *
 * Any task can increment and decrement the count field without lock.
 * So in general, code holding cgroup_mutex can't rely on the count
 * field not changing.  However, if the count goes to zero, then only
753
 * cgroup_attach_task() can increment it again.  Because a count of zero
754 755 756 757 758 759 760 761 762 763 764 765 766
 * means that no tasks are currently attached, therefore there is no
 * way a task attached to that cgroup can fork (the other way to
 * increment the count).  So code holding cgroup_mutex can safely
 * assume that if the count is zero, it will stay zero. Similarly, if
 * a task holds cgroup_mutex on a cgroup with zero count, it
 * knows that the cgroup won't be removed, as cgroup_rmdir()
 * needs that mutex.
 *
 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
 * (usually) take cgroup_mutex.  These are the two most performance
 * critical pieces of code here.  The exception occurs on cgroup_exit(),
 * when a task in a notify_on_release cgroup exits.  Then cgroup_mutex
 * is taken, and if the cgroup count is zero, a usermode call made
L
Li Zefan 已提交
767 768
 * to the release agent with the name of the cgroup (path relative to
 * the root of cgroup file system) as the argument.
769 770 771 772 773 774 775 776 777 778 779
 *
 * A cgroup can only be deleted if both its 'count' of using tasks
 * is zero, and its list of 'children' cgroups is empty.  Since all
 * tasks in the system use _some_ cgroup, and since there is always at
 * least one task in the system (init, pid == 1), therefore, top_cgroup
 * always has either children cgroups and/or using tasks.  So we don't
 * need a special hack to ensure that top_cgroup cannot be deleted.
 *
 *	The task_lock() exception
 *
 * The need for this exception arises from the action of
780
 * cgroup_attach_task(), which overwrites one task's cgroup pointer with
L
Li Zefan 已提交
781
 * another.  It does so using cgroup_mutex, however there are
782 783 784
 * several performance critical places that need to reference
 * task->cgroup without the expense of grabbing a system global
 * mutex.  Therefore except as noted below, when dereferencing or, as
785
 * in cgroup_attach_task(), modifying a task's cgroup pointer we use
786 787 788 789
 * task_lock(), which acts on a spinlock (task->alloc_lock) already in
 * the task_struct routinely used for such matters.
 *
 * P.S.  One more locking exception.  RCU is used to guard the
790
 * update of a tasks cgroup pointer by cgroup_attach_task()
791 792 793 794 795 796 797 798 799
 */

/*
 * A couple of forward declarations required, due to cyclic reference loop:
 * cgroup_mkdir -> cgroup_create -> cgroup_populate_dir ->
 * cgroup_add_file -> cgroup_create_file -> cgroup_dir_inode_operations
 * -> cgroup_mkdir.
 */

800
static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
801
static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry);
802
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask);
803
static const struct inode_operations cgroup_dir_inode_operations;
804
static const struct file_operations proc_cgroupstats_operations;
805 806

static struct backing_dev_info cgroup_backing_dev_info = {
807
	.name		= "cgroup",
808
	.capabilities	= BDI_CAP_NO_ACCT_AND_WRITEBACK,
809
};
810

A
Al Viro 已提交
811
static struct inode *cgroup_new_inode(umode_t mode, struct super_block *sb)
812 813 814 815
{
	struct inode *inode = new_inode(sb);

	if (inode) {
816
		inode->i_ino = get_next_ino();
817
		inode->i_mode = mode;
818 819
		inode->i_uid = current_fsuid();
		inode->i_gid = current_fsgid();
820 821 822 823 824 825
		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
		inode->i_mapping->backing_dev_info = &cgroup_backing_dev_info;
	}
	return inode;
}

826 827 828 829 830 831 832 833 834 835 836
static struct cgroup_name *cgroup_alloc_name(struct dentry *dentry)
{
	struct cgroup_name *name;

	name = kmalloc(sizeof(*name) + dentry->d_name.len + 1, GFP_KERNEL);
	if (!name)
		return NULL;
	strcpy(name->name, dentry->d_name.name);
	return name;
}

837 838
static void cgroup_free_fn(struct work_struct *work)
{
839
	struct cgroup *cgrp = container_of(work, struct cgroup, destroy_work);
840 841 842 843 844

	mutex_lock(&cgroup_mutex);
	cgrp->root->number_of_cgroups--;
	mutex_unlock(&cgroup_mutex);

845 846 847 848 849 850 851
	/*
	 * We get a ref to the parent's dentry, and put the ref when
	 * this cgroup is being freed, so it's guaranteed that the
	 * parent won't be destroyed before its children.
	 */
	dput(cgrp->parent->dentry);

852 853
	/*
	 * Drop the active superblock reference that we took when we
854 855
	 * created the cgroup. This will free cgrp->root, if we are
	 * holding the last reference to @sb.
856 857 858
	 */
	deactivate_super(cgrp->root->sb);

859
	cgroup_pidlist_destroy_all(cgrp);
860 861 862

	simple_xattrs_free(&cgrp->xattrs);

863
	kfree(rcu_dereference_raw(cgrp->name));
864 865 866 867 868 869 870
	kfree(cgrp);
}

static void cgroup_free_rcu(struct rcu_head *head)
{
	struct cgroup *cgrp = container_of(head, struct cgroup, rcu_head);

871
	INIT_WORK(&cgrp->destroy_work, cgroup_free_fn);
872
	queue_work(cgroup_destroy_wq, &cgrp->destroy_work);
873 874
}

875 876 877 878
static void cgroup_diput(struct dentry *dentry, struct inode *inode)
{
	/* is dentry a directory ? if so, kfree() associated cgroup */
	if (S_ISDIR(inode->i_mode)) {
879
		struct cgroup *cgrp = dentry->d_fsdata;
880

881
		BUG_ON(!(cgroup_is_dead(cgrp)));
882
		call_rcu(&cgrp->rcu_head, cgroup_free_rcu);
T
Tejun Heo 已提交
883 884 885 886 887 888 889
	} else {
		struct cfent *cfe = __d_cfe(dentry);
		struct cgroup *cgrp = dentry->d_parent->d_fsdata;

		WARN_ONCE(!list_empty(&cfe->node) &&
			  cgrp != &cgrp->root->top_cgroup,
			  "cfe still linked for %s\n", cfe->type->name);
L
Li Zefan 已提交
890
		simple_xattrs_free(&cfe->xattrs);
T
Tejun Heo 已提交
891
		kfree(cfe);
892 893 894 895 896 897 898 899 900 901 902 903 904
	}
	iput(inode);
}

static void remove_dir(struct dentry *d)
{
	struct dentry *parent = dget(d->d_parent);

	d_delete(d);
	simple_rmdir(parent->d_inode, d);
	dput(parent);
}

905
static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
T
Tejun Heo 已提交
906 907 908 909 910 911
{
	struct cfent *cfe;

	lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
	lockdep_assert_held(&cgroup_mutex);

912 913 914 915
	/*
	 * If we're doing cleanup due to failure of cgroup_create(),
	 * the corresponding @cfe may not exist.
	 */
T
Tejun Heo 已提交
916 917 918 919 920 921 922 923
	list_for_each_entry(cfe, &cgrp->files, node) {
		struct dentry *d = cfe->dentry;

		if (cft && cfe->type != cft)
			continue;

		dget(d);
		d_delete(d);
924
		simple_unlink(cgrp->dentry->d_inode, d);
T
Tejun Heo 已提交
925 926 927
		list_del_init(&cfe->node);
		dput(d);

928
		break;
929
	}
T
Tejun Heo 已提交
930 931
}

932
/**
933
 * cgroup_clear_dir - remove subsys files in a cgroup directory
934
 * @cgrp: target cgroup
935 936
 * @subsys_mask: mask of the subsystem ids whose files should be removed
 */
937
static void cgroup_clear_dir(struct cgroup *cgrp, unsigned long subsys_mask)
T
Tejun Heo 已提交
938
{
939
	struct cgroup_subsys *ss;
940
	int i;
T
Tejun Heo 已提交
941

942
	for_each_subsys(ss, i) {
943
		struct cftype_set *set;
944 945

		if (!test_bit(i, &subsys_mask))
946 947
			continue;
		list_for_each_entry(set, &ss->cftsets, node)
948
			cgroup_addrm_files(cgrp, set->cfts, false);
949
	}
950 951 952 953 954 955 956
}

/*
 * NOTE : the dentry must have been dget()'ed
 */
static void cgroup_d_remove_dir(struct dentry *dentry)
{
N
Nick Piggin 已提交
957
	struct dentry *parent;
958

N
Nick Piggin 已提交
959 960
	parent = dentry->d_parent;
	spin_lock(&parent->d_lock);
961
	spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
962
	list_del_init(&dentry->d_u.d_child);
N
Nick Piggin 已提交
963 964
	spin_unlock(&dentry->d_lock);
	spin_unlock(&parent->d_lock);
965 966 967
	remove_dir(dentry);
}

B
Ben Blum 已提交
968
/*
B
Ben Blum 已提交
969 970 971
 * Call with cgroup_mutex held. Drops reference counts on modules, including
 * any duplicate ones that parse_cgroupfs_options took. If this function
 * returns an error, no reference counts are touched.
B
Ben Blum 已提交
972
 */
973
static int rebind_subsystems(struct cgroupfs_root *root,
974
			     unsigned long added_mask, unsigned removed_mask)
975
{
976
	struct cgroup *cgrp = &root->top_cgroup;
977
	struct cgroup_subsys *ss;
978
	unsigned long pinned = 0;
979
	int i, ret;
980

B
Ben Blum 已提交
981
	BUG_ON(!mutex_is_locked(&cgroup_mutex));
T
Tejun Heo 已提交
982
	BUG_ON(!mutex_is_locked(&cgroup_root_mutex));
B
Ben Blum 已提交
983

984
	/* Check that any added subsystems are currently free */
985
	for_each_subsys(ss, i) {
986
		if (!(added_mask & (1 << i)))
987
			continue;
988

989
		/* is the subsystem mounted elsewhere? */
990
		if (ss->root != &cgroup_dummy_root) {
991 992 993 994 995 996 997 998
			ret = -EBUSY;
			goto out_put;
		}

		/* pin the module */
		if (!try_module_get(ss->module)) {
			ret = -ENOENT;
			goto out_put;
999
		}
1000
		pinned |= 1 << i;
1001 1002
	}

1003 1004 1005 1006
	/* subsys could be missing if unloaded between parsing and here */
	if (added_mask != pinned) {
		ret = -ENOENT;
		goto out_put;
1007 1008
	}

1009 1010
	ret = cgroup_populate_dir(cgrp, added_mask);
	if (ret)
1011
		goto out_put;
1012 1013 1014 1015 1016 1017

	/*
	 * Nothing can fail from this point on.  Remove files for the
	 * removed subsystems and rebind each subsystem.
	 */
	cgroup_clear_dir(cgrp, removed_mask);
1018

1019
	for_each_subsys(ss, i) {
1020
		unsigned long bit = 1UL << i;
1021

1022
		if (bit & added_mask) {
1023
			/* We're binding this subsystem to this hierarchy */
1024 1025 1026
			BUG_ON(cgroup_css(cgrp, ss));
			BUG_ON(!cgroup_css(cgroup_dummy_top, ss));
			BUG_ON(cgroup_css(cgroup_dummy_top, ss)->cgroup != cgroup_dummy_top);
1027

1028
			rcu_assign_pointer(cgrp->subsys[i],
1029 1030
					   cgroup_css(cgroup_dummy_top, ss));
			cgroup_css(cgrp, ss)->cgroup = cgrp;
1031

1032
			ss->root = root;
1033
			if (ss->bind)
1034
				ss->bind(cgroup_css(cgrp, ss));
1035

B
Ben Blum 已提交
1036
			/* refcount was already taken, and we're keeping it */
1037
			root->subsys_mask |= bit;
1038
		} else if (bit & removed_mask) {
1039
			/* We're removing this subsystem */
1040 1041
			BUG_ON(cgroup_css(cgrp, ss) != cgroup_css(cgroup_dummy_top, ss));
			BUG_ON(cgroup_css(cgrp, ss)->cgroup != cgrp);
1042

1043
			if (ss->bind)
1044
				ss->bind(cgroup_css(cgroup_dummy_top, ss));
1045

1046
			cgroup_css(cgroup_dummy_top, ss)->cgroup = cgroup_dummy_top;
1047 1048
			RCU_INIT_POINTER(cgrp->subsys[i], NULL);

1049
			cgroup_subsys[i]->root = &cgroup_dummy_root;
1050

B
Ben Blum 已提交
1051 1052
			/* subsystem is now free - drop reference on module */
			module_put(ss->module);
1053
			root->subsys_mask &= ~bit;
1054 1055 1056
		}
	}

1057 1058 1059 1060 1061 1062
	/*
	 * Mark @root has finished binding subsystems.  @root->subsys_mask
	 * now matches the bound subsystems.
	 */
	root->flags |= CGRP_ROOT_SUBSYS_BOUND;

1063
	return 0;
1064 1065 1066 1067 1068 1069

out_put:
	for_each_subsys(ss, i)
		if (pinned & (1 << i))
			module_put(ss->module);
	return ret;
1070 1071
}

1072
static int cgroup_show_options(struct seq_file *seq, struct dentry *dentry)
1073
{
1074
	struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
1075
	struct cgroup_subsys *ss;
T
Tejun Heo 已提交
1076
	int ssid;
1077

T
Tejun Heo 已提交
1078
	mutex_lock(&cgroup_root_mutex);
T
Tejun Heo 已提交
1079 1080 1081
	for_each_subsys(ss, ssid)
		if (root->subsys_mask & (1 << ssid))
			seq_printf(seq, ",%s", ss->name);
1082 1083
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR)
		seq_puts(seq, ",sane_behavior");
1084
	if (root->flags & CGRP_ROOT_NOPREFIX)
1085
		seq_puts(seq, ",noprefix");
1086
	if (root->flags & CGRP_ROOT_XATTR)
A
Aristeu Rozanski 已提交
1087
		seq_puts(seq, ",xattr");
1088 1089
	if (strlen(root->release_agent_path))
		seq_printf(seq, ",release_agent=%s", root->release_agent_path);
1090
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags))
1091
		seq_puts(seq, ",clone_children");
1092 1093
	if (strlen(root->name))
		seq_printf(seq, ",name=%s", root->name);
T
Tejun Heo 已提交
1094
	mutex_unlock(&cgroup_root_mutex);
1095 1096 1097 1098
	return 0;
}

struct cgroup_sb_opts {
1099
	unsigned long subsys_mask;
1100
	unsigned long flags;
1101
	char *release_agent;
1102
	bool cpuset_clone_children;
1103
	char *name;
1104 1105
	/* User explicitly requested empty subsystem */
	bool none;
1106 1107

	struct cgroupfs_root *new_root;
1108

1109 1110
};

B
Ben Blum 已提交
1111
/*
1112 1113 1114 1115
 * Convert a hierarchy specifier into a bitmask of subsystems and
 * flags. Call with cgroup_mutex held to protect the cgroup_subsys[]
 * array. This function takes refcounts on subsystems to be used, unless it
 * returns error, in which case no refcounts are taken.
B
Ben Blum 已提交
1116
 */
B
Ben Blum 已提交
1117
static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
1118
{
1119 1120
	char *token, *o = data;
	bool all_ss = false, one_ss = false;
1121
	unsigned long mask = (unsigned long)-1;
1122 1123
	struct cgroup_subsys *ss;
	int i;
1124

B
Ben Blum 已提交
1125 1126
	BUG_ON(!mutex_is_locked(&cgroup_mutex));

1127 1128 1129
#ifdef CONFIG_CPUSETS
	mask = ~(1UL << cpuset_subsys_id);
#endif
1130

1131
	memset(opts, 0, sizeof(*opts));
1132 1133 1134 1135

	while ((token = strsep(&o, ",")) != NULL) {
		if (!*token)
			return -EINVAL;
1136
		if (!strcmp(token, "none")) {
1137 1138
			/* Explicitly have no subsystems */
			opts->none = true;
1139 1140 1141 1142 1143 1144 1145 1146 1147
			continue;
		}
		if (!strcmp(token, "all")) {
			/* Mutually exclusive option 'all' + subsystem name */
			if (one_ss)
				return -EINVAL;
			all_ss = true;
			continue;
		}
1148 1149 1150 1151
		if (!strcmp(token, "__DEVEL__sane_behavior")) {
			opts->flags |= CGRP_ROOT_SANE_BEHAVIOR;
			continue;
		}
1152
		if (!strcmp(token, "noprefix")) {
1153
			opts->flags |= CGRP_ROOT_NOPREFIX;
1154 1155 1156
			continue;
		}
		if (!strcmp(token, "clone_children")) {
1157
			opts->cpuset_clone_children = true;
1158 1159
			continue;
		}
A
Aristeu Rozanski 已提交
1160
		if (!strcmp(token, "xattr")) {
1161
			opts->flags |= CGRP_ROOT_XATTR;
A
Aristeu Rozanski 已提交
1162 1163
			continue;
		}
1164
		if (!strncmp(token, "release_agent=", 14)) {
1165 1166 1167
			/* Specifying two release agents is forbidden */
			if (opts->release_agent)
				return -EINVAL;
1168
			opts->release_agent =
1169
				kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
1170 1171
			if (!opts->release_agent)
				return -ENOMEM;
1172 1173 1174
			continue;
		}
		if (!strncmp(token, "name=", 5)) {
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
			const char *name = token + 5;
			/* Can't specify an empty name */
			if (!strlen(name))
				return -EINVAL;
			/* Must match [\w.-]+ */
			for (i = 0; i < strlen(name); i++) {
				char c = name[i];
				if (isalnum(c))
					continue;
				if ((c == '.') || (c == '-') || (c == '_'))
					continue;
				return -EINVAL;
			}
			/* Specifying two names is forbidden */
			if (opts->name)
				return -EINVAL;
			opts->name = kstrndup(name,
1192
					      MAX_CGROUP_ROOT_NAMELEN - 1,
1193 1194 1195
					      GFP_KERNEL);
			if (!opts->name)
				return -ENOMEM;
1196 1197 1198 1199

			continue;
		}

1200
		for_each_subsys(ss, i) {
1201 1202 1203 1204 1205 1206 1207 1208
			if (strcmp(token, ss->name))
				continue;
			if (ss->disabled)
				continue;

			/* Mutually exclusive option 'all' + subsystem name */
			if (all_ss)
				return -EINVAL;
1209
			set_bit(i, &opts->subsys_mask);
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
			one_ss = true;

			break;
		}
		if (i == CGROUP_SUBSYS_COUNT)
			return -ENOENT;
	}

	/*
	 * If the 'all' option was specified select all the subsystems,
1220 1221
	 * otherwise if 'none', 'name=' and a subsystem name options
	 * were not specified, let's default to 'all'
1222
	 */
1223 1224 1225 1226
	if (all_ss || (!one_ss && !opts->none && !opts->name))
		for_each_subsys(ss, i)
			if (!ss->disabled)
				set_bit(i, &opts->subsys_mask);
1227

1228 1229
	/* Consistency checks */

1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
	if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) {
		pr_warning("cgroup: sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n");

		if (opts->flags & CGRP_ROOT_NOPREFIX) {
			pr_err("cgroup: sane_behavior: noprefix is not allowed\n");
			return -EINVAL;
		}

		if (opts->cpuset_clone_children) {
			pr_err("cgroup: sane_behavior: clone_children is not allowed\n");
			return -EINVAL;
		}
	}

1244 1245 1246 1247 1248
	/*
	 * Option noprefix was introduced just for backward compatibility
	 * with the old cpuset, so we allow noprefix only if mounting just
	 * the cpuset subsystem.
	 */
1249
	if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
1250 1251
		return -EINVAL;

1252 1253

	/* Can't specify "none" and some subsystems */
1254
	if (opts->subsys_mask && opts->none)
1255 1256 1257 1258 1259 1260
		return -EINVAL;

	/*
	 * We either have to specify by name or by subsystems. (So all
	 * empty hierarchies must have a name).
	 */
1261
	if (!opts->subsys_mask && !opts->name)
1262 1263 1264 1265 1266 1267 1268 1269 1270
		return -EINVAL;

	return 0;
}

static int cgroup_remount(struct super_block *sb, int *flags, char *data)
{
	int ret = 0;
	struct cgroupfs_root *root = sb->s_fs_info;
1271
	struct cgroup *cgrp = &root->top_cgroup;
1272
	struct cgroup_sb_opts opts;
1273
	unsigned long added_mask, removed_mask;
1274

1275 1276 1277 1278 1279
	if (root->flags & CGRP_ROOT_SANE_BEHAVIOR) {
		pr_err("cgroup: sane_behavior: remount is not allowed\n");
		return -EINVAL;
	}

1280
	mutex_lock(&cgrp->dentry->d_inode->i_mutex);
1281
	mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1282
	mutex_lock(&cgroup_root_mutex);
1283 1284 1285 1286 1287 1288

	/* See what subsystems are wanted */
	ret = parse_cgroupfs_options(data, &opts);
	if (ret)
		goto out_unlock;

1289
	if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
1290 1291 1292
		pr_warning("cgroup: option changes via remount are deprecated (pid=%d comm=%s)\n",
			   task_tgid_nr(current), current->comm);

1293 1294
	added_mask = opts.subsys_mask & ~root->subsys_mask;
	removed_mask = root->subsys_mask & ~opts.subsys_mask;
1295

B
Ben Blum 已提交
1296
	/* Don't allow flags or name to change at remount */
1297
	if (((opts.flags ^ root->flags) & CGRP_ROOT_OPTION_MASK) ||
B
Ben Blum 已提交
1298
	    (opts.name && strcmp(opts.name, root->name))) {
1299 1300 1301
		pr_err("cgroup: option or name mismatch, new: 0x%lx \"%s\", old: 0x%lx \"%s\"\n",
		       opts.flags & CGRP_ROOT_OPTION_MASK, opts.name ?: "",
		       root->flags & CGRP_ROOT_OPTION_MASK, root->name);
1302 1303 1304 1305
		ret = -EINVAL;
		goto out_unlock;
	}

1306 1307 1308
	/* remounting is not allowed for populated hierarchies */
	if (root->number_of_cgroups > 1) {
		ret = -EBUSY;
1309
		goto out_unlock;
B
Ben Blum 已提交
1310
	}
1311

1312
	ret = rebind_subsystems(root, added_mask, removed_mask);
1313
	if (ret)
1314
		goto out_unlock;
1315

1316 1317
	if (opts.release_agent)
		strcpy(root->release_agent_path, opts.release_agent);
1318
 out_unlock:
1319
	kfree(opts.release_agent);
1320
	kfree(opts.name);
T
Tejun Heo 已提交
1321
	mutex_unlock(&cgroup_root_mutex);
1322
	mutex_unlock(&cgroup_mutex);
1323
	mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
1324 1325 1326
	return ret;
}

1327
static const struct super_operations cgroup_ops = {
1328 1329 1330 1331 1332 1333
	.statfs = simple_statfs,
	.drop_inode = generic_delete_inode,
	.show_options = cgroup_show_options,
	.remount_fs = cgroup_remount,
};

1334 1335 1336 1337
static void init_cgroup_housekeeping(struct cgroup *cgrp)
{
	INIT_LIST_HEAD(&cgrp->sibling);
	INIT_LIST_HEAD(&cgrp->children);
T
Tejun Heo 已提交
1338
	INIT_LIST_HEAD(&cgrp->files);
1339
	INIT_LIST_HEAD(&cgrp->cset_links);
1340
	INIT_LIST_HEAD(&cgrp->release_list);
1341 1342
	INIT_LIST_HEAD(&cgrp->pidlists);
	mutex_init(&cgrp->pidlist_mutex);
T
Tejun Heo 已提交
1343
	cgrp->dummy_css.cgroup = cgrp;
A
Aristeu Rozanski 已提交
1344
	simple_xattrs_init(&cgrp->xattrs);
1345
}
1346

1347 1348
static void init_cgroup_root(struct cgroupfs_root *root)
{
1349
	struct cgroup *cgrp = &root->top_cgroup;
1350

1351 1352
	INIT_LIST_HEAD(&root->root_list);
	root->number_of_cgroups = 1;
1353
	cgrp->root = root;
1354
	RCU_INIT_POINTER(cgrp->name, &root_cgroup_name);
1355
	init_cgroup_housekeeping(cgrp);
1356
	idr_init(&root->cgroup_idr);
1357 1358
}

1359
static int cgroup_init_root_id(struct cgroupfs_root *root, int start, int end)
1360
{
1361
	int id;
1362

T
Tejun Heo 已提交
1363 1364 1365
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&cgroup_root_mutex);

1366 1367
	id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, start, end,
			      GFP_KERNEL);
1368 1369 1370 1371
	if (id < 0)
		return id;

	root->hierarchy_id = id;
1372 1373 1374 1375 1376
	return 0;
}

static void cgroup_exit_root_id(struct cgroupfs_root *root)
{
T
Tejun Heo 已提交
1377 1378 1379
	lockdep_assert_held(&cgroup_mutex);
	lockdep_assert_held(&cgroup_root_mutex);

1380
	if (root->hierarchy_id) {
1381
		idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1382 1383
		root->hierarchy_id = 0;
	}
1384 1385
}

1386 1387
static int cgroup_test_super(struct super_block *sb, void *data)
{
1388
	struct cgroup_sb_opts *opts = data;
1389 1390
	struct cgroupfs_root *root = sb->s_fs_info;

1391 1392 1393
	/* If we asked for a name then it must match */
	if (opts->name && strcmp(opts->name, root->name))
		return 0;
1394

1395 1396 1397 1398
	/*
	 * If we asked for subsystems (or explicitly for no
	 * subsystems) then they must match
	 */
1399 1400
	if ((opts->subsys_mask || opts->none)
	    && (opts->subsys_mask != root->subsys_mask))
1401 1402 1403 1404 1405
		return 0;

	return 1;
}

1406 1407 1408 1409
static struct cgroupfs_root *cgroup_root_from_opts(struct cgroup_sb_opts *opts)
{
	struct cgroupfs_root *root;

1410
	if (!opts->subsys_mask && !opts->none)
1411 1412 1413 1414 1415 1416 1417
		return NULL;

	root = kzalloc(sizeof(*root), GFP_KERNEL);
	if (!root)
		return ERR_PTR(-ENOMEM);

	init_cgroup_root(root);
1418

1419 1420 1421 1422 1423 1424 1425 1426
	/*
	 * We need to set @root->subsys_mask now so that @root can be
	 * matched by cgroup_test_super() before it finishes
	 * initialization; otherwise, competing mounts with the same
	 * options may try to bind the same subsystems instead of waiting
	 * for the first one leading to unexpected mount errors.
	 * SUBSYS_BOUND will be set once actual binding is complete.
	 */
1427
	root->subsys_mask = opts->subsys_mask;
1428 1429 1430 1431 1432
	root->flags = opts->flags;
	if (opts->release_agent)
		strcpy(root->release_agent_path, opts->release_agent);
	if (opts->name)
		strcpy(root->name, opts->name);
1433 1434
	if (opts->cpuset_clone_children)
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->top_cgroup.flags);
1435 1436 1437
	return root;
}

1438
static void cgroup_free_root(struct cgroupfs_root *root)
1439
{
1440 1441 1442
	if (root) {
		/* hierarhcy ID shoulid already have been released */
		WARN_ON_ONCE(root->hierarchy_id);
1443

1444
		idr_destroy(&root->cgroup_idr);
1445 1446
		kfree(root);
	}
1447 1448
}

1449 1450 1451
static int cgroup_set_super(struct super_block *sb, void *data)
{
	int ret;
1452 1453 1454 1455 1456 1457
	struct cgroup_sb_opts *opts = data;

	/* If we don't have a new root, we can't set up a new sb */
	if (!opts->new_root)
		return -EINVAL;

1458
	BUG_ON(!opts->subsys_mask && !opts->none);
1459 1460 1461 1462 1463

	ret = set_anon_super(sb, NULL);
	if (ret)
		return ret;

1464 1465
	sb->s_fs_info = opts->new_root;
	opts->new_root->sb = sb;
1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476

	sb->s_blocksize = PAGE_CACHE_SIZE;
	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
	sb->s_magic = CGROUP_SUPER_MAGIC;
	sb->s_op = &cgroup_ops;

	return 0;
}

static int cgroup_get_rootdir(struct super_block *sb)
{
A
Al Viro 已提交
1477 1478
	static const struct dentry_operations cgroup_dops = {
		.d_iput = cgroup_diput,
1479
		.d_delete = always_delete_dentry,
A
Al Viro 已提交
1480 1481
	};

1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
	struct inode *inode =
		cgroup_new_inode(S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR, sb);

	if (!inode)
		return -ENOMEM;

	inode->i_fop = &simple_dir_operations;
	inode->i_op = &cgroup_dir_inode_operations;
	/* directories start off with i_nlink == 2 (for "." entry) */
	inc_nlink(inode);
1492 1493
	sb->s_root = d_make_root(inode);
	if (!sb->s_root)
1494
		return -ENOMEM;
A
Al Viro 已提交
1495 1496
	/* for everything else we want ->d_op set */
	sb->s_d_op = &cgroup_dops;
1497 1498 1499
	return 0;
}

A
Al Viro 已提交
1500
static struct dentry *cgroup_mount(struct file_system_type *fs_type,
1501
			 int flags, const char *unused_dev_name,
A
Al Viro 已提交
1502
			 void *data)
1503 1504
{
	struct cgroup_sb_opts opts;
1505
	struct cgroupfs_root *root;
1506 1507
	int ret = 0;
	struct super_block *sb;
1508
	struct cgroupfs_root *new_root;
1509
	struct list_head tmp_links;
T
Tejun Heo 已提交
1510
	struct inode *inode;
1511
	const struct cred *cred;
1512 1513

	/* First find the desired set of subsystems */
B
Ben Blum 已提交
1514
	mutex_lock(&cgroup_mutex);
1515
	ret = parse_cgroupfs_options(data, &opts);
B
Ben Blum 已提交
1516
	mutex_unlock(&cgroup_mutex);
1517 1518
	if (ret)
		goto out_err;
1519

1520 1521 1522 1523 1524 1525 1526
	/*
	 * Allocate a new cgroup root. We may not need it if we're
	 * reusing an existing hierarchy.
	 */
	new_root = cgroup_root_from_opts(&opts);
	if (IS_ERR(new_root)) {
		ret = PTR_ERR(new_root);
1527
		goto out_err;
1528
	}
1529
	opts.new_root = new_root;
1530

1531
	/* Locate an existing or new sb for this hierarchy */
D
David Howells 已提交
1532
	sb = sget(fs_type, cgroup_test_super, cgroup_set_super, 0, &opts);
1533
	if (IS_ERR(sb)) {
1534
		ret = PTR_ERR(sb);
1535
		cgroup_free_root(opts.new_root);
1536
		goto out_err;
1537 1538
	}

1539 1540 1541 1542
	root = sb->s_fs_info;
	BUG_ON(!root);
	if (root == opts.new_root) {
		/* We used the new root structure, so this is a new hierarchy */
1543
		struct cgroup *root_cgrp = &root->top_cgroup;
1544
		struct cgroupfs_root *existing_root;
1545
		int i;
1546
		struct css_set *cset;
1547 1548 1549 1550 1551 1552

		BUG_ON(sb->s_root != NULL);

		ret = cgroup_get_rootdir(sb);
		if (ret)
			goto drop_new_super;
1553
		inode = sb->s_root->d_inode;
1554

1555
		mutex_lock(&inode->i_mutex);
1556
		mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1557
		mutex_lock(&cgroup_root_mutex);
1558

1559 1560 1561 1562 1563
		root_cgrp->id = idr_alloc(&root->cgroup_idr, root_cgrp,
					   0, 1, GFP_KERNEL);
		if (root_cgrp->id < 0)
			goto unlock_drop;

T
Tejun Heo 已提交
1564 1565 1566 1567 1568 1569
		/* Check for name clashes with existing mounts */
		ret = -EBUSY;
		if (strlen(root->name))
			for_each_active_root(existing_root)
				if (!strcmp(existing_root->name, root->name))
					goto unlock_drop;
1570

1571 1572 1573 1574 1575 1576 1577
		/*
		 * We're accessing css_set_count without locking
		 * css_set_lock here, but that's OK - it can only be
		 * increased by someone holding cgroup_lock, and
		 * that's us. The worst that can happen is that we
		 * have some link structures left over
		 */
1578
		ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
T
Tejun Heo 已提交
1579 1580
		if (ret)
			goto unlock_drop;
1581

1582 1583
		/* ID 0 is reserved for dummy root, 1 for unified hierarchy */
		ret = cgroup_init_root_id(root, 2, 0);
1584 1585 1586
		if (ret)
			goto unlock_drop;

1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
		sb->s_root->d_fsdata = root_cgrp;
		root_cgrp->dentry = sb->s_root;

		/*
		 * We're inside get_sb() and will call lookup_one_len() to
		 * create the root files, which doesn't work if SELinux is
		 * in use.  The following cred dancing somehow works around
		 * it.  See 2ce9738ba ("cgroupfs: use init_cred when
		 * populating new cgroupfs mount") for more details.
		 */
		cred = override_creds(&init_cred);

1599
		ret = cgroup_addrm_files(root_cgrp, cgroup_base_files, true);
1600 1601 1602
		if (ret)
			goto rm_base_files;

1603
		ret = rebind_subsystems(root, root->subsys_mask, 0);
1604 1605 1606 1607 1608
		if (ret)
			goto rm_base_files;

		revert_creds(cred);

B
Ben Blum 已提交
1609 1610 1611 1612 1613
		/*
		 * There must be no failure case after here, since rebinding
		 * takes care of subsystems' refcounts, which are explicitly
		 * dropped in the failure exit path.
		 */
1614

1615 1616
		list_add(&root->root_list, &cgroup_roots);
		cgroup_root_count++;
1617

1618 1619 1620
		/* Link the top cgroup in this hierarchy into all
		 * the css_set objects */
		write_lock(&css_set_lock);
1621
		hash_for_each(css_set_table, i, cset, hlist)
1622
			link_css_set(&tmp_links, cset, root_cgrp);
1623 1624
		write_unlock(&css_set_lock);

1625
		free_cgrp_cset_links(&tmp_links);
1626

1627
		BUG_ON(!list_empty(&root_cgrp->children));
1628 1629
		BUG_ON(root->number_of_cgroups != 1);

T
Tejun Heo 已提交
1630
		mutex_unlock(&cgroup_root_mutex);
1631
		mutex_unlock(&cgroup_mutex);
1632
		mutex_unlock(&inode->i_mutex);
1633 1634 1635 1636 1637
	} else {
		/*
		 * We re-used an existing hierarchy - the new root (if
		 * any) is not needed
		 */
1638
		cgroup_free_root(opts.new_root);
1639

1640
		if ((root->flags ^ opts.flags) & CGRP_ROOT_OPTION_MASK) {
1641 1642 1643 1644 1645 1646 1647
			if ((root->flags | opts.flags) & CGRP_ROOT_SANE_BEHAVIOR) {
				pr_err("cgroup: sane_behavior: new mount options should match the existing superblock\n");
				ret = -EINVAL;
				goto drop_new_super;
			} else {
				pr_warning("cgroup: new mount options do not match the existing superblock, will be ignored\n");
			}
1648
		}
1649 1650
	}

1651 1652
	kfree(opts.release_agent);
	kfree(opts.name);
A
Al Viro 已提交
1653
	return dget(sb->s_root);
1654

1655 1656
 rm_base_files:
	free_cgrp_cset_links(&tmp_links);
1657
	cgroup_addrm_files(&root->top_cgroup, cgroup_base_files, false);
1658
	revert_creds(cred);
T
Tejun Heo 已提交
1659
 unlock_drop:
1660
	cgroup_exit_root_id(root);
T
Tejun Heo 已提交
1661 1662 1663
	mutex_unlock(&cgroup_root_mutex);
	mutex_unlock(&cgroup_mutex);
	mutex_unlock(&inode->i_mutex);
1664
 drop_new_super:
1665
	deactivate_locked_super(sb);
1666 1667 1668
 out_err:
	kfree(opts.release_agent);
	kfree(opts.name);
A
Al Viro 已提交
1669
	return ERR_PTR(ret);
1670 1671 1672 1673
}

static void cgroup_kill_sb(struct super_block *sb) {
	struct cgroupfs_root *root = sb->s_fs_info;
1674
	struct cgroup *cgrp = &root->top_cgroup;
1675
	struct cgrp_cset_link *link, *tmp_link;
1676 1677 1678 1679 1680
	int ret;

	BUG_ON(!root);

	BUG_ON(root->number_of_cgroups != 1);
1681
	BUG_ON(!list_empty(&cgrp->children));
1682

1683
	mutex_lock(&cgrp->dentry->d_inode->i_mutex);
1684
	mutex_lock(&cgroup_mutex);
T
Tejun Heo 已提交
1685
	mutex_lock(&cgroup_root_mutex);
1686 1687

	/* Rebind all subsystems back to the default hierarchy */
1688 1689 1690 1691 1692
	if (root->flags & CGRP_ROOT_SUBSYS_BOUND) {
		ret = rebind_subsystems(root, 0, root->subsys_mask);
		/* Shouldn't be able to fail ... */
		BUG_ON(ret);
	}
1693

1694
	/*
1695
	 * Release all the links from cset_links to this hierarchy's
1696 1697 1698
	 * root cgroup
	 */
	write_lock(&css_set_lock);
K
KOSAKI Motohiro 已提交
1699

1700 1701 1702
	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
		list_del(&link->cset_link);
		list_del(&link->cgrp_link);
1703 1704 1705 1706
		kfree(link);
	}
	write_unlock(&css_set_lock);

1707 1708
	if (!list_empty(&root->root_list)) {
		list_del(&root->root_list);
1709
		cgroup_root_count--;
1710
	}
1711

1712 1713
	cgroup_exit_root_id(root);

T
Tejun Heo 已提交
1714
	mutex_unlock(&cgroup_root_mutex);
1715
	mutex_unlock(&cgroup_mutex);
1716
	mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
1717

A
Aristeu Rozanski 已提交
1718 1719
	simple_xattrs_free(&cgrp->xattrs);

1720
	kill_litter_super(sb);
1721
	cgroup_free_root(root);
1722 1723 1724 1725
}

static struct file_system_type cgroup_fs_type = {
	.name = "cgroup",
A
Al Viro 已提交
1726
	.mount = cgroup_mount,
1727 1728 1729
	.kill_sb = cgroup_kill_sb,
};

1730 1731
static struct kobject *cgroup_kobj;

L
Li Zefan 已提交
1732 1733 1734 1735 1736 1737
/**
 * cgroup_path - generate the path of a cgroup
 * @cgrp: the cgroup in question
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
1738 1739 1740 1741 1742 1743
 * Writes path of cgroup into buf.  Returns 0 on success, -errno on error.
 *
 * We can't generate cgroup path using dentry->d_name, as accessing
 * dentry->name must be protected by irq-unsafe dentry->d_lock or parent
 * inode's i_mutex, while on the other hand cgroup_path() can be called
 * with some irq-safe spinlocks held.
1744
 */
1745
int cgroup_path(const struct cgroup *cgrp, char *buf, int buflen)
1746
{
1747
	int ret = -ENAMETOOLONG;
1748
	char *start;
1749

1750 1751 1752
	if (!cgrp->parent) {
		if (strlcpy(buf, "/", buflen) >= buflen)
			return -ENAMETOOLONG;
1753 1754 1755
		return 0;
	}

1756 1757
	start = buf + buflen - 1;
	*start = '\0';
1758

1759
	rcu_read_lock();
1760
	do {
1761 1762 1763 1764
		const char *name = cgroup_name(cgrp);
		int len;

		len = strlen(name);
1765
		if ((start -= len) < buf)
1766 1767
			goto out;
		memcpy(start, name, len);
1768

1769
		if (--start < buf)
1770
			goto out;
1771
		*start = '/';
1772 1773

		cgrp = cgrp->parent;
1774
	} while (cgrp->parent);
1775
	ret = 0;
1776
	memmove(buf, start, buf + buflen - start);
1777 1778 1779
out:
	rcu_read_unlock();
	return ret;
1780
}
B
Ben Blum 已提交
1781
EXPORT_SYMBOL_GPL(cgroup_path);
1782

1783
/**
1784
 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
1785 1786 1787 1788
 * @task: target task
 * @buf: the buffer to write the path into
 * @buflen: the length of the buffer
 *
1789 1790 1791 1792 1793 1794
 * Determine @task's cgroup on the first (the one with the lowest non-zero
 * hierarchy_id) cgroup hierarchy and copy its path into @buf.  This
 * function grabs cgroup_mutex and shouldn't be used inside locks used by
 * cgroup controller callbacks.
 *
 * Returns 0 on success, fails with -%ENAMETOOLONG if @buflen is too short.
1795
 */
1796
int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
1797 1798
{
	struct cgroupfs_root *root;
1799 1800 1801 1802 1803
	struct cgroup *cgrp;
	int hierarchy_id = 1, ret = 0;

	if (buflen < 2)
		return -ENAMETOOLONG;
1804 1805 1806

	mutex_lock(&cgroup_mutex);

1807 1808
	root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);

1809 1810 1811
	if (root) {
		cgrp = task_cgroup_from_root(task, root);
		ret = cgroup_path(cgrp, buf, buflen);
1812 1813 1814
	} else {
		/* if no hierarchy exists, everyone is in "/" */
		memcpy(buf, "/", 2);
1815 1816 1817 1818 1819
	}

	mutex_unlock(&cgroup_mutex);
	return ret;
}
1820
EXPORT_SYMBOL_GPL(task_cgroup_path);
1821

1822 1823 1824
/*
 * Control Group taskset
 */
1825 1826 1827
struct task_and_cgroup {
	struct task_struct	*task;
	struct cgroup		*cgrp;
L
Li Zefan 已提交
1828
	struct css_set		*cset;
1829 1830
};

1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
struct cgroup_taskset {
	struct task_and_cgroup	single;
	struct flex_array	*tc_array;
	int			tc_array_len;
	int			idx;
	struct cgroup		*cur_cgrp;
};

/**
 * cgroup_taskset_first - reset taskset and return the first task
 * @tset: taskset of interest
 *
 * @tset iteration is initialized and the first task is returned.
 */
struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset)
{
	if (tset->tc_array) {
		tset->idx = 0;
		return cgroup_taskset_next(tset);
	} else {
		tset->cur_cgrp = tset->single.cgrp;
		return tset->single.task;
	}
}
EXPORT_SYMBOL_GPL(cgroup_taskset_first);

/**
 * cgroup_taskset_next - iterate to the next task in taskset
 * @tset: taskset of interest
 *
 * Return the next task in @tset.  Iteration must have been initialized
 * with cgroup_taskset_first().
 */
struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset)
{
	struct task_and_cgroup *tc;

	if (!tset->tc_array || tset->idx >= tset->tc_array_len)
		return NULL;

	tc = flex_array_get(tset->tc_array, tset->idx++);
	tset->cur_cgrp = tc->cgrp;
	return tc->task;
}
EXPORT_SYMBOL_GPL(cgroup_taskset_next);

/**
1878
 * cgroup_taskset_cur_css - return the matching css for the current task
1879
 * @tset: taskset of interest
1880
 * @subsys_id: the ID of the target subsystem
1881
 *
1882 1883 1884
 * Return the css for the current (last returned) task of @tset for
 * subsystem specified by @subsys_id.  This function must be preceded by
 * either cgroup_taskset_first() or cgroup_taskset_next().
1885
 */
1886 1887
struct cgroup_subsys_state *cgroup_taskset_cur_css(struct cgroup_taskset *tset,
						   int subsys_id)
1888
{
1889
	return cgroup_css(tset->cur_cgrp, cgroup_subsys[subsys_id]);
1890
}
1891
EXPORT_SYMBOL_GPL(cgroup_taskset_cur_css);
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903

/**
 * cgroup_taskset_size - return the number of tasks in taskset
 * @tset: taskset of interest
 */
int cgroup_taskset_size(struct cgroup_taskset *tset)
{
	return tset->tc_array ? tset->tc_array_len : 1;
}
EXPORT_SYMBOL_GPL(cgroup_taskset_size);


B
Ben Blum 已提交
1904 1905 1906
/*
 * cgroup_task_migrate - move a task from one cgroup to another.
 *
1907
 * Must be called with cgroup_mutex and threadgroup locked.
B
Ben Blum 已提交
1908
 */
1909 1910 1911
static void cgroup_task_migrate(struct cgroup *old_cgrp,
				struct task_struct *tsk,
				struct css_set *new_cset)
B
Ben Blum 已提交
1912
{
1913
	struct css_set *old_cset;
B
Ben Blum 已提交
1914 1915

	/*
1916 1917 1918
	 * We are synchronized through threadgroup_lock() against PF_EXITING
	 * setting such that we can't race against cgroup_exit() changing the
	 * css_set to init_css_set and dropping the old one.
B
Ben Blum 已提交
1919
	 */
1920
	WARN_ON_ONCE(tsk->flags & PF_EXITING);
1921
	old_cset = task_css_set(tsk);
B
Ben Blum 已提交
1922 1923

	task_lock(tsk);
1924
	rcu_assign_pointer(tsk->cgroups, new_cset);
B
Ben Blum 已提交
1925 1926 1927 1928 1929
	task_unlock(tsk);

	/* Update the css_set linked lists if we're using them */
	write_lock(&css_set_lock);
	if (!list_empty(&tsk->cg_list))
1930
		list_move(&tsk->cg_list, &new_cset->tasks);
B
Ben Blum 已提交
1931 1932 1933
	write_unlock(&css_set_lock);

	/*
1934 1935 1936
	 * We just gained a reference on old_cset by taking it from the
	 * task. As trading it for new_cset is protected by cgroup_mutex,
	 * we're safe to drop it here; it will be freed under RCU.
B
Ben Blum 已提交
1937
	 */
1938 1939
	set_bit(CGRP_RELEASABLE, &old_cgrp->flags);
	put_css_set(old_cset);
B
Ben Blum 已提交
1940 1941
}

L
Li Zefan 已提交
1942
/**
1943
 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
B
Ben Blum 已提交
1944
 * @cgrp: the cgroup to attach to
1945 1946
 * @tsk: the task or the leader of the threadgroup to be attached
 * @threadgroup: attach the whole threadgroup?
B
Ben Blum 已提交
1947
 *
1948
 * Call holding cgroup_mutex and the group_rwsem of the leader. Will take
1949
 * task_lock of @tsk or each thread in the threadgroup individually in turn.
B
Ben Blum 已提交
1950
 */
T
Tejun Heo 已提交
1951 1952
static int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk,
			      bool threadgroup)
B
Ben Blum 已提交
1953 1954 1955
{
	int retval, i, group_size;
	struct cgroupfs_root *root = cgrp->root;
T
Tejun Heo 已提交
1956
	struct cgroup_subsys_state *css, *failed_css = NULL;
B
Ben Blum 已提交
1957
	/* threadgroup list cursor and array */
1958
	struct task_struct *leader = tsk;
1959
	struct task_and_cgroup *tc;
1960
	struct flex_array *group;
1961
	struct cgroup_taskset tset = { };
B
Ben Blum 已提交
1962 1963 1964 1965 1966

	/*
	 * step 0: in order to do expensive, possibly blocking operations for
	 * every thread, we cannot iterate the thread group list, since it needs
	 * rcu or tasklist locked. instead, build an array of all threads in the
1967 1968
	 * group - group_rwsem prevents new threads from appearing, and if
	 * threads exit, this will just be an over-estimate.
B
Ben Blum 已提交
1969
	 */
1970 1971 1972 1973
	if (threadgroup)
		group_size = get_nr_threads(tsk);
	else
		group_size = 1;
1974
	/* flex_array supports very large thread-groups better than kmalloc. */
1975
	group = flex_array_alloc(sizeof(*tc), group_size, GFP_KERNEL);
B
Ben Blum 已提交
1976 1977
	if (!group)
		return -ENOMEM;
1978
	/* pre-allocate to guarantee space while iterating in rcu read-side. */
1979
	retval = flex_array_prealloc(group, 0, group_size, GFP_KERNEL);
1980 1981
	if (retval)
		goto out_free_group_list;
B
Ben Blum 已提交
1982 1983

	i = 0;
1984 1985 1986 1987 1988 1989
	/*
	 * Prevent freeing of tasks while we take a snapshot. Tasks that are
	 * already PF_EXITING could be freed from underneath us unless we
	 * take an rcu_read_lock.
	 */
	rcu_read_lock();
B
Ben Blum 已提交
1990
	do {
1991 1992
		struct task_and_cgroup ent;

1993 1994
		/* @tsk either already exited or can't exit until the end */
		if (tsk->flags & PF_EXITING)
1995
			goto next;
1996

B
Ben Blum 已提交
1997 1998
		/* as per above, nr_threads may decrease, but not increase. */
		BUG_ON(i >= group_size);
1999 2000
		ent.task = tsk;
		ent.cgrp = task_cgroup_from_root(tsk, root);
2001 2002
		/* nothing to do if this task is already in the cgroup */
		if (ent.cgrp == cgrp)
2003
			goto next;
2004 2005 2006 2007
		/*
		 * saying GFP_ATOMIC has no effect here because we did prealloc
		 * earlier, but it's good form to communicate our expectations.
		 */
2008
		retval = flex_array_put(group, i, &ent, GFP_ATOMIC);
2009
		BUG_ON(retval != 0);
B
Ben Blum 已提交
2010
		i++;
2011
	next:
2012 2013
		if (!threadgroup)
			break;
B
Ben Blum 已提交
2014
	} while_each_thread(leader, tsk);
2015
	rcu_read_unlock();
B
Ben Blum 已提交
2016 2017
	/* remember the number of threads in the array for later. */
	group_size = i;
2018 2019
	tset.tc_array = group;
	tset.tc_array_len = group_size;
B
Ben Blum 已提交
2020

2021 2022
	/* methods shouldn't be called if no task is actually migrating */
	retval = 0;
2023
	if (!group_size)
2024
		goto out_free_group_list;
2025

B
Ben Blum 已提交
2026 2027 2028
	/*
	 * step 1: check that we can legitimately attach to the cgroup.
	 */
T
Tejun Heo 已提交
2029 2030 2031
	for_each_css(css, i, cgrp) {
		if (css->ss->can_attach) {
			retval = css->ss->can_attach(css, &tset);
B
Ben Blum 已提交
2032
			if (retval) {
T
Tejun Heo 已提交
2033
				failed_css = css;
B
Ben Blum 已提交
2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
				goto out_cancel_attach;
			}
		}
	}

	/*
	 * step 2: make sure css_sets exist for all threads to be migrated.
	 * we use find_css_set, which allocates a new one if necessary.
	 */
	for (i = 0; i < group_size; i++) {
2044 2045
		struct css_set *old_cset;

2046
		tc = flex_array_get(group, i);
2047
		old_cset = task_css_set(tc->task);
L
Li Zefan 已提交
2048 2049
		tc->cset = find_css_set(old_cset, cgrp);
		if (!tc->cset) {
2050 2051
			retval = -ENOMEM;
			goto out_put_css_set_refs;
B
Ben Blum 已提交
2052 2053 2054 2055
		}
	}

	/*
2056 2057 2058
	 * step 3: now that we're guaranteed success wrt the css_sets,
	 * proceed to move all tasks to the new cgroup.  There are no
	 * failure cases after here, so this is the commit point.
B
Ben Blum 已提交
2059 2060
	 */
	for (i = 0; i < group_size; i++) {
2061
		tc = flex_array_get(group, i);
L
Li Zefan 已提交
2062
		cgroup_task_migrate(tc->cgrp, tc->task, tc->cset);
B
Ben Blum 已提交
2063 2064 2065 2066
	}
	/* nothing is sensitive to fork() after this point. */

	/*
2067
	 * step 4: do subsystem attach callbacks.
B
Ben Blum 已提交
2068
	 */
T
Tejun Heo 已提交
2069 2070 2071
	for_each_css(css, i, cgrp)
		if (css->ss->attach)
			css->ss->attach(css, &tset);
B
Ben Blum 已提交
2072 2073 2074 2075 2076

	/*
	 * step 5: success! and cleanup
	 */
	retval = 0;
2077 2078 2079 2080
out_put_css_set_refs:
	if (retval) {
		for (i = 0; i < group_size; i++) {
			tc = flex_array_get(group, i);
L
Li Zefan 已提交
2081
			if (!tc->cset)
2082
				break;
L
Li Zefan 已提交
2083
			put_css_set(tc->cset);
2084
		}
B
Ben Blum 已提交
2085 2086 2087
	}
out_cancel_attach:
	if (retval) {
T
Tejun Heo 已提交
2088 2089
		for_each_css(css, i, cgrp) {
			if (css == failed_css)
B
Ben Blum 已提交
2090
				break;
T
Tejun Heo 已提交
2091 2092
			if (css->ss->cancel_attach)
				css->ss->cancel_attach(css, &tset);
B
Ben Blum 已提交
2093 2094 2095
		}
	}
out_free_group_list:
2096
	flex_array_free(group);
B
Ben Blum 已提交
2097 2098 2099 2100 2101
	return retval;
}

/*
 * Find the task_struct of the task to attach by vpid and pass it along to the
2102 2103
 * function to attach either it or all tasks in its threadgroup. Will lock
 * cgroup_mutex and threadgroup; may take task_lock of task.
2104
 */
B
Ben Blum 已提交
2105
static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup)
2106 2107
{
	struct task_struct *tsk;
2108
	const struct cred *cred = current_cred(), *tcred;
2109 2110
	int ret;

B
Ben Blum 已提交
2111 2112 2113
	if (!cgroup_lock_live_group(cgrp))
		return -ENODEV;

2114 2115
retry_find_task:
	rcu_read_lock();
2116
	if (pid) {
2117
		tsk = find_task_by_vpid(pid);
B
Ben Blum 已提交
2118 2119
		if (!tsk) {
			rcu_read_unlock();
2120 2121
			ret= -ESRCH;
			goto out_unlock_cgroup;
2122
		}
B
Ben Blum 已提交
2123 2124 2125 2126
		/*
		 * even if we're attaching all tasks in the thread group, we
		 * only need to check permissions on one of them.
		 */
2127
		tcred = __task_cred(tsk);
2128 2129 2130
		if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
		    !uid_eq(cred->euid, tcred->uid) &&
		    !uid_eq(cred->euid, tcred->suid)) {
2131
			rcu_read_unlock();
2132 2133
			ret = -EACCES;
			goto out_unlock_cgroup;
2134
		}
2135 2136
	} else
		tsk = current;
2137 2138

	if (threadgroup)
2139
		tsk = tsk->group_leader;
2140 2141

	/*
2142
	 * Workqueue threads may acquire PF_NO_SETAFFINITY and become
2143 2144 2145
	 * trapped in a cpuset, or RT worker may be born in a cgroup
	 * with no rt_runtime allocated.  Just say no.
	 */
2146
	if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
2147 2148 2149 2150 2151
		ret = -EINVAL;
		rcu_read_unlock();
		goto out_unlock_cgroup;
	}

2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168
	get_task_struct(tsk);
	rcu_read_unlock();

	threadgroup_lock(tsk);
	if (threadgroup) {
		if (!thread_group_leader(tsk)) {
			/*
			 * a race with de_thread from another thread's exec()
			 * may strip us of our leadership, if this happens,
			 * there is no choice but to throw this task away and
			 * try again; this is
			 * "double-double-toil-and-trouble-check locking".
			 */
			threadgroup_unlock(tsk);
			put_task_struct(tsk);
			goto retry_find_task;
		}
2169 2170 2171 2172
	}

	ret = cgroup_attach_task(cgrp, tsk, threadgroup);

2173 2174
	threadgroup_unlock(tsk);

2175
	put_task_struct(tsk);
2176
out_unlock_cgroup:
T
Tejun Heo 已提交
2177
	mutex_unlock(&cgroup_mutex);
2178 2179 2180
	return ret;
}

2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
/**
 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
 * @from: attach to all cgroups of a given task
 * @tsk: the task to be attached
 */
int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
{
	struct cgroupfs_root *root;
	int retval = 0;

T
Tejun Heo 已提交
2191
	mutex_lock(&cgroup_mutex);
2192
	for_each_active_root(root) {
L
Li Zefan 已提交
2193
		struct cgroup *from_cgrp = task_cgroup_from_root(from, root);
2194

L
Li Zefan 已提交
2195
		retval = cgroup_attach_task(from_cgrp, tsk, false);
2196 2197 2198
		if (retval)
			break;
	}
T
Tejun Heo 已提交
2199
	mutex_unlock(&cgroup_mutex);
2200 2201 2202 2203 2204

	return retval;
}
EXPORT_SYMBOL_GPL(cgroup_attach_task_all);

2205 2206
static int cgroup_tasks_write(struct cgroup_subsys_state *css,
			      struct cftype *cft, u64 pid)
B
Ben Blum 已提交
2207
{
2208
	return attach_task_by_pid(css->cgroup, pid, false);
B
Ben Blum 已提交
2209 2210
}

2211 2212
static int cgroup_procs_write(struct cgroup_subsys_state *css,
			      struct cftype *cft, u64 tgid)
2213
{
2214
	return attach_task_by_pid(css->cgroup, tgid, true);
2215 2216
}

2217 2218
static int cgroup_release_agent_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, const char *buffer)
2219
{
2220
	BUILD_BUG_ON(sizeof(css->cgroup->root->release_agent_path) < PATH_MAX);
2221 2222
	if (strlen(buffer) >= PATH_MAX)
		return -EINVAL;
2223
	if (!cgroup_lock_live_group(css->cgroup))
2224
		return -ENODEV;
T
Tejun Heo 已提交
2225
	mutex_lock(&cgroup_root_mutex);
2226
	strcpy(css->cgroup->root->release_agent_path, buffer);
T
Tejun Heo 已提交
2227
	mutex_unlock(&cgroup_root_mutex);
T
Tejun Heo 已提交
2228
	mutex_unlock(&cgroup_mutex);
2229 2230 2231
	return 0;
}

2232
static int cgroup_release_agent_show(struct seq_file *seq, void *v)
2233
{
2234
	struct cgroup *cgrp = seq_css(seq)->cgroup;
2235

2236 2237 2238 2239
	if (!cgroup_lock_live_group(cgrp))
		return -ENODEV;
	seq_puts(seq, cgrp->root->release_agent_path);
	seq_putc(seq, '\n');
T
Tejun Heo 已提交
2240
	mutex_unlock(&cgroup_mutex);
2241 2242 2243
	return 0;
}

2244
static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
2245
{
2246 2247 2248
	struct cgroup *cgrp = seq_css(seq)->cgroup;

	seq_printf(seq, "%d\n", cgroup_sane_behavior(cgrp));
2249 2250 2251
	return 0;
}

2252 2253 2254
/* A buffer size big enough for numbers or short strings */
#define CGROUP_LOCAL_BUFFER_SIZE 64

2255 2256
static ssize_t cgroup_file_write(struct file *file, const char __user *userbuf,
				 size_t nbytes, loff_t *ppos)
2257
{
2258 2259 2260 2261 2262 2263
	struct cfent *cfe = __d_cfe(file->f_dentry);
	struct cftype *cft = __d_cft(file->f_dentry);
	struct cgroup_subsys_state *css = cfe->css;
	size_t max_bytes = cft->max_write_len ?: CGROUP_LOCAL_BUFFER_SIZE - 1;
	char *buf;
	int ret;
2264 2265 2266 2267

	if (nbytes >= max_bytes)
		return -E2BIG;

2268 2269 2270
	buf = kmalloc(nbytes + 1, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
2271

2272 2273 2274 2275
	if (copy_from_user(buf, userbuf, nbytes)) {
		ret = -EFAULT;
		goto out_free;
	}
2276

2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294
	buf[nbytes] = '\0';

	if (cft->write_string) {
		ret = cft->write_string(css, cft, strstrip(buf));
	} else if (cft->write_u64) {
		unsigned long long v;
		ret = kstrtoull(buf, 0, &v);
		if (!ret)
			ret = cft->write_u64(css, cft, v);
	} else if (cft->write_s64) {
		long long v;
		ret = kstrtoll(buf, 0, &v);
		if (!ret)
			ret = cft->write_s64(css, cft, v);
	} else if (cft->trigger) {
		ret = cft->trigger(css, (unsigned int)cft->private);
	} else {
		ret = -EINVAL;
2295
	}
2296 2297 2298
out_free:
	kfree(buf);
	return ret ?: nbytes;
2299 2300
}

2301 2302 2303 2304 2305
/*
 * seqfile ops/methods for returning structured data. Currently just
 * supports string->u64 maps, but can be extended in future.
 */

2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
{
	struct cftype *cft = seq_cft(seq);

	if (cft->seq_start) {
		return cft->seq_start(seq, ppos);
	} else {
		/*
		 * The same behavior and code as single_open().  Returns
		 * !NULL if pos is at the beginning; otherwise, NULL.
		 */
		return NULL + !*ppos;
	}
}

static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
{
	struct cftype *cft = seq_cft(seq);

	if (cft->seq_next) {
		return cft->seq_next(seq, v, ppos);
	} else {
		/*
		 * The same behavior and code as single_open(), always
		 * terminate after the initial read.
		 */
		++*ppos;
		return NULL;
	}
}

static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
{
	struct cftype *cft = seq_cft(seq);

	if (cft->seq_stop)
		cft->seq_stop(seq, v);
}

2345 2346
static int cgroup_seqfile_show(struct seq_file *m, void *arg)
{
2347 2348
	struct cftype *cft = seq_cft(m);
	struct cgroup_subsys_state *css = seq_css(m);
2349

2350 2351
	if (cft->seq_show)
		return cft->seq_show(m, arg);
2352

2353 2354 2355 2356 2357 2358 2359 2360
	if (cft->read_u64)
		seq_printf(m, "%llu\n", cft->read_u64(css, cft));
	else if (cft->read_s64)
		seq_printf(m, "%lld\n", cft->read_s64(css, cft));
	else
		return -EINVAL;
	return 0;
}
2361

2362 2363 2364 2365 2366 2367 2368
static struct seq_operations cgroup_seq_operations = {
	.start		= cgroup_seqfile_start,
	.next		= cgroup_seqfile_next,
	.stop		= cgroup_seqfile_stop,
	.show		= cgroup_seqfile_show,
};

2369 2370
static int cgroup_file_open(struct inode *inode, struct file *file)
{
2371 2372
	struct cfent *cfe = __d_cfe(file->f_dentry);
	struct cftype *cft = __d_cft(file->f_dentry);
2373 2374
	struct cgroup *cgrp = __d_cgrp(cfe->dentry->d_parent);
	struct cgroup_subsys_state *css;
2375
	struct cgroup_open_file *of;
2376 2377 2378 2379 2380
	int err;

	err = generic_file_open(inode, file);
	if (err)
		return err;
2381

2382 2383 2384 2385 2386
	/*
	 * If the file belongs to a subsystem, pin the css.  Will be
	 * unpinned either on open failure or release.  This ensures that
	 * @css stays alive for all file operations.
	 */
2387
	rcu_read_lock();
2388 2389 2390
	css = cgroup_css(cgrp, cft->ss);
	if (cft->ss && !css_tryget(css))
		css = NULL;
2391
	rcu_read_unlock();
2392

2393
	if (!css)
2394
		return -ENODEV;
2395

2396 2397 2398 2399 2400 2401 2402 2403
	/*
	 * @cfe->css is used by read/write/close to determine the
	 * associated css.  @file->private_data would be a better place but
	 * that's already used by seqfile.  Multiple accessors may use it
	 * simultaneously which is okay as the association never changes.
	 */
	WARN_ON_ONCE(cfe->css && cfe->css != css);
	cfe->css = css;
2404

2405 2406 2407 2408 2409
	of = __seq_open_private(file, &cgroup_seq_operations,
				sizeof(struct cgroup_open_file));
	if (of) {
		of->cfe = cfe;
		return 0;
2410
	}
2411

2412
	if (css->ss)
2413
		css_put(css);
2414
	return -ENOMEM;
2415 2416 2417 2418
}

static int cgroup_file_release(struct inode *inode, struct file *file)
{
2419
	struct cfent *cfe = __d_cfe(file->f_dentry);
2420
	struct cgroup_subsys_state *css = cfe->css;
2421

T
Tejun Heo 已提交
2422
	if (css->ss)
2423
		css_put(css);
2424
	return seq_release_private(inode, file);
2425 2426 2427 2428 2429 2430 2431 2432
}

/*
 * cgroup_rename - Only allow simple rename of directories in place.
 */
static int cgroup_rename(struct inode *old_dir, struct dentry *old_dentry,
			    struct inode *new_dir, struct dentry *new_dentry)
{
2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
	int ret;
	struct cgroup_name *name, *old_name;
	struct cgroup *cgrp;

	/*
	 * It's convinient to use parent dir's i_mutex to protected
	 * cgrp->name.
	 */
	lockdep_assert_held(&old_dir->i_mutex);

2443 2444 2445 2446 2447 2448
	if (!S_ISDIR(old_dentry->d_inode->i_mode))
		return -ENOTDIR;
	if (new_dentry->d_inode)
		return -EEXIST;
	if (old_dir != new_dir)
		return -EIO;
2449 2450 2451

	cgrp = __d_cgrp(old_dentry);

2452 2453 2454 2455 2456 2457 2458
	/*
	 * This isn't a proper migration and its usefulness is very
	 * limited.  Disallow if sane_behavior.
	 */
	if (cgroup_sane_behavior(cgrp))
		return -EPERM;

2459 2460 2461 2462 2463 2464 2465 2466 2467 2468
	name = cgroup_alloc_name(new_dentry);
	if (!name)
		return -ENOMEM;

	ret = simple_rename(old_dir, old_dentry, new_dir, new_dentry);
	if (ret) {
		kfree(name);
		return ret;
	}

2469
	old_name = rcu_dereference_protected(cgrp->name, true);
2470 2471 2472 2473
	rcu_assign_pointer(cgrp->name, name);

	kfree_rcu(old_name, rcu_head);
	return 0;
2474 2475
}

A
Aristeu Rozanski 已提交
2476 2477 2478 2479 2480
static struct simple_xattrs *__d_xattrs(struct dentry *dentry)
{
	if (S_ISDIR(dentry->d_inode->i_mode))
		return &__d_cgrp(dentry)->xattrs;
	else
L
Li Zefan 已提交
2481
		return &__d_cfe(dentry)->xattrs;
A
Aristeu Rozanski 已提交
2482 2483 2484 2485 2486
}

static inline int xattr_enabled(struct dentry *dentry)
{
	struct cgroupfs_root *root = dentry->d_sb->s_fs_info;
2487
	return root->flags & CGRP_ROOT_XATTR;
A
Aristeu Rozanski 已提交
2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
}

static bool is_valid_xattr(const char *name)
{
	if (!strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) ||
	    !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN))
		return true;
	return false;
}

static int cgroup_setxattr(struct dentry *dentry, const char *name,
			   const void *val, size_t size, int flags)
{
	if (!xattr_enabled(dentry))
		return -EOPNOTSUPP;
	if (!is_valid_xattr(name))
		return -EINVAL;
	return simple_xattr_set(__d_xattrs(dentry), name, val, size, flags);
}

static int cgroup_removexattr(struct dentry *dentry, const char *name)
{
	if (!xattr_enabled(dentry))
		return -EOPNOTSUPP;
	if (!is_valid_xattr(name))
		return -EINVAL;
	return simple_xattr_remove(__d_xattrs(dentry), name);
}

static ssize_t cgroup_getxattr(struct dentry *dentry, const char *name,
			       void *buf, size_t size)
{
	if (!xattr_enabled(dentry))
		return -EOPNOTSUPP;
	if (!is_valid_xattr(name))
		return -EINVAL;
	return simple_xattr_get(__d_xattrs(dentry), name, buf, size);
}

static ssize_t cgroup_listxattr(struct dentry *dentry, char *buf, size_t size)
{
	if (!xattr_enabled(dentry))
		return -EOPNOTSUPP;
	return simple_xattr_list(__d_xattrs(dentry), buf, size);
}

2534
static const struct file_operations cgroup_file_operations = {
2535
	.read = seq_read,
2536 2537 2538 2539 2540 2541
	.write = cgroup_file_write,
	.llseek = generic_file_llseek,
	.open = cgroup_file_open,
	.release = cgroup_file_release,
};

A
Aristeu Rozanski 已提交
2542 2543 2544 2545 2546 2547 2548
static const struct inode_operations cgroup_file_inode_operations = {
	.setxattr = cgroup_setxattr,
	.getxattr = cgroup_getxattr,
	.listxattr = cgroup_listxattr,
	.removexattr = cgroup_removexattr,
};

2549
static const struct inode_operations cgroup_dir_inode_operations = {
A
Al Viro 已提交
2550
	.lookup = simple_lookup,
2551 2552 2553
	.mkdir = cgroup_mkdir,
	.rmdir = cgroup_rmdir,
	.rename = cgroup_rename,
A
Aristeu Rozanski 已提交
2554 2555 2556 2557
	.setxattr = cgroup_setxattr,
	.getxattr = cgroup_getxattr,
	.listxattr = cgroup_listxattr,
	.removexattr = cgroup_removexattr,
2558 2559
};

A
Al Viro 已提交
2560
static int cgroup_create_file(struct dentry *dentry, umode_t mode,
2561 2562
				struct super_block *sb)
{
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
	struct inode *inode;

	if (!dentry)
		return -ENOENT;
	if (dentry->d_inode)
		return -EEXIST;

	inode = cgroup_new_inode(mode, sb);
	if (!inode)
		return -ENOMEM;

	if (S_ISDIR(mode)) {
		inode->i_op = &cgroup_dir_inode_operations;
		inode->i_fop = &simple_dir_operations;

		/* start off with i_nlink == 2 (for "." entry) */
		inc_nlink(inode);
T
Tejun Heo 已提交
2580
		inc_nlink(dentry->d_parent->d_inode);
2581

2582 2583 2584 2585 2586 2587 2588 2589 2590
		/*
		 * Control reaches here with cgroup_mutex held.
		 * @inode->i_mutex should nest outside cgroup_mutex but we
		 * want to populate it immediately without releasing
		 * cgroup_mutex.  As @inode isn't visible to anyone else
		 * yet, trylock will always succeed without affecting
		 * lockdep checks.
		 */
		WARN_ON_ONCE(!mutex_trylock(&inode->i_mutex));
2591 2592 2593
	} else if (S_ISREG(mode)) {
		inode->i_size = 0;
		inode->i_fop = &cgroup_file_operations;
A
Aristeu Rozanski 已提交
2594
		inode->i_op = &cgroup_file_inode_operations;
2595 2596 2597 2598 2599 2600
	}
	d_instantiate(dentry, inode);
	dget(dentry);	/* Extra count - pin the dentry in core */
	return 0;
}

L
Li Zefan 已提交
2601 2602 2603 2604 2605 2606 2607 2608 2609
/**
 * cgroup_file_mode - deduce file mode of a control file
 * @cft: the control file in question
 *
 * returns cft->mode if ->mode is not 0
 * returns S_IRUGO|S_IWUSR if it has both a read and a write handler
 * returns S_IRUGO if it has only a read handler
 * returns S_IWUSR if it has only a write hander
 */
A
Al Viro 已提交
2610
static umode_t cgroup_file_mode(const struct cftype *cft)
L
Li Zefan 已提交
2611
{
A
Al Viro 已提交
2612
	umode_t mode = 0;
L
Li Zefan 已提交
2613 2614 2615 2616

	if (cft->mode)
		return cft->mode;

2617
	if (cft->read_u64 || cft->read_s64 || cft->seq_show)
L
Li Zefan 已提交
2618 2619
		mode |= S_IRUGO;

2620 2621
	if (cft->write_u64 || cft->write_s64 || cft->write_string ||
	    cft->trigger)
L
Li Zefan 已提交
2622 2623 2624 2625 2626
		mode |= S_IWUSR;

	return mode;
}

2627
static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft)
2628
{
2629
	struct dentry *dir = cgrp->dentry;
T
Tejun Heo 已提交
2630
	struct cgroup *parent = __d_cgrp(dir);
2631
	struct dentry *dentry;
T
Tejun Heo 已提交
2632
	struct cfent *cfe;
2633
	int error;
A
Al Viro 已提交
2634
	umode_t mode;
2635
	char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 };
2636

T
Tejun Heo 已提交
2637 2638
	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {
2639
		strcpy(name, cft->ss->name);
2640 2641 2642
		strcat(name, ".");
	}
	strcat(name, cft->name);
T
Tejun Heo 已提交
2643

2644
	BUG_ON(!mutex_is_locked(&dir->d_inode->i_mutex));
T
Tejun Heo 已提交
2645 2646 2647 2648 2649

	cfe = kzalloc(sizeof(*cfe), GFP_KERNEL);
	if (!cfe)
		return -ENOMEM;

2650
	dentry = lookup_one_len(name, dir, strlen(name));
T
Tejun Heo 已提交
2651
	if (IS_ERR(dentry)) {
2652
		error = PTR_ERR(dentry);
T
Tejun Heo 已提交
2653 2654 2655
		goto out;
	}

2656 2657 2658 2659 2660
	cfe->type = (void *)cft;
	cfe->dentry = dentry;
	dentry->d_fsdata = cfe;
	simple_xattrs_init(&cfe->xattrs);

T
Tejun Heo 已提交
2661 2662 2663 2664 2665 2666 2667 2668 2669
	mode = cgroup_file_mode(cft);
	error = cgroup_create_file(dentry, mode | S_IFREG, cgrp->root->sb);
	if (!error) {
		list_add_tail(&cfe->node, &parent->files);
		cfe = NULL;
	}
	dput(dentry);
out:
	kfree(cfe);
2670 2671 2672
	return error;
}

2673 2674 2675 2676 2677 2678 2679
/**
 * cgroup_addrm_files - add or remove files to a cgroup directory
 * @cgrp: the target cgroup
 * @cfts: array of cftypes to be added
 * @is_add: whether to add or remove
 *
 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
2680 2681 2682
 * For removals, this function never fails.  If addition fails, this
 * function doesn't remove files already added.  The caller is responsible
 * for cleaning up.
2683
 */
2684 2685
static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[],
			      bool is_add)
2686
{
A
Aristeu Rozanski 已提交
2687
	struct cftype *cft;
2688 2689 2690 2691
	int ret;

	lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
	lockdep_assert_held(&cgroup_mutex);
T
Tejun Heo 已提交
2692 2693

	for (cft = cfts; cft->name[0] != '\0'; cft++) {
2694
		/* does cft->flags tell us to skip this file on @cgrp? */
2695 2696
		if ((cft->flags & CFTYPE_INSANE) && cgroup_sane_behavior(cgrp))
			continue;
2697 2698 2699 2700 2701
		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
			continue;
		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
			continue;

2702
		if (is_add) {
2703
			ret = cgroup_add_file(cgrp, cft);
2704
			if (ret) {
2705
				pr_warn("cgroup_addrm_files: failed to add %s, err=%d\n",
2706 2707 2708
					cft->name, ret);
				return ret;
			}
2709 2710
		} else {
			cgroup_rm_file(cgrp, cft);
T
Tejun Heo 已提交
2711
		}
2712
	}
2713
	return 0;
2714 2715
}

2716
static void cgroup_cfts_prepare(void)
2717
	__acquires(&cgroup_mutex)
2718 2719 2720 2721
{
	/*
	 * Thanks to the entanglement with vfs inode locking, we can't walk
	 * the existing cgroups under cgroup_mutex and create files.
2722 2723
	 * Instead, we use css_for_each_descendant_pre() and drop RCU read
	 * lock before calling cgroup_addrm_files().
2724 2725 2726 2727
	 */
	mutex_lock(&cgroup_mutex);
}

2728
static int cgroup_cfts_commit(struct cftype *cfts, bool is_add)
2729
	__releases(&cgroup_mutex)
2730 2731
{
	LIST_HEAD(pending);
2732
	struct cgroup_subsys *ss = cfts[0].ss;
2733
	struct cgroup *root = &ss->root->top_cgroup;
2734
	struct super_block *sb = ss->root->sb;
2735 2736
	struct dentry *prev = NULL;
	struct inode *inode;
2737
	struct cgroup_subsys_state *css;
2738
	u64 update_before;
2739
	int ret = 0;
2740 2741

	/* %NULL @cfts indicates abort and don't bother if @ss isn't attached */
2742
	if (!cfts || ss->root == &cgroup_dummy_root ||
2743 2744
	    !atomic_inc_not_zero(&sb->s_active)) {
		mutex_unlock(&cgroup_mutex);
2745
		return 0;
2746 2747 2748
	}

	/*
2749 2750
	 * All cgroups which are created after we drop cgroup_mutex will
	 * have the updated set of files, so we only need to update the
2751
	 * cgroups created before the current @cgroup_serial_nr_next.
2752
	 */
2753
	update_before = cgroup_serial_nr_next;
2754 2755 2756 2757 2758

	mutex_unlock(&cgroup_mutex);

	/* add/rm files for all cgroups created before */
	rcu_read_lock();
2759
	css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
2760 2761
		struct cgroup *cgrp = css->cgroup;

2762 2763 2764 2765 2766 2767 2768 2769 2770
		if (cgroup_is_dead(cgrp))
			continue;

		inode = cgrp->dentry->d_inode;
		dget(cgrp->dentry);
		rcu_read_unlock();

		dput(prev);
		prev = cgrp->dentry;
2771 2772 2773

		mutex_lock(&inode->i_mutex);
		mutex_lock(&cgroup_mutex);
2774
		if (cgrp->serial_nr < update_before && !cgroup_is_dead(cgrp))
2775
			ret = cgroup_addrm_files(cgrp, cfts, is_add);
2776 2777 2778
		mutex_unlock(&cgroup_mutex);
		mutex_unlock(&inode->i_mutex);

2779
		rcu_read_lock();
2780 2781
		if (ret)
			break;
2782
	}
2783 2784 2785
	rcu_read_unlock();
	dput(prev);
	deactivate_super(sb);
2786
	return ret;
2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802
}

/**
 * cgroup_add_cftypes - add an array of cftypes to a subsystem
 * @ss: target cgroup subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
 * Register @cfts to @ss.  Files described by @cfts are created for all
 * existing cgroups to which @ss is attached and all future cgroups will
 * have them too.  This function can be called anytime whether @ss is
 * attached or not.
 *
 * Returns 0 on successful registration, -errno on failure.  Note that this
 * function currently returns 0 as long as @cfts registration is successful
 * even if some file creation attempts on existing cgroups fail.
 */
A
Aristeu Rozanski 已提交
2803
int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
2804 2805
{
	struct cftype_set *set;
2806
	struct cftype *cft;
2807
	int ret;
2808 2809 2810 2811 2812

	set = kzalloc(sizeof(*set), GFP_KERNEL);
	if (!set)
		return -ENOMEM;

2813 2814 2815
	for (cft = cfts; cft->name[0] != '\0'; cft++)
		cft->ss = ss;

2816 2817 2818
	cgroup_cfts_prepare();
	set->cfts = cfts;
	list_add_tail(&set->node, &ss->cftsets);
2819
	ret = cgroup_cfts_commit(cfts, true);
2820
	if (ret)
2821
		cgroup_rm_cftypes(cfts);
2822
	return ret;
2823 2824 2825
}
EXPORT_SYMBOL_GPL(cgroup_add_cftypes);

2826 2827 2828 2829
/**
 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
 * @cfts: zero-length name terminated array of cftypes
 *
2830 2831 2832
 * Unregister @cfts.  Files described by @cfts are removed from all
 * existing cgroups and all future cgroups won't have them either.  This
 * function can be called anytime whether @cfts' subsys is attached or not.
2833 2834
 *
 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
2835
 * registered.
2836
 */
2837
int cgroup_rm_cftypes(struct cftype *cfts)
2838 2839 2840
{
	struct cftype_set *set;

2841 2842 2843
	if (!cfts || !cfts[0].ss)
		return -ENOENT;

2844 2845
	cgroup_cfts_prepare();

2846
	list_for_each_entry(set, &cfts[0].ss->cftsets, node) {
2847
		if (set->cfts == cfts) {
2848 2849
			list_del(&set->node);
			kfree(set);
2850
			cgroup_cfts_commit(cfts, false);
2851 2852 2853 2854
			return 0;
		}
	}

2855
	cgroup_cfts_commit(NULL, false);
2856 2857 2858
	return -ENOENT;
}

L
Li Zefan 已提交
2859 2860 2861 2862 2863 2864
/**
 * cgroup_task_count - count the number of tasks in a cgroup.
 * @cgrp: the cgroup in question
 *
 * Return the number of tasks in the cgroup.
 */
2865
int cgroup_task_count(const struct cgroup *cgrp)
2866 2867
{
	int count = 0;
2868
	struct cgrp_cset_link *link;
2869 2870

	read_lock(&css_set_lock);
2871 2872
	list_for_each_entry(link, &cgrp->cset_links, cset_link)
		count += atomic_read(&link->cset->refcount);
2873
	read_unlock(&css_set_lock);
2874 2875 2876
	return count;
}

2877
/*
2878 2879 2880
 * To reduce the fork() overhead for systems that are not actually using
 * their cgroups capability, we don't maintain the lists running through
 * each css_set to its tasks until we see the list actually used - in other
2881
 * words after the first call to css_task_iter_start().
2882
 */
2883
static void cgroup_enable_task_cg_lists(void)
2884 2885 2886 2887
{
	struct task_struct *p, *g;
	write_lock(&css_set_lock);
	use_task_css_set_links = 1;
2888 2889 2890 2891 2892 2893 2894 2895
	/*
	 * We need tasklist_lock because RCU is not safe against
	 * while_each_thread(). Besides, a forking task that has passed
	 * cgroup_post_fork() without seeing use_task_css_set_links = 1
	 * is not guaranteed to have its child immediately visible in the
	 * tasklist if we walk through it with RCU.
	 */
	read_lock(&tasklist_lock);
2896 2897
	do_each_thread(g, p) {
		task_lock(p);
2898 2899 2900 2901 2902 2903
		/*
		 * We should check if the process is exiting, otherwise
		 * it will race with cgroup_exit() in that the list
		 * entry won't be deleted though the process has exited.
		 */
		if (!(p->flags & PF_EXITING) && list_empty(&p->cg_list))
2904
			list_add(&p->cg_list, &task_css_set(p)->tasks);
2905 2906
		task_unlock(p);
	} while_each_thread(g, p);
2907
	read_unlock(&tasklist_lock);
2908 2909 2910
	write_unlock(&css_set_lock);
}

2911
/**
2912 2913 2914
 * css_next_child - find the next child of a given css
 * @pos_css: the current position (%NULL to initiate traversal)
 * @parent_css: css whose children to walk
2915
 *
2916
 * This function returns the next child of @parent_css and should be called
2917 2918 2919
 * under either cgroup_mutex or RCU read lock.  The only requirement is
 * that @parent_css and @pos_css are accessible.  The next sibling is
 * guaranteed to be returned regardless of their states.
2920
 */
2921 2922 2923
struct cgroup_subsys_state *
css_next_child(struct cgroup_subsys_state *pos_css,
	       struct cgroup_subsys_state *parent_css)
2924
{
2925 2926
	struct cgroup *pos = pos_css ? pos_css->cgroup : NULL;
	struct cgroup *cgrp = parent_css->cgroup;
2927 2928
	struct cgroup *next;

2929
	cgroup_assert_mutex_or_rcu_locked();
2930 2931 2932 2933

	/*
	 * @pos could already have been removed.  Once a cgroup is removed,
	 * its ->sibling.next is no longer updated when its next sibling
2934 2935 2936 2937 2938 2939 2940
	 * changes.  As CGRP_DEAD assertion is serialized and happens
	 * before the cgroup is taken off the ->sibling list, if we see it
	 * unasserted, it's guaranteed that the next sibling hasn't
	 * finished its grace period even if it's already removed, and thus
	 * safe to dereference from this RCU critical section.  If
	 * ->sibling.next is inaccessible, cgroup_is_dead() is guaranteed
	 * to be visible as %true here.
2941 2942 2943 2944 2945 2946 2947 2948
	 *
	 * If @pos is dead, its next pointer can't be dereferenced;
	 * however, as each cgroup is given a monotonically increasing
	 * unique serial number and always appended to the sibling list,
	 * the next one can be found by walking the parent's children until
	 * we see a cgroup with higher serial number than @pos's.  While
	 * this path can be slower, it's taken only when either the current
	 * cgroup is removed or iteration and removal race.
2949
	 */
2950 2951 2952
	if (!pos) {
		next = list_entry_rcu(cgrp->children.next, struct cgroup, sibling);
	} else if (likely(!cgroup_is_dead(pos))) {
2953
		next = list_entry_rcu(pos->sibling.next, struct cgroup, sibling);
2954 2955 2956 2957
	} else {
		list_for_each_entry_rcu(next, &cgrp->children, sibling)
			if (next->serial_nr > pos->serial_nr)
				break;
2958 2959
	}

2960 2961 2962
	if (&next->sibling == &cgrp->children)
		return NULL;

2963
	return cgroup_css(next, parent_css->ss);
2964
}
2965
EXPORT_SYMBOL_GPL(css_next_child);
2966

2967
/**
2968
 * css_next_descendant_pre - find the next descendant for pre-order walk
2969
 * @pos: the current position (%NULL to initiate traversal)
2970
 * @root: css whose descendants to walk
2971
 *
2972
 * To be used by css_for_each_descendant_pre().  Find the next descendant
2973 2974
 * to visit for pre-order traversal of @root's descendants.  @root is
 * included in the iteration and the first node to be visited.
2975
 *
2976 2977 2978 2979
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @root are accessible and @pos is a descendant of @root.
2980
 */
2981 2982 2983
struct cgroup_subsys_state *
css_next_descendant_pre(struct cgroup_subsys_state *pos,
			struct cgroup_subsys_state *root)
2984
{
2985
	struct cgroup_subsys_state *next;
2986

2987
	cgroup_assert_mutex_or_rcu_locked();
2988

2989
	/* if first iteration, visit @root */
2990
	if (!pos)
2991
		return root;
2992 2993

	/* visit the first child if exists */
2994
	next = css_next_child(NULL, pos);
2995 2996 2997 2998
	if (next)
		return next;

	/* no child, visit my or the closest ancestor's next sibling */
2999 3000
	while (pos != root) {
		next = css_next_child(pos, css_parent(pos));
3001
		if (next)
3002
			return next;
3003
		pos = css_parent(pos);
3004
	}
3005 3006 3007

	return NULL;
}
3008
EXPORT_SYMBOL_GPL(css_next_descendant_pre);
3009

3010
/**
3011 3012
 * css_rightmost_descendant - return the rightmost descendant of a css
 * @pos: css of interest
3013
 *
3014 3015
 * Return the rightmost descendant of @pos.  If there's no descendant, @pos
 * is returned.  This can be used during pre-order traversal to skip
3016
 * subtree of @pos.
3017
 *
3018 3019 3020 3021
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct rightmost descendant as
 * long as @pos is accessible.
3022
 */
3023 3024
struct cgroup_subsys_state *
css_rightmost_descendant(struct cgroup_subsys_state *pos)
3025
{
3026
	struct cgroup_subsys_state *last, *tmp;
3027

3028
	cgroup_assert_mutex_or_rcu_locked();
3029 3030 3031 3032 3033

	do {
		last = pos;
		/* ->prev isn't RCU safe, walk ->next till the end */
		pos = NULL;
3034
		css_for_each_child(tmp, last)
3035 3036 3037 3038 3039
			pos = tmp;
	} while (pos);

	return last;
}
3040
EXPORT_SYMBOL_GPL(css_rightmost_descendant);
3041

3042 3043
static struct cgroup_subsys_state *
css_leftmost_descendant(struct cgroup_subsys_state *pos)
3044
{
3045
	struct cgroup_subsys_state *last;
3046 3047 3048

	do {
		last = pos;
3049
		pos = css_next_child(NULL, pos);
3050 3051 3052 3053 3054 3055
	} while (pos);

	return last;
}

/**
3056
 * css_next_descendant_post - find the next descendant for post-order walk
3057
 * @pos: the current position (%NULL to initiate traversal)
3058
 * @root: css whose descendants to walk
3059
 *
3060
 * To be used by css_for_each_descendant_post().  Find the next descendant
3061 3062
 * to visit for post-order traversal of @root's descendants.  @root is
 * included in the iteration and the last node to be visited.
3063
 *
3064 3065 3066 3067 3068
 * While this function requires cgroup_mutex or RCU read locking, it
 * doesn't require the whole traversal to be contained in a single critical
 * section.  This function will return the correct next descendant as long
 * as both @pos and @cgroup are accessible and @pos is a descendant of
 * @cgroup.
3069
 */
3070 3071 3072
struct cgroup_subsys_state *
css_next_descendant_post(struct cgroup_subsys_state *pos,
			 struct cgroup_subsys_state *root)
3073
{
3074
	struct cgroup_subsys_state *next;
3075

3076
	cgroup_assert_mutex_or_rcu_locked();
3077

3078 3079 3080
	/* if first iteration, visit leftmost descendant which may be @root */
	if (!pos)
		return css_leftmost_descendant(root);
3081

3082 3083 3084 3085
	/* if we visited @root, we're done */
	if (pos == root)
		return NULL;

3086
	/* if there's an unvisited sibling, visit its leftmost descendant */
3087
	next = css_next_child(pos, css_parent(pos));
3088
	if (next)
3089
		return css_leftmost_descendant(next);
3090 3091

	/* no sibling left, visit parent */
3092
	return css_parent(pos);
3093
}
3094
EXPORT_SYMBOL_GPL(css_next_descendant_post);
3095

3096
/**
3097
 * css_advance_task_iter - advance a task itererator to the next css_set
3098 3099 3100
 * @it: the iterator to advance
 *
 * Advance @it to the next css_set to walk.
3101
 */
3102
static void css_advance_task_iter(struct css_task_iter *it)
3103 3104 3105 3106 3107 3108 3109 3110
{
	struct list_head *l = it->cset_link;
	struct cgrp_cset_link *link;
	struct css_set *cset;

	/* Advance to the next non-empty css_set */
	do {
		l = l->next;
3111
		if (l == &it->origin_css->cgroup->cset_links) {
3112 3113 3114 3115 3116 3117 3118 3119 3120 3121
			it->cset_link = NULL;
			return;
		}
		link = list_entry(l, struct cgrp_cset_link, cset_link);
		cset = link->cset;
	} while (list_empty(&cset->tasks));
	it->cset_link = l;
	it->task = cset->tasks.next;
}

3122
/**
3123 3124
 * css_task_iter_start - initiate task iteration
 * @css: the css to walk tasks of
3125 3126
 * @it: the task iterator to use
 *
3127 3128 3129 3130
 * Initiate iteration through the tasks of @css.  The caller can call
 * css_task_iter_next() to walk through the tasks until the function
 * returns NULL.  On completion of iteration, css_task_iter_end() must be
 * called.
3131 3132 3133 3134 3135
 *
 * Note that this function acquires a lock which is released when the
 * iteration finishes.  The caller can't sleep while iteration is in
 * progress.
 */
3136 3137
void css_task_iter_start(struct cgroup_subsys_state *css,
			 struct css_task_iter *it)
3138
	__acquires(css_set_lock)
3139 3140
{
	/*
3141 3142 3143
	 * The first time anyone tries to iterate across a css, we need to
	 * enable the list linking each css_set to its tasks, and fix up
	 * all existing tasks.
3144
	 */
3145 3146 3147
	if (!use_task_css_set_links)
		cgroup_enable_task_cg_lists();

3148
	read_lock(&css_set_lock);
3149

3150 3151
	it->origin_css = css;
	it->cset_link = &css->cgroup->cset_links;
3152

3153
	css_advance_task_iter(it);
3154 3155
}

3156
/**
3157
 * css_task_iter_next - return the next task for the iterator
3158 3159 3160
 * @it: the task iterator being iterated
 *
 * The "next" function for task iteration.  @it should have been
3161 3162
 * initialized via css_task_iter_start().  Returns NULL when the iteration
 * reaches the end.
3163
 */
3164
struct task_struct *css_task_iter_next(struct css_task_iter *it)
3165 3166 3167
{
	struct task_struct *res;
	struct list_head *l = it->task;
3168
	struct cgrp_cset_link *link;
3169 3170

	/* If the iterator cg is NULL, we have no tasks */
3171
	if (!it->cset_link)
3172 3173 3174 3175
		return NULL;
	res = list_entry(l, struct task_struct, cg_list);
	/* Advance iterator to find next entry */
	l = l->next;
3176 3177
	link = list_entry(it->cset_link, struct cgrp_cset_link, cset_link);
	if (l == &link->cset->tasks) {
3178 3179 3180 3181
		/*
		 * We reached the end of this task list - move on to the
		 * next cgrp_cset_link.
		 */
3182
		css_advance_task_iter(it);
3183 3184 3185 3186 3187 3188
	} else {
		it->task = l;
	}
	return res;
}

3189
/**
3190
 * css_task_iter_end - finish task iteration
3191 3192
 * @it: the task iterator to finish
 *
3193
 * Finish task iteration started by css_task_iter_start().
3194
 */
3195
void css_task_iter_end(struct css_task_iter *it)
3196
	__releases(css_set_lock)
3197 3198 3199 3200
{
	read_unlock(&css_set_lock);
}

3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
static inline int started_after_time(struct task_struct *t1,
				     struct timespec *time,
				     struct task_struct *t2)
{
	int start_diff = timespec_compare(&t1->start_time, time);
	if (start_diff > 0) {
		return 1;
	} else if (start_diff < 0) {
		return 0;
	} else {
		/*
		 * Arbitrarily, if two processes started at the same
		 * time, we'll say that the lower pointer value
		 * started first. Note that t2 may have exited by now
		 * so this may not be a valid pointer any longer, but
		 * that's fine - it still serves to distinguish
		 * between two tasks started (effectively) simultaneously.
		 */
		return t1 > t2;
	}
}

/*
 * This function is a callback from heap_insert() and is used to order
 * the heap.
 * In this case we order the heap in descending task start time.
 */
static inline int started_after(void *p1, void *p2)
{
	struct task_struct *t1 = p1;
	struct task_struct *t2 = p2;
	return started_after_time(t1, &t2->start_time, t2);
}

/**
3236 3237
 * css_scan_tasks - iterate though all the tasks in a css
 * @css: the css to iterate tasks of
T
Tejun Heo 已提交
3238 3239 3240 3241
 * @test: optional test callback
 * @process: process callback
 * @data: data passed to @test and @process
 * @heap: optional pre-allocated heap used for task iteration
3242
 *
3243 3244
 * Iterate through all the tasks in @css, calling @test for each, and if it
 * returns %true, call @process for it also.
3245
 *
T
Tejun Heo 已提交
3246
 * @test may be NULL, meaning always true (select all tasks), which
3247
 * effectively duplicates css_task_iter_{start,next,end}() but does not
T
Tejun Heo 已提交
3248 3249 3250
 * lock css_set_lock for the call to @process.
 *
 * It is guaranteed that @process will act on every task that is a member
3251 3252 3253
 * of @css for the duration of this call.  This function may or may not
 * call @process for tasks that exit or move to a different css during the
 * call, or are forked or move into the css during the call.
3254
 *
T
Tejun Heo 已提交
3255 3256 3257
 * Note that @test may be called with locks held, and may in some
 * situations be called multiple times for the same task, so it should be
 * cheap.
3258
 *
T
Tejun Heo 已提交
3259 3260 3261 3262
 * If @heap is non-NULL, a heap has been pre-allocated and will be used for
 * heap operations (and its "gt" member will be overwritten), else a
 * temporary heap will be used (allocation of which may cause this function
 * to fail).
3263
 */
3264 3265 3266 3267
int css_scan_tasks(struct cgroup_subsys_state *css,
		   bool (*test)(struct task_struct *, void *),
		   void (*process)(struct task_struct *, void *),
		   void *data, struct ptr_heap *heap)
3268 3269
{
	int retval, i;
3270
	struct css_task_iter it;
3271 3272 3273 3274 3275 3276
	struct task_struct *p, *dropped;
	/* Never dereference latest_task, since it's not refcounted */
	struct task_struct *latest_task = NULL;
	struct ptr_heap tmp_heap;
	struct timespec latest_time = { 0, 0 };

T
Tejun Heo 已提交
3277
	if (heap) {
3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
		/* The caller supplied our heap and pre-allocated its memory */
		heap->gt = &started_after;
	} else {
		/* We need to allocate our own heap memory */
		heap = &tmp_heap;
		retval = heap_init(heap, PAGE_SIZE, GFP_KERNEL, &started_after);
		if (retval)
			/* cannot allocate the heap */
			return retval;
	}

 again:
	/*
3291
	 * Scan tasks in the css, using the @test callback to determine
T
Tejun Heo 已提交
3292 3293 3294 3295 3296 3297 3298
	 * which are of interest, and invoking @process callback on the
	 * ones which need an update.  Since we don't want to hold any
	 * locks during the task updates, gather tasks to be processed in a
	 * heap structure.  The heap is sorted by descending task start
	 * time.  If the statically-sized heap fills up, we overflow tasks
	 * that started later, and in future iterations only consider tasks
	 * that started after the latest task in the previous pass. This
3299 3300 3301
	 * guarantees forward progress and that we don't miss any tasks.
	 */
	heap->size = 0;
3302 3303
	css_task_iter_start(css, &it);
	while ((p = css_task_iter_next(&it))) {
3304 3305 3306 3307
		/*
		 * Only affect tasks that qualify per the caller's callback,
		 * if he provided one
		 */
T
Tejun Heo 已提交
3308
		if (test && !test(p, data))
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335
			continue;
		/*
		 * Only process tasks that started after the last task
		 * we processed
		 */
		if (!started_after_time(p, &latest_time, latest_task))
			continue;
		dropped = heap_insert(heap, p);
		if (dropped == NULL) {
			/*
			 * The new task was inserted; the heap wasn't
			 * previously full
			 */
			get_task_struct(p);
		} else if (dropped != p) {
			/*
			 * The new task was inserted, and pushed out a
			 * different task
			 */
			get_task_struct(p);
			put_task_struct(dropped);
		}
		/*
		 * Else the new task was newer than anything already in
		 * the heap and wasn't inserted
		 */
	}
3336
	css_task_iter_end(&it);
3337 3338 3339

	if (heap->size) {
		for (i = 0; i < heap->size; i++) {
3340
			struct task_struct *q = heap->ptrs[i];
3341
			if (i == 0) {
3342 3343
				latest_time = q->start_time;
				latest_task = q;
3344 3345
			}
			/* Process the task per the caller's callback */
T
Tejun Heo 已提交
3346
			process(q, data);
3347
			put_task_struct(q);
3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362
		}
		/*
		 * If we had to process any tasks at all, scan again
		 * in case some of them were in the middle of forking
		 * children that didn't get processed.
		 * Not the most efficient way to do it, but it avoids
		 * having to take callback_mutex in the fork path
		 */
		goto again;
	}
	if (heap == &tmp_heap)
		heap_free(&tmp_heap);
	return 0;
}

T
Tejun Heo 已提交
3363
static void cgroup_transfer_one_task(struct task_struct *task, void *data)
3364
{
T
Tejun Heo 已提交
3365
	struct cgroup *new_cgroup = data;
3366

T
Tejun Heo 已提交
3367
	mutex_lock(&cgroup_mutex);
3368
	cgroup_attach_task(new_cgroup, task, false);
T
Tejun Heo 已提交
3369
	mutex_unlock(&cgroup_mutex);
3370 3371 3372 3373 3374 3375 3376 3377 3378
}

/**
 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
 * @to: cgroup to which the tasks will be moved
 * @from: cgroup in which the tasks currently reside
 */
int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
{
3379 3380
	return css_scan_tasks(&from->dummy_css, NULL, cgroup_transfer_one_task,
			      to, NULL);
3381 3382
}

3383
/*
3384
 * Stuff for reading the 'tasks'/'procs' files.
3385 3386 3387 3388 3389 3390 3391 3392
 *
 * Reading this file can return large amounts of data if a cgroup has
 * *lots* of attached tasks. So it may need several calls to read(),
 * but we cannot guarantee that the information we produce is correct
 * unless we produce it entirely atomically.
 *
 */

3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418
/* which pidlist file are we talking about? */
enum cgroup_filetype {
	CGROUP_FILE_PROCS,
	CGROUP_FILE_TASKS,
};

/*
 * A pidlist is a list of pids that virtually represents the contents of one
 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
 * a pair (one each for procs, tasks) for each pid namespace that's relevant
 * to the cgroup.
 */
struct cgroup_pidlist {
	/*
	 * used to find which pidlist is wanted. doesn't change as long as
	 * this particular list stays in the list.
	*/
	struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
	/* array of xids */
	pid_t *list;
	/* how many elements the above list has */
	int length;
	/* each of these stored in a list by its cgroup */
	struct list_head links;
	/* pointer to the cgroup we belong to, for list removal purposes */
	struct cgroup *owner;
3419 3420
	/* for delayed destruction */
	struct delayed_work destroy_dwork;
3421 3422
};

3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435
/*
 * The following two functions "fix" the issue where there are more pids
 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
 * TODO: replace with a kernel-wide solution to this problem
 */
#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
static void *pidlist_allocate(int count)
{
	if (PIDLIST_TOO_LARGE(count))
		return vmalloc(count * sizeof(pid_t));
	else
		return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
}
3436

3437 3438 3439 3440 3441 3442 3443 3444
static void pidlist_free(void *p)
{
	if (is_vmalloc_addr(p))
		vfree(p);
	else
		kfree(p);
}

3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471
/*
 * Used to destroy all pidlists lingering waiting for destroy timer.  None
 * should be left afterwards.
 */
static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
{
	struct cgroup_pidlist *l, *tmp_l;

	mutex_lock(&cgrp->pidlist_mutex);
	list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
	mutex_unlock(&cgrp->pidlist_mutex);

	flush_workqueue(cgroup_pidlist_destroy_wq);
	BUG_ON(!list_empty(&cgrp->pidlists));
}

static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
{
	struct delayed_work *dwork = to_delayed_work(work);
	struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
						destroy_dwork);
	struct cgroup_pidlist *tofree = NULL;

	mutex_lock(&l->owner->pidlist_mutex);

	/*
3472 3473
	 * Destroy iff we didn't get queued again.  The state won't change
	 * as destroy_dwork can only be queued while locked.
3474
	 */
3475
	if (!delayed_work_pending(dwork)) {
3476 3477 3478 3479 3480 3481 3482 3483 3484 3485
		list_del(&l->links);
		pidlist_free(l->list);
		put_pid_ns(l->key.ns);
		tofree = l;
	}

	mutex_unlock(&l->owner->pidlist_mutex);
	kfree(tofree);
}

3486
/*
3487
 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
3488
 * Returns the number of unique elements.
3489
 */
3490
static int pidlist_uniq(pid_t *list, int length)
3491
{
3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515
	int src, dest = 1;

	/*
	 * we presume the 0th element is unique, so i starts at 1. trivial
	 * edge cases first; no work needs to be done for either
	 */
	if (length == 0 || length == 1)
		return length;
	/* src and dest walk down the list; dest counts unique elements */
	for (src = 1; src < length; src++) {
		/* find next unique element */
		while (list[src] == list[src-1]) {
			src++;
			if (src == length)
				goto after;
		}
		/* dest always points to where the next unique element goes */
		list[dest] = list[src];
		dest++;
	}
after:
	return dest;
}

3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548
/*
 * The two pid files - task and cgroup.procs - guaranteed that the result
 * is sorted, which forced this whole pidlist fiasco.  As pid order is
 * different per namespace, each namespace needs differently sorted list,
 * making it impossible to use, for example, single rbtree of member tasks
 * sorted by task pointer.  As pidlists can be fairly large, allocating one
 * per open file is dangerous, so cgroup had to implement shared pool of
 * pidlists keyed by cgroup and namespace.
 *
 * All this extra complexity was caused by the original implementation
 * committing to an entirely unnecessary property.  In the long term, we
 * want to do away with it.  Explicitly scramble sort order if
 * sane_behavior so that no such expectation exists in the new interface.
 *
 * Scrambling is done by swapping every two consecutive bits, which is
 * non-identity one-to-one mapping which disturbs sort order sufficiently.
 */
static pid_t pid_fry(pid_t pid)
{
	unsigned a = pid & 0x55555555;
	unsigned b = pid & 0xAAAAAAAA;

	return (a << 1) | (b >> 1);
}

static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
{
	if (cgroup_sane_behavior(cgrp))
		return pid_fry(pid);
	else
		return pid;
}

3549 3550 3551 3552 3553
static int cmppid(const void *a, const void *b)
{
	return *(pid_t *)a - *(pid_t *)b;
}

3554 3555 3556 3557 3558
static int fried_cmppid(const void *a, const void *b)
{
	return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
}

T
Tejun Heo 已提交
3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573
static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
						  enum cgroup_filetype type)
{
	struct cgroup_pidlist *l;
	/* don't need task_nsproxy() if we're looking at ourself */
	struct pid_namespace *ns = task_active_pid_ns(current);

	lockdep_assert_held(&cgrp->pidlist_mutex);

	list_for_each_entry(l, &cgrp->pidlists, links)
		if (l->key.type == type && l->key.ns == ns)
			return l;
	return NULL;
}

3574 3575 3576 3577 3578 3579
/*
 * find the appropriate pidlist for our purpose (given procs vs tasks)
 * returns with the lock on that pidlist already held, and takes care
 * of the use count, or returns NULL with no locks held if we're out of
 * memory.
 */
T
Tejun Heo 已提交
3580 3581
static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
						enum cgroup_filetype type)
3582 3583
{
	struct cgroup_pidlist *l;
3584

T
Tejun Heo 已提交
3585 3586 3587 3588 3589 3590
	lockdep_assert_held(&cgrp->pidlist_mutex);

	l = cgroup_pidlist_find(cgrp, type);
	if (l)
		return l;

3591
	/* entry not found; create a new one */
3592
	l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
T
Tejun Heo 已提交
3593
	if (!l)
3594
		return l;
T
Tejun Heo 已提交
3595

3596
	INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
3597
	l->key.type = type;
T
Tejun Heo 已提交
3598 3599
	/* don't need task_nsproxy() if we're looking at ourself */
	l->key.ns = get_pid_ns(task_active_pid_ns(current));
3600 3601 3602 3603 3604
	l->owner = cgrp;
	list_add(&l->links, &cgrp->pidlists);
	return l;
}

3605 3606 3607
/*
 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
 */
3608 3609
static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
			      struct cgroup_pidlist **lp)
3610 3611 3612 3613
{
	pid_t *array;
	int length;
	int pid, n = 0; /* used for populating the array */
3614
	struct css_task_iter it;
3615
	struct task_struct *tsk;
3616 3617
	struct cgroup_pidlist *l;

3618 3619
	lockdep_assert_held(&cgrp->pidlist_mutex);

3620 3621 3622 3623 3624 3625 3626
	/*
	 * If cgroup gets more users after we read count, we won't have
	 * enough space - tough.  This race is indistinguishable to the
	 * caller from the case that the additional cgroup users didn't
	 * show up until sometime later on.
	 */
	length = cgroup_task_count(cgrp);
3627
	array = pidlist_allocate(length);
3628 3629 3630
	if (!array)
		return -ENOMEM;
	/* now, populate the array */
3631 3632
	css_task_iter_start(&cgrp->dummy_css, &it);
	while ((tsk = css_task_iter_next(&it))) {
3633
		if (unlikely(n == length))
3634
			break;
3635
		/* get tgid or pid for procs or tasks file respectively */
3636 3637 3638 3639
		if (type == CGROUP_FILE_PROCS)
			pid = task_tgid_vnr(tsk);
		else
			pid = task_pid_vnr(tsk);
3640 3641
		if (pid > 0) /* make sure to only use valid results */
			array[n++] = pid;
3642
	}
3643
	css_task_iter_end(&it);
3644 3645
	length = n;
	/* now sort & (if procs) strip out duplicates */
3646 3647 3648 3649
	if (cgroup_sane_behavior(cgrp))
		sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
	else
		sort(array, length, sizeof(pid_t), cmppid, NULL);
3650
	if (type == CGROUP_FILE_PROCS)
3651
		length = pidlist_uniq(array, length);
T
Tejun Heo 已提交
3652 3653

	l = cgroup_pidlist_find_create(cgrp, type);
3654
	if (!l) {
T
Tejun Heo 已提交
3655
		mutex_unlock(&cgrp->pidlist_mutex);
3656
		pidlist_free(array);
3657
		return -ENOMEM;
3658
	}
T
Tejun Heo 已提交
3659 3660

	/* store array, freeing old if necessary */
3661
	pidlist_free(l->list);
3662 3663
	l->list = array;
	l->length = length;
3664
	*lp = l;
3665
	return 0;
3666 3667
}

B
Balbir Singh 已提交
3668
/**
L
Li Zefan 已提交
3669
 * cgroupstats_build - build and fill cgroupstats
B
Balbir Singh 已提交
3670 3671 3672
 * @stats: cgroupstats to fill information into
 * @dentry: A dentry entry belonging to the cgroup for which stats have
 * been requested.
L
Li Zefan 已提交
3673 3674 3675
 *
 * Build and fill cgroupstats so that taskstats can export it to user
 * space.
B
Balbir Singh 已提交
3676 3677 3678 3679
 */
int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
{
	int ret = -EINVAL;
3680
	struct cgroup *cgrp;
3681
	struct css_task_iter it;
B
Balbir Singh 已提交
3682
	struct task_struct *tsk;
3683

B
Balbir Singh 已提交
3684
	/*
3685 3686
	 * Validate dentry by checking the superblock operations,
	 * and make sure it's a directory.
B
Balbir Singh 已提交
3687
	 */
3688 3689
	if (dentry->d_sb->s_op != &cgroup_ops ||
	    !S_ISDIR(dentry->d_inode->i_mode))
B
Balbir Singh 已提交
3690 3691 3692
		 goto err;

	ret = 0;
3693
	cgrp = dentry->d_fsdata;
B
Balbir Singh 已提交
3694

3695 3696
	css_task_iter_start(&cgrp->dummy_css, &it);
	while ((tsk = css_task_iter_next(&it))) {
B
Balbir Singh 已提交
3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715
		switch (tsk->state) {
		case TASK_RUNNING:
			stats->nr_running++;
			break;
		case TASK_INTERRUPTIBLE:
			stats->nr_sleeping++;
			break;
		case TASK_UNINTERRUPTIBLE:
			stats->nr_uninterruptible++;
			break;
		case TASK_STOPPED:
			stats->nr_stopped++;
			break;
		default:
			if (delayacct_is_task_waiting_on_io(tsk))
				stats->nr_io_wait++;
			break;
		}
	}
3716
	css_task_iter_end(&it);
B
Balbir Singh 已提交
3717 3718 3719 3720 3721

err:
	return ret;
}

3722

3723
/*
3724
 * seq_file methods for the tasks/procs files. The seq_file position is the
3725
 * next pid to display; the seq_file iterator is a pointer to the pid
3726
 * in the cgroup->l->list array.
3727
 */
3728

3729
static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
3730
{
3731 3732 3733 3734 3735 3736
	/*
	 * Initially we receive a position value that corresponds to
	 * one more than the last pid shown (or 0 on the first call or
	 * after a seek to the start). Use a binary-search to find the
	 * next pid to display, if any
	 */
3737
	struct cgroup_open_file *of = s->private;
3738
	struct cgroup *cgrp = seq_css(s)->cgroup;
3739
	struct cgroup_pidlist *l;
3740
	enum cgroup_filetype type = seq_cft(s)->private;
3741
	int index = 0, pid = *pos;
3742 3743 3744 3745 3746
	int *iter, ret;

	mutex_lock(&cgrp->pidlist_mutex);

	/*
3747
	 * !NULL @of->priv indicates that this isn't the first start()
3748
	 * after open.  If the matching pidlist is around, we can use that.
3749
	 * Look for it.  Note that @of->priv can't be used directly.  It
3750 3751
	 * could already have been destroyed.
	 */
3752 3753
	if (of->priv)
		of->priv = cgroup_pidlist_find(cgrp, type);
3754 3755 3756 3757 3758

	/*
	 * Either this is the first start() after open or the matching
	 * pidlist has been destroyed inbetween.  Create a new one.
	 */
3759 3760 3761
	if (!of->priv) {
		ret = pidlist_array_load(cgrp, type,
					 (struct cgroup_pidlist **)&of->priv);
3762 3763 3764
		if (ret)
			return ERR_PTR(ret);
	}
3765
	l = of->priv;
3766 3767

	if (pid) {
3768
		int end = l->length;
S
Stephen Rothwell 已提交
3769

3770 3771
		while (index < end) {
			int mid = (index + end) / 2;
3772
			if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
3773 3774
				index = mid;
				break;
3775
			} else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
3776 3777 3778 3779 3780 3781
				index = mid + 1;
			else
				end = mid;
		}
	}
	/* If we're off the end of the array, we're done */
3782
	if (index >= l->length)
3783 3784
		return NULL;
	/* Update the abstract position to be the actual pid that we found */
3785
	iter = l->list + index;
3786
	*pos = cgroup_pid_fry(cgrp, *iter);
3787 3788 3789
	return iter;
}

3790
static void cgroup_pidlist_stop(struct seq_file *s, void *v)
3791
{
3792 3793
	struct cgroup_open_file *of = s->private;
	struct cgroup_pidlist *l = of->priv;
3794

3795 3796
	if (l)
		mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
3797
				 CGROUP_PIDLIST_DESTROY_DELAY);
3798
	mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
3799 3800
}

3801
static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
3802
{
3803 3804
	struct cgroup_open_file *of = s->private;
	struct cgroup_pidlist *l = of->priv;
3805 3806
	pid_t *p = v;
	pid_t *end = l->list + l->length;
3807 3808 3809 3810 3811 3812 3813 3814
	/*
	 * Advance to the next pid in the array. If this goes off the
	 * end, we're done
	 */
	p++;
	if (p >= end) {
		return NULL;
	} else {
3815
		*pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
3816 3817 3818 3819
		return p;
	}
}

3820
static int cgroup_pidlist_show(struct seq_file *s, void *v)
3821 3822 3823
{
	return seq_printf(s, "%d\n", *(int *)v);
}
3824

3825 3826 3827 3828 3829 3830 3831 3832 3833
/*
 * seq_operations functions for iterating on pidlists through seq_file -
 * independent of whether it's tasks or procs
 */
static const struct seq_operations cgroup_pidlist_seq_operations = {
	.start = cgroup_pidlist_start,
	.stop = cgroup_pidlist_stop,
	.next = cgroup_pidlist_next,
	.show = cgroup_pidlist_show,
3834 3835
};

3836 3837
static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
					 struct cftype *cft)
3838
{
3839
	return notify_on_release(css->cgroup);
3840 3841
}

3842 3843
static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
					  struct cftype *cft, u64 val)
3844
{
3845
	clear_bit(CGRP_RELEASABLE, &css->cgroup->flags);
3846
	if (val)
3847
		set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3848
	else
3849
		clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
3850 3851 3852
	return 0;
}

3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869
/*
 * When dput() is called asynchronously, if umount has been done and
 * then deactivate_super() in cgroup_free_fn() kills the superblock,
 * there's a small window that vfs will see the root dentry with non-zero
 * refcnt and trigger BUG().
 *
 * That's why we hold a reference before dput() and drop it right after.
 */
static void cgroup_dput(struct cgroup *cgrp)
{
	struct super_block *sb = cgrp->root->sb;

	atomic_inc(&sb->s_active);
	dput(cgrp->dentry);
	deactivate_super(sb);
}

3870 3871
static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
3872
{
3873
	return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3874 3875
}

3876 3877
static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
3878 3879
{
	if (val)
3880
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3881
	else
3882
		clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
3883 3884 3885
	return 0;
}

3886
static struct cftype cgroup_base_files[] = {
3887
	{
3888
		.name = "cgroup.procs",
3889 3890 3891 3892
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
3893
		.private = CGROUP_FILE_PROCS,
B
Ben Blum 已提交
3894 3895
		.write_u64 = cgroup_procs_write,
		.mode = S_IRUGO | S_IWUSR,
3896
	},
3897 3898
	{
		.name = "cgroup.clone_children",
3899
		.flags = CFTYPE_INSANE,
3900 3901 3902
		.read_u64 = cgroup_clone_children_read,
		.write_u64 = cgroup_clone_children_write,
	},
3903 3904 3905
	{
		.name = "cgroup.sane_behavior",
		.flags = CFTYPE_ONLY_ON_ROOT,
3906
		.seq_show = cgroup_sane_behavior_show,
3907
	},
3908 3909 3910 3911 3912 3913 3914 3915 3916

	/*
	 * Historical crazy stuff.  These don't have "cgroup."  prefix and
	 * don't exist if sane_behavior.  If you're depending on these, be
	 * prepared to be burned.
	 */
	{
		.name = "tasks",
		.flags = CFTYPE_INSANE,		/* use "procs" instead */
3917 3918 3919 3920
		.seq_start = cgroup_pidlist_start,
		.seq_next = cgroup_pidlist_next,
		.seq_stop = cgroup_pidlist_stop,
		.seq_show = cgroup_pidlist_show,
3921
		.private = CGROUP_FILE_TASKS,
3922 3923 3924 3925 3926 3927 3928 3929 3930
		.write_u64 = cgroup_tasks_write,
		.mode = S_IRUGO | S_IWUSR,
	},
	{
		.name = "notify_on_release",
		.flags = CFTYPE_INSANE,
		.read_u64 = cgroup_read_notify_on_release,
		.write_u64 = cgroup_write_notify_on_release,
	},
3931 3932
	{
		.name = "release_agent",
3933
		.flags = CFTYPE_INSANE | CFTYPE_ONLY_ON_ROOT,
3934
		.seq_show = cgroup_release_agent_show,
3935 3936 3937
		.write_string = cgroup_release_agent_write,
		.max_write_len = PATH_MAX,
	},
T
Tejun Heo 已提交
3938
	{ }	/* terminate */
3939 3940
};

3941
/**
3942
 * cgroup_populate_dir - create subsys files in a cgroup directory
3943 3944
 * @cgrp: target cgroup
 * @subsys_mask: mask of the subsystem ids whose files should be added
3945 3946
 *
 * On failure, no file is added.
3947
 */
3948
static int cgroup_populate_dir(struct cgroup *cgrp, unsigned long subsys_mask)
3949 3950
{
	struct cgroup_subsys *ss;
3951
	int i, ret = 0;
3952

3953
	/* process cftsets of each subsystem */
3954
	for_each_subsys(ss, i) {
3955
		struct cftype_set *set;
3956 3957

		if (!test_bit(i, &subsys_mask))
3958
			continue;
3959

3960
		list_for_each_entry(set, &ss->cftsets, node) {
3961
			ret = cgroup_addrm_files(cgrp, set->cfts, true);
3962 3963 3964
			if (ret < 0)
				goto err;
		}
3965 3966
	}
	return 0;
3967 3968 3969
err:
	cgroup_clear_dir(cgrp, subsys_mask);
	return ret;
3970 3971
}

3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993
/*
 * css destruction is four-stage process.
 *
 * 1. Destruction starts.  Killing of the percpu_ref is initiated.
 *    Implemented in kill_css().
 *
 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
 *    and thus css_tryget() is guaranteed to fail, the css can be offlined
 *    by invoking offline_css().  After offlining, the base ref is put.
 *    Implemented in css_killed_work_fn().
 *
 * 3. When the percpu_ref reaches zero, the only possible remaining
 *    accessors are inside RCU read sections.  css_release() schedules the
 *    RCU callback.
 *
 * 4. After the grace period, the css can be freed.  Implemented in
 *    css_free_work_fn().
 *
 * It is actually hairier because both step 2 and 4 require process context
 * and thus involve punting to css->destroy_work adding two additional
 * steps to the already complex sequence.
 */
3994
static void css_free_work_fn(struct work_struct *work)
3995 3996
{
	struct cgroup_subsys_state *css =
3997
		container_of(work, struct cgroup_subsys_state, destroy_work);
3998
	struct cgroup *cgrp = css->cgroup;
3999

4000 4001 4002
	if (css->parent)
		css_put(css->parent);

4003 4004
	css->ss->css_free(css);
	cgroup_dput(cgrp);
4005 4006
}

4007
static void css_free_rcu_fn(struct rcu_head *rcu_head)
4008 4009
{
	struct cgroup_subsys_state *css =
4010
		container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
4011

4012 4013
	/*
	 * css holds an extra ref to @cgrp->dentry which is put on the last
4014
	 * css_put().  dput() requires process context which we don't have.
4015 4016
	 */
	INIT_WORK(&css->destroy_work, css_free_work_fn);
4017
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4018 4019
}

4020 4021 4022 4023 4024
static void css_release(struct percpu_ref *ref)
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4025
	call_rcu(&css->rcu_head, css_free_rcu_fn);
4026 4027
}

4028 4029
static void init_css(struct cgroup_subsys_state *css, struct cgroup_subsys *ss,
		     struct cgroup *cgrp)
4030
{
4031
	css->cgroup = cgrp;
4032
	css->ss = ss;
4033
	css->flags = 0;
4034 4035

	if (cgrp->parent)
4036
		css->parent = cgroup_css(cgrp->parent, ss);
4037
	else
4038
		css->flags |= CSS_ROOT;
4039

4040
	BUG_ON(cgroup_css(cgrp, ss));
4041 4042
}

4043
/* invoke ->css_online() on a new CSS and mark it online if successful */
4044
static int online_css(struct cgroup_subsys_state *css)
4045
{
4046
	struct cgroup_subsys *ss = css->ss;
T
Tejun Heo 已提交
4047 4048
	int ret = 0;

4049 4050
	lockdep_assert_held(&cgroup_mutex);

4051
	if (ss->css_online)
4052
		ret = ss->css_online(css);
4053
	if (!ret) {
4054
		css->flags |= CSS_ONLINE;
4055
		css->cgroup->nr_css++;
4056 4057
		rcu_assign_pointer(css->cgroup->subsys[ss->subsys_id], css);
	}
T
Tejun Heo 已提交
4058
	return ret;
4059 4060
}

4061
/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
4062
static void offline_css(struct cgroup_subsys_state *css)
4063
{
4064
	struct cgroup_subsys *ss = css->ss;
4065 4066 4067 4068 4069 4070

	lockdep_assert_held(&cgroup_mutex);

	if (!(css->flags & CSS_ONLINE))
		return;

4071
	if (ss->css_offline)
4072
		ss->css_offline(css);
4073

4074
	css->flags &= ~CSS_ONLINE;
4075
	css->cgroup->nr_css--;
4076
	RCU_INIT_POINTER(css->cgroup->subsys[ss->subsys_id], css);
4077 4078
}

4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134
/**
 * create_css - create a cgroup_subsys_state
 * @cgrp: the cgroup new css will be associated with
 * @ss: the subsys of new css
 *
 * Create a new css associated with @cgrp - @ss pair.  On success, the new
 * css is online and installed in @cgrp with all interface files created.
 * Returns 0 on success, -errno on failure.
 */
static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss)
{
	struct cgroup *parent = cgrp->parent;
	struct cgroup_subsys_state *css;
	int err;

	lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
	lockdep_assert_held(&cgroup_mutex);

	css = ss->css_alloc(cgroup_css(parent, ss));
	if (IS_ERR(css))
		return PTR_ERR(css);

	err = percpu_ref_init(&css->refcnt, css_release);
	if (err)
		goto err_free;

	init_css(css, ss, cgrp);

	err = cgroup_populate_dir(cgrp, 1 << ss->subsys_id);
	if (err)
		goto err_free;

	err = online_css(css);
	if (err)
		goto err_free;

	dget(cgrp->dentry);
	css_get(css->parent);

	if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
	    parent->parent) {
		pr_warning("cgroup: %s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
			   current->comm, current->pid, ss->name);
		if (!strcmp(ss->name, "memory"))
			pr_warning("cgroup: \"memory\" requires setting use_hierarchy to 1 on the root.\n");
		ss->warned_broken_hierarchy = true;
	}

	return 0;

err_free:
	percpu_ref_cancel_init(&css->refcnt);
	ss->css_free(css);
	return err;
}

4135
/*
L
Li Zefan 已提交
4136 4137 4138 4139
 * cgroup_create - create a cgroup
 * @parent: cgroup that will be parent of the new cgroup
 * @dentry: dentry of the new cgroup
 * @mode: mode to set on new inode
4140
 *
L
Li Zefan 已提交
4141
 * Must be called with the mutex on the parent inode held
4142 4143
 */
static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
A
Al Viro 已提交
4144
			     umode_t mode)
4145
{
4146
	struct cgroup *cgrp;
4147
	struct cgroup_name *name;
4148
	struct cgroupfs_root *root = parent->root;
T
Tejun Heo 已提交
4149
	int ssid, err = 0;
4150 4151 4152
	struct cgroup_subsys *ss;
	struct super_block *sb = root->sb;

T
Tejun Heo 已提交
4153
	/* allocate the cgroup and its ID, 0 is reserved for the root */
4154 4155
	cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL);
	if (!cgrp)
4156 4157
		return -ENOMEM;

4158 4159 4160 4161 4162
	name = cgroup_alloc_name(dentry);
	if (!name)
		goto err_free_cgrp;
	rcu_assign_pointer(cgrp->name, name);

4163 4164 4165 4166 4167
	/*
	 * Temporarily set the pointer to NULL, so idr_find() won't return
	 * a half-baked cgroup.
	 */
	cgrp->id = idr_alloc(&root->cgroup_idr, NULL, 1, 0, GFP_KERNEL);
T
Tejun Heo 已提交
4168
	if (cgrp->id < 0)
4169
		goto err_free_name;
T
Tejun Heo 已提交
4170

4171 4172 4173 4174 4175 4176 4177 4178 4179
	/*
	 * Only live parents can have children.  Note that the liveliness
	 * check isn't strictly necessary because cgroup_mkdir() and
	 * cgroup_rmdir() are fully synchronized by i_mutex; however, do it
	 * anyway so that locking is contained inside cgroup proper and we
	 * don't get nasty surprises if we ever grow another caller.
	 */
	if (!cgroup_lock_live_group(parent)) {
		err = -ENODEV;
T
Tejun Heo 已提交
4180
		goto err_free_id;
4181 4182
	}

4183 4184 4185 4186 4187 4188 4189
	/* Grab a reference on the superblock so the hierarchy doesn't
	 * get deleted on unmount if there are child cgroups.  This
	 * can be done outside cgroup_mutex, since the sb can't
	 * disappear while someone has an open control file on the
	 * fs */
	atomic_inc(&sb->s_active);

4190
	init_cgroup_housekeeping(cgrp);
4191

4192 4193 4194
	dentry->d_fsdata = cgrp;
	cgrp->dentry = dentry;

4195
	cgrp->parent = parent;
4196
	cgrp->dummy_css.parent = &parent->dummy_css;
4197
	cgrp->root = parent->root;
4198

4199 4200 4201
	if (notify_on_release(parent))
		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);

4202 4203
	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
4204

4205 4206 4207 4208 4209
	/*
	 * Create directory.  cgroup_create_file() returns with the new
	 * directory locked on success so that it can be populated without
	 * dropping cgroup_mutex.
	 */
T
Tejun Heo 已提交
4210
	err = cgroup_create_file(dentry, S_IFDIR | mode, sb);
4211
	if (err < 0)
4212
		goto err_unlock;
4213
	lockdep_assert_held(&dentry->d_inode->i_mutex);
4214

4215
	cgrp->serial_nr = cgroup_serial_nr_next++;
4216

4217 4218 4219
	/* allocation complete, commit to creation */
	list_add_tail_rcu(&cgrp->sibling, &cgrp->parent->children);
	root->number_of_cgroups++;
T
Tejun Heo 已提交
4220

4221 4222 4223
	/* hold a ref to the parent's dentry */
	dget(parent->dentry);

4224 4225 4226 4227 4228 4229 4230 4231 4232 4233
	/*
	 * @cgrp is now fully operational.  If something fails after this
	 * point, it'll be released via the normal destruction path.
	 */
	idr_replace(&root->cgroup_idr, cgrp, cgrp->id);

	err = cgroup_addrm_files(cgrp, cgroup_base_files, true);
	if (err)
		goto err_destroy;

4234
	/* let's create and online css's */
T
Tejun Heo 已提交
4235 4236 4237 4238 4239 4240
	for_each_subsys(ss, ssid) {
		if (root->subsys_mask & (1 << ssid)) {
			err = create_css(cgrp, ss);
			if (err)
				goto err_destroy;
		}
4241 4242
	}

4243
	mutex_unlock(&cgroup_mutex);
4244
	mutex_unlock(&cgrp->dentry->d_inode->i_mutex);
4245 4246 4247

	return 0;

4248
err_unlock:
4249 4250 4251
	mutex_unlock(&cgroup_mutex);
	/* Release the reference count that we took on the superblock */
	deactivate_super(sb);
T
Tejun Heo 已提交
4252
err_free_id:
4253
	idr_remove(&root->cgroup_idr, cgrp->id);
4254 4255
err_free_name:
	kfree(rcu_dereference_raw(cgrp->name));
4256
err_free_cgrp:
4257
	kfree(cgrp);
4258
	return err;
4259 4260 4261 4262 4263 4264

err_destroy:
	cgroup_destroy_locked(cgrp);
	mutex_unlock(&cgroup_mutex);
	mutex_unlock(&dentry->d_inode->i_mutex);
	return err;
4265 4266
}

4267
static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
4268 4269 4270 4271 4272 4273 4274
{
	struct cgroup *c_parent = dentry->d_parent->d_fsdata;

	/* the vfs holds inode->i_mutex already */
	return cgroup_create(c_parent, dentry, mode | S_IFDIR);
}

4275 4276 4277 4278 4279
/*
 * This is called when the refcnt of a css is confirmed to be killed.
 * css_tryget() is now guaranteed to fail.
 */
static void css_killed_work_fn(struct work_struct *work)
4280
{
4281 4282 4283
	struct cgroup_subsys_state *css =
		container_of(work, struct cgroup_subsys_state, destroy_work);
	struct cgroup *cgrp = css->cgroup;
4284

4285 4286
	mutex_lock(&cgroup_mutex);

4287 4288 4289 4290 4291 4292
	/*
	 * css_tryget() is guaranteed to fail now.  Tell subsystems to
	 * initate destruction.
	 */
	offline_css(css);

4293 4294 4295 4296 4297
	/*
	 * If @cgrp is marked dead, it's waiting for refs of all css's to
	 * be disabled before proceeding to the second phase of cgroup
	 * destruction.  If we are the last one, kick it off.
	 */
4298
	if (!cgrp->nr_css && cgroup_is_dead(cgrp))
4299 4300 4301
		cgroup_destroy_css_killed(cgrp);

	mutex_unlock(&cgroup_mutex);
4302 4303 4304 4305 4306 4307 4308 4309 4310

	/*
	 * Put the css refs from kill_css().  Each css holds an extra
	 * reference to the cgroup's dentry and cgroup removal proceeds
	 * regardless of css refs.  On the last put of each css, whenever
	 * that may be, the extra dentry ref is put so that dentry
	 * destruction happens only after all css's are released.
	 */
	css_put(css);
4311 4312
}

4313 4314
/* css kill confirmation processing requires process context, bounce */
static void css_killed_ref_fn(struct percpu_ref *ref)
4315 4316 4317 4318
{
	struct cgroup_subsys_state *css =
		container_of(ref, struct cgroup_subsys_state, refcnt);

4319
	INIT_WORK(&css->destroy_work, css_killed_work_fn);
4320
	queue_work(cgroup_destroy_wq, &css->destroy_work);
4321 4322
}

T
Tejun Heo 已提交
4323 4324 4325 4326
/**
 * kill_css - destroy a css
 * @css: css to destroy
 *
4327 4328 4329 4330
 * This function initiates destruction of @css by removing cgroup interface
 * files and putting its base reference.  ->css_offline() will be invoked
 * asynchronously once css_tryget() is guaranteed to fail and when the
 * reference count reaches zero, @css will be released.
T
Tejun Heo 已提交
4331 4332 4333
 */
static void kill_css(struct cgroup_subsys_state *css)
{
4334 4335
	cgroup_clear_dir(css->cgroup, 1 << css->ss->subsys_id);

T
Tejun Heo 已提交
4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352
	/*
	 * Killing would put the base ref, but we need to keep it alive
	 * until after ->css_offline().
	 */
	css_get(css);

	/*
	 * cgroup core guarantees that, by the time ->css_offline() is
	 * invoked, no new css reference will be given out via
	 * css_tryget().  We can't simply call percpu_ref_kill() and
	 * proceed to offlining css's because percpu_ref_kill() doesn't
	 * guarantee that the ref is seen as killed on all CPUs on return.
	 *
	 * Use percpu_ref_kill_and_confirm() to get notifications as each
	 * css is confirmed to be seen as killed on all CPUs.
	 */
	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378
}

/**
 * cgroup_destroy_locked - the first stage of cgroup destruction
 * @cgrp: cgroup to be destroyed
 *
 * css's make use of percpu refcnts whose killing latency shouldn't be
 * exposed to userland and are RCU protected.  Also, cgroup core needs to
 * guarantee that css_tryget() won't succeed by the time ->css_offline() is
 * invoked.  To satisfy all the requirements, destruction is implemented in
 * the following two steps.
 *
 * s1. Verify @cgrp can be destroyed and mark it dying.  Remove all
 *     userland visible parts and start killing the percpu refcnts of
 *     css's.  Set up so that the next stage will be kicked off once all
 *     the percpu refcnts are confirmed to be killed.
 *
 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
 *     rest of destruction.  Once all cgroup references are gone, the
 *     cgroup is RCU-freed.
 *
 * This function implements s1.  After this step, @cgrp is gone as far as
 * the userland is concerned and a new cgroup with the same name may be
 * created.  As cgroup doesn't care about the names internally, this
 * doesn't cause any problem.
 */
4379 4380
static int cgroup_destroy_locked(struct cgroup *cgrp)
	__releases(&cgroup_mutex) __acquires(&cgroup_mutex)
4381
{
4382
	struct dentry *d = cgrp->dentry;
T
Tejun Heo 已提交
4383
	struct cgroup_subsys_state *css;
4384
	struct cgroup *child;
4385
	bool empty;
T
Tejun Heo 已提交
4386
	int ssid;
4387

4388 4389 4390
	lockdep_assert_held(&d->d_inode->i_mutex);
	lockdep_assert_held(&cgroup_mutex);

4391
	/*
T
Tejun Heo 已提交
4392 4393
	 * css_set_lock synchronizes access to ->cset_links and prevents
	 * @cgrp from being removed while __put_css_set() is in progress.
4394 4395
	 */
	read_lock(&css_set_lock);
4396
	empty = list_empty(&cgrp->cset_links);
4397 4398
	read_unlock(&css_set_lock);
	if (!empty)
4399
		return -EBUSY;
L
Li Zefan 已提交
4400

4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416
	/*
	 * Make sure there's no live children.  We can't test ->children
	 * emptiness as dead children linger on it while being destroyed;
	 * otherwise, "rmdir parent/child parent" may fail with -EBUSY.
	 */
	empty = true;
	rcu_read_lock();
	list_for_each_entry_rcu(child, &cgrp->children, sibling) {
		empty = cgroup_is_dead(child);
		if (!empty)
			break;
	}
	rcu_read_unlock();
	if (!empty)
		return -EBUSY;

4417
	/*
T
Tejun Heo 已提交
4418 4419 4420
	 * Initiate massacre of all css's.  cgroup_destroy_css_killed()
	 * will be invoked to perform the rest of destruction once the
	 * percpu refs of all css's are confirmed to be killed.
4421
	 */
T
Tejun Heo 已提交
4422 4423
	for_each_css(css, ssid, cgrp)
		kill_css(css);
4424 4425 4426 4427

	/*
	 * Mark @cgrp dead.  This prevents further task migration and child
	 * creation by disabling cgroup_lock_live_group().  Note that
4428
	 * CGRP_DEAD assertion is depended upon by css_next_child() to
4429
	 * resume iteration after dropping RCU read lock.  See
4430
	 * css_next_child() for details.
4431
	 */
4432
	set_bit(CGRP_DEAD, &cgrp->flags);
4433

4434 4435 4436 4437 4438 4439 4440
	/* CGRP_DEAD is set, remove from ->release_list for the last time */
	raw_spin_lock(&release_list_lock);
	if (!list_empty(&cgrp->release_list))
		list_del_init(&cgrp->release_list);
	raw_spin_unlock(&release_list_lock);

	/*
4441 4442 4443 4444 4445 4446 4447 4448
	 * If @cgrp has css's attached, the second stage of cgroup
	 * destruction is kicked off from css_killed_work_fn() after the
	 * refs of all attached css's are killed.  If @cgrp doesn't have
	 * any css, we kick it off here.
	 */
	if (!cgrp->nr_css)
		cgroup_destroy_css_killed(cgrp);

4449
	/*
4450 4451 4452
	 * Clear the base files and remove @cgrp directory.  The removal
	 * puts the base ref but we aren't quite done with @cgrp yet, so
	 * hold onto it.
4453
	 */
4454
	cgroup_addrm_files(cgrp, cgroup_base_files, false);
4455 4456 4457
	dget(d);
	cgroup_d_remove_dir(d);

4458 4459 4460
	return 0;
};

4461
/**
4462
 * cgroup_destroy_css_killed - the second step of cgroup destruction
4463 4464 4465
 * @work: cgroup->destroy_free_work
 *
 * This function is invoked from a work item for a cgroup which is being
4466 4467 4468
 * destroyed after all css's are offlined and performs the rest of
 * destruction.  This is the second step of destruction described in the
 * comment above cgroup_destroy_locked().
4469
 */
4470
static void cgroup_destroy_css_killed(struct cgroup *cgrp)
4471 4472 4473 4474
{
	struct cgroup *parent = cgrp->parent;
	struct dentry *d = cgrp->dentry;

4475
	lockdep_assert_held(&cgroup_mutex);
4476

4477
	/* delete this cgroup from parent->children */
4478
	list_del_rcu(&cgrp->sibling);
4479 4480

	/*
4481 4482 4483
	 * We should remove the cgroup object from idr before its grace
	 * period starts, so we won't be looking up a cgroup while the
	 * cgroup is being freed.
4484
	 */
4485 4486
	idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
	cgrp->id = -1;
4487

4488 4489
	dput(d);

4490
	set_bit(CGRP_RELEASABLE, &parent->flags);
4491
	check_for_release(parent);
4492 4493
}

4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504
static int cgroup_rmdir(struct inode *unused_dir, struct dentry *dentry)
{
	int ret;

	mutex_lock(&cgroup_mutex);
	ret = cgroup_destroy_locked(dentry->d_fsdata);
	mutex_unlock(&cgroup_mutex);

	return ret;
}

4505 4506 4507 4508 4509 4510 4511 4512 4513
static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
{
	INIT_LIST_HEAD(&ss->cftsets);

	/*
	 * base_cftset is embedded in subsys itself, no need to worry about
	 * deregistration.
	 */
	if (ss->base_cftypes) {
4514 4515 4516 4517 4518
		struct cftype *cft;

		for (cft = ss->base_cftypes; cft->name[0] != '\0'; cft++)
			cft->ss = ss;

4519 4520 4521 4522 4523
		ss->base_cftset.cfts = ss->base_cftypes;
		list_add_tail(&ss->base_cftset.node, &ss->cftsets);
	}
}

4524
static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
4525 4526
{
	struct cgroup_subsys_state *css;
D
Diego Calleja 已提交
4527 4528

	printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4529

4530 4531
	mutex_lock(&cgroup_mutex);

4532 4533 4534
	/* init base cftset */
	cgroup_init_cftsets(ss);

4535
	/* Create the top cgroup state for this subsystem */
4536
	ss->root = &cgroup_dummy_root;
4537
	css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
4538 4539
	/* We don't handle early failures gracefully */
	BUG_ON(IS_ERR(css));
4540
	init_css(css, ss, cgroup_dummy_top);
4541

L
Li Zefan 已提交
4542
	/* Update the init_css_set to contain a subsys
4543
	 * pointer to this state - since the subsystem is
L
Li Zefan 已提交
4544 4545
	 * newly registered, all tasks and hence the
	 * init_css_set is in the subsystem's top cgroup. */
4546
	init_css_set.subsys[ss->subsys_id] = css;
4547 4548 4549

	need_forkexit_callback |= ss->fork || ss->exit;

L
Li Zefan 已提交
4550 4551 4552 4553 4554
	/* At system boot, before all subsystems have been
	 * registered, no tasks have been forked, so we don't
	 * need to invoke fork callbacks here. */
	BUG_ON(!list_empty(&init_task.tasks));

4555
	BUG_ON(online_css(css));
4556

4557 4558
	mutex_unlock(&cgroup_mutex);

4559 4560 4561 4562 4563 4564 4565 4566 4567 4568
	/* this function shouldn't be used with modular subsystems, since they
	 * need to register a subsys_id, among other things */
	BUG_ON(ss->module);
}

/**
 * cgroup_load_subsys: load and register a modular subsystem at runtime
 * @ss: the subsystem to load
 *
 * This function should be called in a modular subsystem's initcall. If the
T
Thomas Weber 已提交
4569
 * subsystem is built as a module, it will be assigned a new subsys_id and set
4570 4571 4572 4573 4574 4575
 * up for use. If the subsystem is built-in anyway, work is delegated to the
 * simpler cgroup_init_subsys.
 */
int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
{
	struct cgroup_subsys_state *css;
4576
	int i, ret;
4577
	struct hlist_node *tmp;
4578
	struct css_set *cset;
4579
	unsigned long key;
4580 4581 4582

	/* check name and function validity */
	if (ss->name == NULL || strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN ||
4583
	    ss->css_alloc == NULL || ss->css_free == NULL)
4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599
		return -EINVAL;

	/*
	 * we don't support callbacks in modular subsystems. this check is
	 * before the ss->module check for consistency; a subsystem that could
	 * be a module should still have no callbacks even if the user isn't
	 * compiling it as one.
	 */
	if (ss->fork || ss->exit)
		return -EINVAL;

	/*
	 * an optionally modular subsystem is built-in: we want to do nothing,
	 * since cgroup_init_subsys will have already taken care of it.
	 */
	if (ss->module == NULL) {
4600
		/* a sanity check */
4601
		BUG_ON(cgroup_subsys[ss->subsys_id] != ss);
4602 4603 4604
		return 0;
	}

4605 4606 4607
	/* init base cftset */
	cgroup_init_cftsets(ss);

4608
	mutex_lock(&cgroup_mutex);
4609
	mutex_lock(&cgroup_root_mutex);
4610
	cgroup_subsys[ss->subsys_id] = ss;
4611 4612

	/*
4613
	 * no ss->css_alloc seems to need anything important in the ss
4614
	 * struct, so this can happen first (i.e. before the dummy root
4615
	 * attachment).
4616
	 */
4617
	css = ss->css_alloc(cgroup_css(cgroup_dummy_top, ss));
4618
	if (IS_ERR(css)) {
4619 4620
		/* failure case - need to deassign the cgroup_subsys[] slot. */
		cgroup_subsys[ss->subsys_id] = NULL;
4621
		mutex_unlock(&cgroup_root_mutex);
4622 4623 4624 4625
		mutex_unlock(&cgroup_mutex);
		return PTR_ERR(css);
	}

4626
	ss->root = &cgroup_dummy_root;
4627 4628

	/* our new subsystem will be attached to the dummy hierarchy. */
4629
	init_css(css, ss, cgroup_dummy_top);
4630 4631 4632 4633 4634 4635 4636 4637 4638 4639

	/*
	 * Now we need to entangle the css into the existing css_sets. unlike
	 * in cgroup_init_subsys, there are now multiple css_sets, so each one
	 * will need a new pointer to it; done by iterating the css_set_table.
	 * furthermore, modifying the existing css_sets will corrupt the hash
	 * table state, so each changed css_set will need its hash recomputed.
	 * this is all done under the css_set_lock.
	 */
	write_lock(&css_set_lock);
4640
	hash_for_each_safe(css_set_table, i, tmp, cset, hlist) {
4641
		/* skip entries that we already rehashed */
4642
		if (cset->subsys[ss->subsys_id])
4643 4644
			continue;
		/* remove existing entry */
4645
		hash_del(&cset->hlist);
4646
		/* set new value */
4647
		cset->subsys[ss->subsys_id] = css;
4648
		/* recompute hash and restore entry */
4649 4650
		key = css_set_hash(cset->subsys);
		hash_add(css_set_table, &cset->hlist, key);
4651 4652 4653
	}
	write_unlock(&css_set_lock);

4654
	ret = online_css(css);
4655 4656
	if (ret) {
		ss->css_free(css);
T
Tejun Heo 已提交
4657
		goto err_unload;
4658
	}
4659

4660
	/* success! */
4661
	mutex_unlock(&cgroup_root_mutex);
4662 4663
	mutex_unlock(&cgroup_mutex);
	return 0;
4664 4665

err_unload:
4666
	mutex_unlock(&cgroup_root_mutex);
4667 4668 4669 4670
	mutex_unlock(&cgroup_mutex);
	/* @ss can't be mounted here as try_module_get() would fail */
	cgroup_unload_subsys(ss);
	return ret;
4671
}
4672
EXPORT_SYMBOL_GPL(cgroup_load_subsys);
4673

B
Ben Blum 已提交
4674 4675 4676 4677 4678 4679 4680 4681 4682 4683
/**
 * cgroup_unload_subsys: unload a modular subsystem
 * @ss: the subsystem to unload
 *
 * This function should be called in a modular subsystem's exitcall. When this
 * function is invoked, the refcount on the subsystem's module will be 0, so
 * the subsystem will not be attached to any hierarchy.
 */
void cgroup_unload_subsys(struct cgroup_subsys *ss)
{
4684
	struct cgrp_cset_link *link;
4685
	struct cgroup_subsys_state *css;
B
Ben Blum 已提交
4686 4687 4688 4689 4690

	BUG_ON(ss->module == NULL);

	/*
	 * we shouldn't be called if the subsystem is in use, and the use of
4691
	 * try_module_get() in rebind_subsystems() should ensure that it
B
Ben Blum 已提交
4692 4693
	 * doesn't start being used while we're killing it off.
	 */
4694
	BUG_ON(ss->root != &cgroup_dummy_root);
B
Ben Blum 已提交
4695 4696

	mutex_lock(&cgroup_mutex);
4697
	mutex_lock(&cgroup_root_mutex);
4698

4699 4700 4701
	css = cgroup_css(cgroup_dummy_top, ss);
	if (css)
		offline_css(css);
4702

B
Ben Blum 已提交
4703
	/* deassign the subsys_id */
4704
	cgroup_subsys[ss->subsys_id] = NULL;
B
Ben Blum 已提交
4705 4706

	/*
4707 4708 4709
	 * disentangle the css from all css_sets attached to the dummy
	 * top. as in loading, we need to pay our respects to the hashtable
	 * gods.
B
Ben Blum 已提交
4710 4711
	 */
	write_lock(&css_set_lock);
4712
	list_for_each_entry(link, &cgroup_dummy_top->cset_links, cset_link) {
4713
		struct css_set *cset = link->cset;
4714
		unsigned long key;
B
Ben Blum 已提交
4715

4716 4717 4718 4719
		hash_del(&cset->hlist);
		cset->subsys[ss->subsys_id] = NULL;
		key = css_set_hash(cset->subsys);
		hash_add(css_set_table, &cset->hlist, key);
B
Ben Blum 已提交
4720 4721 4722 4723
	}
	write_unlock(&css_set_lock);

	/*
4724 4725
	 * remove subsystem's css from the cgroup_dummy_top and free it -
	 * need to free before marking as null because ss->css_free needs
L
Li Zefan 已提交
4726
	 * the cgrp->subsys pointer to find their state.
B
Ben Blum 已提交
4727
	 */
4728 4729
	if (css)
		ss->css_free(css);
4730
	RCU_INIT_POINTER(cgroup_dummy_top->subsys[ss->subsys_id], NULL);
B
Ben Blum 已提交
4731

4732
	mutex_unlock(&cgroup_root_mutex);
B
Ben Blum 已提交
4733 4734 4735 4736
	mutex_unlock(&cgroup_mutex);
}
EXPORT_SYMBOL_GPL(cgroup_unload_subsys);

4737
/**
L
Li Zefan 已提交
4738 4739 4740 4741
 * cgroup_init_early - cgroup initialization at system boot
 *
 * Initialize cgroups at system boot, and initialize any
 * subsystems that request early init.
4742 4743 4744
 */
int __init cgroup_init_early(void)
{
4745
	struct cgroup_subsys *ss;
4746
	int i;
4747

4748
	atomic_set(&init_css_set.refcount, 1);
4749
	INIT_LIST_HEAD(&init_css_set.cgrp_links);
4750
	INIT_LIST_HEAD(&init_css_set.tasks);
4751
	INIT_HLIST_NODE(&init_css_set.hlist);
4752
	css_set_count = 1;
4753 4754
	init_cgroup_root(&cgroup_dummy_root);
	cgroup_root_count = 1;
4755
	RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
4756

4757
	init_cgrp_cset_link.cset = &init_css_set;
4758 4759
	init_cgrp_cset_link.cgrp = cgroup_dummy_top;
	list_add(&init_cgrp_cset_link.cset_link, &cgroup_dummy_top->cset_links);
4760
	list_add(&init_cgrp_cset_link.cgrp_link, &init_css_set.cgrp_links);
4761

4762 4763
	/* at bootup time, we don't worry about modular subsystems */
	for_each_builtin_subsys(ss, i) {
4764 4765
		BUG_ON(!ss->name);
		BUG_ON(strlen(ss->name) > MAX_CGROUP_TYPE_NAMELEN);
4766 4767
		BUG_ON(!ss->css_alloc);
		BUG_ON(!ss->css_free);
4768
		if (ss->subsys_id != i) {
D
Diego Calleja 已提交
4769
			printk(KERN_ERR "cgroup: Subsys %s id == %d\n",
4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780
			       ss->name, ss->subsys_id);
			BUG();
		}

		if (ss->early_init)
			cgroup_init_subsys(ss);
	}
	return 0;
}

/**
L
Li Zefan 已提交
4781 4782 4783 4784
 * cgroup_init - cgroup initialization
 *
 * Register cgroup filesystem and /proc file, and initialize
 * any subsystems that didn't request early init.
4785 4786 4787
 */
int __init cgroup_init(void)
{
4788
	struct cgroup_subsys *ss;
4789
	unsigned long key;
4790
	int i, err;
4791 4792 4793 4794

	err = bdi_init(&cgroup_backing_dev_info);
	if (err)
		return err;
4795

4796
	for_each_builtin_subsys(ss, i) {
4797 4798 4799 4800
		if (!ss->early_init)
			cgroup_init_subsys(ss);
	}

4801
	/* allocate id for the dummy hierarchy */
T
Tejun Heo 已提交
4802 4803 4804
	mutex_lock(&cgroup_mutex);
	mutex_lock(&cgroup_root_mutex);

4805 4806 4807 4808
	/* Add init_css_set to the hash table */
	key = css_set_hash(init_css_set.subsys);
	hash_add(css_set_table, &init_css_set.hlist, key);

4809
	BUG_ON(cgroup_init_root_id(&cgroup_dummy_root, 0, 1));
4810

4811 4812 4813 4814
	err = idr_alloc(&cgroup_dummy_root.cgroup_idr, cgroup_dummy_top,
			0, 1, GFP_KERNEL);
	BUG_ON(err < 0);

T
Tejun Heo 已提交
4815 4816 4817
	mutex_unlock(&cgroup_root_mutex);
	mutex_unlock(&cgroup_mutex);

4818 4819 4820 4821 4822 4823
	cgroup_kobj = kobject_create_and_add("cgroup", fs_kobj);
	if (!cgroup_kobj) {
		err = -ENOMEM;
		goto out;
	}

4824
	err = register_filesystem(&cgroup_fs_type);
4825 4826
	if (err < 0) {
		kobject_put(cgroup_kobj);
4827
		goto out;
4828
	}
4829

L
Li Zefan 已提交
4830
	proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations);
4831

4832
out:
4833 4834 4835
	if (err)
		bdi_destroy(&cgroup_backing_dev_info);

4836 4837
	return err;
}
4838

4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850
static int __init cgroup_wq_init(void)
{
	/*
	 * There isn't much point in executing destruction path in
	 * parallel.  Good chunk is serialized with cgroup_mutex anyway.
	 * Use 1 for @max_active.
	 *
	 * We would prefer to do this in cgroup_init() above, but that
	 * is called before init_workqueues(): so leave this until after.
	 */
	cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
	BUG_ON(!cgroup_destroy_wq);
4851 4852 4853 4854 4855 4856 4857 4858 4859

	/*
	 * Used to destroy pidlists and separate to serve as flush domain.
	 * Cap @max_active to 1 too.
	 */
	cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
						    0, 1);
	BUG_ON(!cgroup_pidlist_destroy_wq);

4860 4861 4862 4863
	return 0;
}
core_initcall(cgroup_wq_init);

4864 4865 4866 4867 4868 4869
/*
 * proc_cgroup_show()
 *  - Print task's cgroup paths into seq_file, one line for each hierarchy
 *  - Used for /proc/<pid>/cgroup.
 *  - No need to task_lock(tsk) on this tsk->cgroup reference, as it
 *    doesn't really matter if tsk->cgroup changes after we read it,
4870
 *    and we take cgroup_mutex, keeping cgroup_attach_task() from changing it
4871 4872 4873 4874 4875 4876
 *    anyway.  No need to check that tsk->cgroup != NULL, thanks to
 *    the_top_cgroup_hack in cgroup_exit(), which sets an exiting tasks
 *    cgroup to top_cgroup.
 */

/* TODO: Use a proper seq_file iterator */
4877
int proc_cgroup_show(struct seq_file *m, void *v)
4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899
{
	struct pid *pid;
	struct task_struct *tsk;
	char *buf;
	int retval;
	struct cgroupfs_root *root;

	retval = -ENOMEM;
	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		goto out;

	retval = -ESRCH;
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
	if (!tsk)
		goto out_free;

	retval = 0;

	mutex_lock(&cgroup_mutex);

4900
	for_each_active_root(root) {
4901
		struct cgroup_subsys *ss;
4902
		struct cgroup *cgrp;
T
Tejun Heo 已提交
4903
		int ssid, count = 0;
4904

4905
		seq_printf(m, "%d:", root->hierarchy_id);
T
Tejun Heo 已提交
4906 4907 4908
		for_each_subsys(ss, ssid)
			if (root->subsys_mask & (1 << ssid))
				seq_printf(m, "%s%s", count++ ? "," : "", ss->name);
4909 4910 4911
		if (strlen(root->name))
			seq_printf(m, "%sname=%s", count ? "," : "",
				   root->name);
4912
		seq_putc(m, ':');
4913
		cgrp = task_cgroup_from_root(tsk, root);
4914
		retval = cgroup_path(cgrp, buf, PAGE_SIZE);
4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932
		if (retval < 0)
			goto out_unlock;
		seq_puts(m, buf);
		seq_putc(m, '\n');
	}

out_unlock:
	mutex_unlock(&cgroup_mutex);
	put_task_struct(tsk);
out_free:
	kfree(buf);
out:
	return retval;
}

/* Display information about each subsystem and each hierarchy */
static int proc_cgroupstats_show(struct seq_file *m, void *v)
{
4933
	struct cgroup_subsys *ss;
4934 4935
	int i;

4936
	seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
B
Ben Blum 已提交
4937 4938 4939 4940 4941
	/*
	 * ideally we don't want subsystems moving around while we do this.
	 * cgroup_mutex is also necessary to guarantee an atomic snapshot of
	 * subsys/hierarchy state.
	 */
4942
	mutex_lock(&cgroup_mutex);
4943 4944

	for_each_subsys(ss, i)
4945 4946
		seq_printf(m, "%s\t%d\t%d\t%d\n",
			   ss->name, ss->root->hierarchy_id,
4947
			   ss->root->number_of_cgroups, !ss->disabled);
4948

4949 4950 4951 4952 4953 4954
	mutex_unlock(&cgroup_mutex);
	return 0;
}

static int cgroupstats_open(struct inode *inode, struct file *file)
{
A
Al Viro 已提交
4955
	return single_open(file, proc_cgroupstats_show, NULL);
4956 4957
}

4958
static const struct file_operations proc_cgroupstats_operations = {
4959 4960 4961 4962 4963 4964
	.open = cgroupstats_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

4965 4966
/**
 * cgroup_fork - attach newly forked task to its parents cgroup.
L
Li Zefan 已提交
4967
 * @child: pointer to task_struct of forking parent process.
4968 4969 4970 4971 4972
 *
 * Description: A task inherits its parent's cgroup at fork().
 *
 * A pointer to the shared css_set was automatically copied in
 * fork.c by dup_task_struct().  However, we ignore that copy, since
4973 4974 4975 4976
 * it was not made under the protection of RCU or cgroup_mutex, so
 * might no longer be a valid cgroup pointer.  cgroup_attach_task() might
 * have already changed current->cgroups, allowing the previously
 * referenced cgroup group to be removed and freed.
4977 4978 4979 4980 4981 4982
 *
 * At the point that cgroup_fork() is called, 'current' is the parent
 * task, and the passed argument 'child' points to the child task.
 */
void cgroup_fork(struct task_struct *child)
{
4983
	task_lock(current);
4984
	get_css_set(task_css_set(current));
4985
	child->cgroups = current->cgroups;
4986
	task_unlock(current);
4987
	INIT_LIST_HEAD(&child->cg_list);
4988 4989
}

4990
/**
L
Li Zefan 已提交
4991 4992 4993
 * cgroup_post_fork - called on a new task after adding it to the task list
 * @child: the task in question
 *
4994 4995 4996
 * Adds the task to the list running through its css_set if necessary and
 * call the subsystem fork() callbacks.  Has to be after the task is
 * visible on the task list in case we race with the first call to
4997
 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
4998
 * list.
L
Li Zefan 已提交
4999
 */
5000 5001
void cgroup_post_fork(struct task_struct *child)
{
5002
	struct cgroup_subsys *ss;
5003 5004
	int i;

5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015
	/*
	 * use_task_css_set_links is set to 1 before we walk the tasklist
	 * under the tasklist_lock and we read it here after we added the child
	 * to the tasklist under the tasklist_lock as well. If the child wasn't
	 * yet in the tasklist when we walked through it from
	 * cgroup_enable_task_cg_lists(), then use_task_css_set_links value
	 * should be visible now due to the paired locking and barriers implied
	 * by LOCK/UNLOCK: it is written before the tasklist_lock unlock
	 * in cgroup_enable_task_cg_lists() and read here after the tasklist_lock
	 * lock on fork.
	 */
5016 5017
	if (use_task_css_set_links) {
		write_lock(&css_set_lock);
5018 5019
		task_lock(child);
		if (list_empty(&child->cg_list))
5020
			list_add(&child->cg_list, &task_css_set(child)->tasks);
5021
		task_unlock(child);
5022 5023
		write_unlock(&css_set_lock);
	}
5024 5025 5026 5027 5028 5029 5030

	/*
	 * Call ss->fork().  This must happen after @child is linked on
	 * css_set; otherwise, @child might change state between ->fork()
	 * and addition to css_set.
	 */
	if (need_forkexit_callback) {
5031 5032 5033 5034 5035 5036 5037 5038
		/*
		 * fork/exit callbacks are supported only for builtin
		 * subsystems, and the builtin section of the subsys
		 * array is immutable, so we don't need to lock the
		 * subsys array here. On the other hand, modular section
		 * of the array can be freed at module unload, so we
		 * can't touch that.
		 */
5039
		for_each_builtin_subsys(ss, i)
5040 5041 5042
			if (ss->fork)
				ss->fork(child);
	}
5043
}
5044

5045 5046 5047
/**
 * cgroup_exit - detach cgroup from exiting task
 * @tsk: pointer to task_struct of exiting process
L
Li Zefan 已提交
5048
 * @run_callback: run exit callbacks?
5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076
 *
 * Description: Detach cgroup from @tsk and release it.
 *
 * Note that cgroups marked notify_on_release force every task in
 * them to take the global cgroup_mutex mutex when exiting.
 * This could impact scaling on very large systems.  Be reluctant to
 * use notify_on_release cgroups where very high task exit scaling
 * is required on large systems.
 *
 * the_top_cgroup_hack:
 *
 *    Set the exiting tasks cgroup to the root cgroup (top_cgroup).
 *
 *    We call cgroup_exit() while the task is still competent to
 *    handle notify_on_release(), then leave the task attached to the
 *    root cgroup in each hierarchy for the remainder of its exit.
 *
 *    To do this properly, we would increment the reference count on
 *    top_cgroup, and near the very end of the kernel/exit.c do_exit()
 *    code we would add a second cgroup function call, to drop that
 *    reference.  This would just create an unnecessary hot spot on
 *    the top_cgroup reference count, to no avail.
 *
 *    Normally, holding a reference to a cgroup without bumping its
 *    count is unsafe.   The cgroup could go away, or someone could
 *    attach us to a different cgroup, decrementing the count on
 *    the first cgroup that we never incremented.  But in this case,
 *    top_cgroup isn't going away, and either task has PF_EXITING set,
5077 5078
 *    which wards off any cgroup_attach_task() attempts, or task is a failed
 *    fork, never visible to cgroup_attach_task.
5079 5080 5081
 */
void cgroup_exit(struct task_struct *tsk, int run_callbacks)
{
5082
	struct cgroup_subsys *ss;
5083
	struct css_set *cset;
5084
	int i;
5085 5086 5087 5088 5089 5090 5091 5092 5093

	/*
	 * Unlink from the css_set task list if necessary.
	 * Optimistically check cg_list before taking
	 * css_set_lock
	 */
	if (!list_empty(&tsk->cg_list)) {
		write_lock(&css_set_lock);
		if (!list_empty(&tsk->cg_list))
5094
			list_del_init(&tsk->cg_list);
5095 5096 5097
		write_unlock(&css_set_lock);
	}

5098 5099
	/* Reassign the task to the init_css_set. */
	task_lock(tsk);
5100 5101
	cset = task_css_set(tsk);
	RCU_INIT_POINTER(tsk->cgroups, &init_css_set);
5102 5103

	if (run_callbacks && need_forkexit_callback) {
5104 5105 5106 5107
		/*
		 * fork/exit callbacks are supported only for builtin
		 * subsystems, see cgroup_post_fork() for details.
		 */
5108
		for_each_builtin_subsys(ss, i) {
5109
			if (ss->exit) {
5110 5111
				struct cgroup_subsys_state *old_css = cset->subsys[i];
				struct cgroup_subsys_state *css = task_css(tsk, i);
5112

5113
				ss->exit(css, old_css, tsk);
5114 5115 5116
			}
		}
	}
5117
	task_unlock(tsk);
5118

5119
	put_css_set_taskexit(cset);
5120
}
5121

5122
static void check_for_release(struct cgroup *cgrp)
5123
{
5124
	if (cgroup_is_releasable(cgrp) &&
T
Tejun Heo 已提交
5125
	    list_empty(&cgrp->cset_links) && list_empty(&cgrp->children)) {
5126 5127
		/*
		 * Control Group is currently removeable. If it's not
5128
		 * already queued for a userspace notification, queue
5129 5130
		 * it now
		 */
5131
		int need_schedule_work = 0;
5132

5133
		raw_spin_lock(&release_list_lock);
5134
		if (!cgroup_is_dead(cgrp) &&
5135 5136
		    list_empty(&cgrp->release_list)) {
			list_add(&cgrp->release_list, &release_list);
5137 5138
			need_schedule_work = 1;
		}
5139
		raw_spin_unlock(&release_list_lock);
5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
		if (need_schedule_work)
			schedule_work(&release_agent_work);
	}
}

/*
 * Notify userspace when a cgroup is released, by running the
 * configured release agent with the name of the cgroup (path
 * relative to the root of cgroup file system) as the argument.
 *
 * Most likely, this user command will try to rmdir this cgroup.
 *
 * This races with the possibility that some other task will be
 * attached to this cgroup before it is removed, or that some other
 * user task will 'mkdir' a child cgroup of this cgroup.  That's ok.
 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
 * unused, and this cgroup will be reprieved from its death sentence,
 * to continue to serve a useful existence.  Next time it's released,
 * we will get notified again, if it still has 'notify_on_release' set.
 *
 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
 * means only wait until the task is successfully execve()'d.  The
 * separate release agent task is forked by call_usermodehelper(),
 * then control in this thread returns here, without waiting for the
 * release agent task.  We don't bother to wait because the caller of
 * this routine has no use for the exit status of the release agent
 * task, so no sense holding our caller up for that.
 */
static void cgroup_release_agent(struct work_struct *work)
{
	BUG_ON(work != &release_agent_work);
	mutex_lock(&cgroup_mutex);
5172
	raw_spin_lock(&release_list_lock);
5173 5174 5175
	while (!list_empty(&release_list)) {
		char *argv[3], *envp[3];
		int i;
5176
		char *pathbuf = NULL, *agentbuf = NULL;
5177
		struct cgroup *cgrp = list_entry(release_list.next,
5178 5179
						    struct cgroup,
						    release_list);
5180
		list_del_init(&cgrp->release_list);
5181
		raw_spin_unlock(&release_list_lock);
5182
		pathbuf = kmalloc(PAGE_SIZE, GFP_KERNEL);
5183 5184 5185 5186 5187 5188 5189
		if (!pathbuf)
			goto continue_free;
		if (cgroup_path(cgrp, pathbuf, PAGE_SIZE) < 0)
			goto continue_free;
		agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
		if (!agentbuf)
			goto continue_free;
5190 5191

		i = 0;
5192 5193
		argv[i++] = agentbuf;
		argv[i++] = pathbuf;
5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207
		argv[i] = NULL;

		i = 0;
		/* minimal command environment */
		envp[i++] = "HOME=/";
		envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
		envp[i] = NULL;

		/* Drop the lock while we invoke the usermode helper,
		 * since the exec could involve hitting disk and hence
		 * be a slow process */
		mutex_unlock(&cgroup_mutex);
		call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
		mutex_lock(&cgroup_mutex);
5208 5209 5210
 continue_free:
		kfree(pathbuf);
		kfree(agentbuf);
5211
		raw_spin_lock(&release_list_lock);
5212
	}
5213
	raw_spin_unlock(&release_list_lock);
5214 5215
	mutex_unlock(&cgroup_mutex);
}
5216 5217 5218

static int __init cgroup_disable(char *str)
{
5219
	struct cgroup_subsys *ss;
5220
	char *token;
5221
	int i;
5222 5223 5224 5225

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;
5226

5227 5228 5229 5230 5231
		/*
		 * cgroup_disable, being at boot time, can't know about
		 * module subsystems, so we don't worry about them.
		 */
		for_each_builtin_subsys(ss, i) {
5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242
			if (!strcmp(token, ss->name)) {
				ss->disabled = 1;
				printk(KERN_INFO "Disabling %s control group"
					" subsystem\n", ss->name);
				break;
			}
		}
	}
	return 1;
}
__setup("cgroup_disable=", cgroup_disable);
K
KAMEZAWA Hiroyuki 已提交
5243

5244
/**
5245 5246 5247
 * css_from_dir - get corresponding css from the dentry of a cgroup dir
 * @dentry: directory dentry of interest
 * @ss: subsystem of interest
5248
 *
5249 5250 5251
 * Must be called under cgroup_mutex or RCU read lock.  The caller is
 * responsible for pinning the returned css if it needs to be accessed
 * outside the critical section.
S
Stephane Eranian 已提交
5252
 */
5253 5254
struct cgroup_subsys_state *css_from_dir(struct dentry *dentry,
					 struct cgroup_subsys *ss)
S
Stephane Eranian 已提交
5255 5256 5257
{
	struct cgroup *cgrp;

5258
	cgroup_assert_mutex_or_rcu_locked();
5259

5260 5261 5262
	/* is @dentry a cgroup dir? */
	if (!dentry->d_inode ||
	    dentry->d_inode->i_op != &cgroup_dir_inode_operations)
S
Stephane Eranian 已提交
5263 5264
		return ERR_PTR(-EBADF);

5265
	cgrp = __d_cgrp(dentry);
5266
	return cgroup_css(cgrp, ss) ?: ERR_PTR(-ENOENT);
S
Stephane Eranian 已提交
5267 5268
}

5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280
/**
 * css_from_id - lookup css by id
 * @id: the cgroup id
 * @ss: cgroup subsys to be looked into
 *
 * Returns the css if there's valid one with @id, otherwise returns NULL.
 * Should be called under rcu_read_lock().
 */
struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
{
	struct cgroup *cgrp;

5281
	cgroup_assert_mutex_or_rcu_locked();
5282 5283 5284

	cgrp = idr_find(&ss->root->cgroup_idr, id);
	if (cgrp)
5285
		return cgroup_css(cgrp, ss);
5286
	return NULL;
S
Stephane Eranian 已提交
5287 5288
}

5289
#ifdef CONFIG_CGROUP_DEBUG
5290 5291
static struct cgroup_subsys_state *
debug_css_alloc(struct cgroup_subsys_state *parent_css)
5292 5293 5294 5295 5296 5297 5298 5299 5300
{
	struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);

	if (!css)
		return ERR_PTR(-ENOMEM);

	return css;
}

5301
static void debug_css_free(struct cgroup_subsys_state *css)
5302
{
5303
	kfree(css);
5304 5305
}

5306 5307
static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5308
{
5309
	return cgroup_task_count(css->cgroup);
5310 5311
}

5312 5313
static u64 current_css_set_read(struct cgroup_subsys_state *css,
				struct cftype *cft)
5314 5315 5316 5317
{
	return (u64)(unsigned long)current->cgroups;
}

5318
static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
L
Li Zefan 已提交
5319
					 struct cftype *cft)
5320 5321 5322 5323
{
	u64 count;

	rcu_read_lock();
5324
	count = atomic_read(&task_css_set(current)->refcount);
5325 5326 5327 5328
	rcu_read_unlock();
	return count;
}

5329
static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
5330
{
5331
	struct cgrp_cset_link *link;
5332
	struct css_set *cset;
5333 5334 5335

	read_lock(&css_set_lock);
	rcu_read_lock();
5336
	cset = rcu_dereference(current->cgroups);
5337
	list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
5338 5339 5340 5341 5342 5343 5344
		struct cgroup *c = link->cgrp;
		const char *name;

		if (c->dentry)
			name = c->dentry->d_name.name;
		else
			name = "?";
5345 5346
		seq_printf(seq, "Root %d group %s\n",
			   c->root->hierarchy_id, name);
5347 5348 5349 5350 5351 5352 5353
	}
	rcu_read_unlock();
	read_unlock(&css_set_lock);
	return 0;
}

#define MAX_TASKS_SHOWN_PER_CSS 25
5354
static int cgroup_css_links_read(struct seq_file *seq, void *v)
5355
{
5356
	struct cgroup_subsys_state *css = seq_css(seq);
5357
	struct cgrp_cset_link *link;
5358 5359

	read_lock(&css_set_lock);
5360
	list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
5361
		struct css_set *cset = link->cset;
5362 5363
		struct task_struct *task;
		int count = 0;
5364 5365
		seq_printf(seq, "css_set %p\n", cset);
		list_for_each_entry(task, &cset->tasks, cg_list) {
5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378
			if (count++ > MAX_TASKS_SHOWN_PER_CSS) {
				seq_puts(seq, "  ...\n");
				break;
			} else {
				seq_printf(seq, "  task %d\n",
					   task_pid_vnr(task));
			}
		}
	}
	read_unlock(&css_set_lock);
	return 0;
}

5379
static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
5380
{
5381
	return test_bit(CGRP_RELEASABLE, &css->cgroup->flags);
5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399
}

static struct cftype debug_files[] =  {
	{
		.name = "taskcount",
		.read_u64 = debug_taskcount_read,
	},

	{
		.name = "current_css_set",
		.read_u64 = current_css_set_read,
	},

	{
		.name = "current_css_set_refcount",
		.read_u64 = current_css_set_refcount_read,
	},

5400 5401
	{
		.name = "current_css_set_cg_links",
5402
		.seq_show = current_css_set_cg_links_read,
5403 5404 5405 5406
	},

	{
		.name = "cgroup_css_links",
5407
		.seq_show = cgroup_css_links_read,
5408 5409
	},

5410 5411 5412 5413 5414
	{
		.name = "releasable",
		.read_u64 = releasable_read,
	},

5415 5416
	{ }	/* terminate */
};
5417 5418 5419

struct cgroup_subsys debug_subsys = {
	.name = "debug",
5420 5421
	.css_alloc = debug_css_alloc,
	.css_free = debug_css_free,
5422
	.subsys_id = debug_subsys_id,
5423
	.base_cftypes = debug_files,
5424 5425
};
#endif /* CONFIG_CGROUP_DEBUG */