cpuset.c 77.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  kernel/cpuset.c
 *
 *  Processor and Memory placement constraints for sets of tasks.
 *
 *  Copyright (C) 2003 BULL SA.
P
Paul Jackson 已提交
7
 *  Copyright (C) 2004-2007 Silicon Graphics, Inc.
8
 *  Copyright (C) 2006 Google, Inc
L
Linus Torvalds 已提交
9 10 11 12
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
13
 *  2003-10-10 Written by Simon Derr.
L
Linus Torvalds 已提交
14
 *  2003-10-22 Updates by Stephen Hemminger.
15
 *  2004 May-July Rework by Paul Jackson.
16
 *  2006 Rework by Paul Menage to use generic cgroups
17 18
 *  2008 Rework of the scheduler domains and CPU hotplug handling
 *       by Max Krasnyansky
L
Linus Torvalds 已提交
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/cpuset.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/list.h>
37
#include <linux/mempolicy.h>
L
Linus Torvalds 已提交
38
#include <linux/mm.h>
39
#include <linux/memory.h>
40
#include <linux/export.h>
L
Linus Torvalds 已提交
41 42 43 44
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/pagemap.h>
#include <linux/proc_fs.h>
45
#include <linux/rcupdate.h>
L
Linus Torvalds 已提交
46 47
#include <linux/sched.h>
#include <linux/seq_file.h>
48
#include <linux/security.h>
L
Linus Torvalds 已提交
49 50 51 52 53 54 55 56 57
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stat.h>
#include <linux/string.h>
#include <linux/time.h>
#include <linux/backing-dev.h>
#include <linux/sort.h>

#include <asm/uaccess.h>
A
Arun Sharma 已提交
58
#include <linux/atomic.h>
59
#include <linux/mutex.h>
60 61
#include <linux/workqueue.h>
#include <linux/cgroup.h>
L
Linus Torvalds 已提交
62

63 64 65 66 67
/*
 * Tracks how many cpusets are currently defined in system.
 * When there is only one cpuset (the root cpuset) we can
 * short circuit some hooks.
 */
68
int number_of_cpusets __read_mostly;
69

70
/* Forward declare cgroup structures */
71 72 73
struct cgroup_subsys cpuset_subsys;
struct cpuset;

74 75 76 77 78 79 80 81 82
/* See "Frequency meter" comments, below. */

struct fmeter {
	int cnt;		/* unprocessed events count */
	int val;		/* most recent output value */
	time_t time;		/* clock (secs) when val computed */
	spinlock_t lock;	/* guards read or write of above */
};

L
Linus Torvalds 已提交
83
struct cpuset {
84 85
	struct cgroup_subsys_state css;

L
Linus Torvalds 已提交
86
	unsigned long flags;		/* "unsigned long" so bitops work */
87
	cpumask_var_t cpus_allowed;	/* CPUs allowed to tasks in cpuset */
L
Linus Torvalds 已提交
88 89 90 91
	nodemask_t mems_allowed;	/* Memory Nodes allowed to tasks */

	struct cpuset *parent;		/* my parent */

92
	struct fmeter fmeter;		/* memory_pressure filter */
P
Paul Jackson 已提交
93

94 95 96 97 98 99
	/*
	 * Tasks are being attached to this cpuset.  Used to prevent
	 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
	 */
	int attach_in_progress;

P
Paul Jackson 已提交
100 101
	/* partition number for rebuild_sched_domains() */
	int pn;
102

103 104 105
	/* for custom sched domain */
	int relax_domain_level;

106
	/* used for walking a cpuset hierarchy */
107
	struct list_head stack_list;
108 109

	struct work_struct hotplug_work;
L
Linus Torvalds 已提交
110 111
};

112 113 114 115 116 117 118 119 120 121 122 123 124 125
/* Retrieve the cpuset for a cgroup */
static inline struct cpuset *cgroup_cs(struct cgroup *cont)
{
	return container_of(cgroup_subsys_state(cont, cpuset_subsys_id),
			    struct cpuset, css);
}

/* Retrieve the cpuset for a task */
static inline struct cpuset *task_cs(struct task_struct *task)
{
	return container_of(task_subsys_state(task, cpuset_subsys_id),
			    struct cpuset, css);
}

126 127 128 129 130 131 132 133 134 135 136 137 138
#ifdef CONFIG_NUMA
static inline bool task_has_mempolicy(struct task_struct *task)
{
	return task->mempolicy;
}
#else
static inline bool task_has_mempolicy(struct task_struct *task)
{
	return false;
}
#endif


L
Linus Torvalds 已提交
139 140
/* bits in struct cpuset flags field */
typedef enum {
T
Tejun Heo 已提交
141
	CS_ONLINE,
L
Linus Torvalds 已提交
142 143
	CS_CPU_EXCLUSIVE,
	CS_MEM_EXCLUSIVE,
144
	CS_MEM_HARDWALL,
145
	CS_MEMORY_MIGRATE,
P
Paul Jackson 已提交
146
	CS_SCHED_LOAD_BALANCE,
147 148
	CS_SPREAD_PAGE,
	CS_SPREAD_SLAB,
L
Linus Torvalds 已提交
149 150 151
} cpuset_flagbits_t;

/* convenient tests for these bits */
T
Tejun Heo 已提交
152 153 154 155 156
static inline bool is_cpuset_online(const struct cpuset *cs)
{
	return test_bit(CS_ONLINE, &cs->flags);
}

L
Linus Torvalds 已提交
157 158
static inline int is_cpu_exclusive(const struct cpuset *cs)
{
159
	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
L
Linus Torvalds 已提交
160 161 162 163
}

static inline int is_mem_exclusive(const struct cpuset *cs)
{
164
	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
L
Linus Torvalds 已提交
165 166
}

167 168 169 170 171
static inline int is_mem_hardwall(const struct cpuset *cs)
{
	return test_bit(CS_MEM_HARDWALL, &cs->flags);
}

P
Paul Jackson 已提交
172 173 174 175 176
static inline int is_sched_load_balance(const struct cpuset *cs)
{
	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
}

177 178
static inline int is_memory_migrate(const struct cpuset *cs)
{
179
	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
180 181
}

182 183 184 185 186 187 188 189 190 191
static inline int is_spread_page(const struct cpuset *cs)
{
	return test_bit(CS_SPREAD_PAGE, &cs->flags);
}

static inline int is_spread_slab(const struct cpuset *cs)
{
	return test_bit(CS_SPREAD_SLAB, &cs->flags);
}

L
Linus Torvalds 已提交
192
static struct cpuset top_cpuset = {
T
Tejun Heo 已提交
193 194
	.flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
		  (1 << CS_MEM_EXCLUSIVE)),
L
Linus Torvalds 已提交
195 196
};

197 198 199 200 201 202 203 204 205 206 207 208 209
/**
 * cpuset_for_each_child - traverse online children of a cpuset
 * @child_cs: loop cursor pointing to the current child
 * @pos_cgrp: used for iteration
 * @parent_cs: target cpuset to walk children of
 *
 * Walk @child_cs through the online children of @parent_cs.  Must be used
 * with RCU read locked.
 */
#define cpuset_for_each_child(child_cs, pos_cgrp, parent_cs)		\
	cgroup_for_each_child((pos_cgrp), (parent_cs)->css.cgroup)	\
		if (is_cpuset_online(((child_cs) = cgroup_cs((pos_cgrp)))))

L
Linus Torvalds 已提交
210
/*
211 212 213 214 215 216 217 218 219 220 221 222 223 224
 * There are two global mutexes guarding cpuset structures - cpuset_mutex
 * and callback_mutex.  The latter may nest inside the former.  We also
 * require taking task_lock() when dereferencing a task's cpuset pointer.
 * See "The task_lock() exception", at the end of this comment.
 *
 * A task must hold both mutexes to modify cpusets.  If a task holds
 * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it
 * is the only task able to also acquire callback_mutex and be able to
 * modify cpusets.  It can perform various checks on the cpuset structure
 * first, knowing nothing will change.  It can also allocate memory while
 * just holding cpuset_mutex.  While it is performing these checks, various
 * callback routines can briefly acquire callback_mutex to query cpusets.
 * Once it is ready to make the changes, it takes callback_mutex, blocking
 * everyone else.
225 226
 *
 * Calls to the kernel memory allocator can not be made while holding
227
 * callback_mutex, as that would risk double tripping on callback_mutex
228 229 230
 * from one of the callbacks into the cpuset code from within
 * __alloc_pages().
 *
231
 * If a task is only holding callback_mutex, then it has read-only
232 233
 * access to cpusets.
 *
234 235 236
 * Now, the task_struct fields mems_allowed and mempolicy may be changed
 * by other task, we use alloc_lock in the task_struct fields to protect
 * them.
237
 *
238
 * The cpuset_common_file_read() handlers only hold callback_mutex across
239 240 241
 * small pieces of code, such as when reading out possibly multi-word
 * cpumasks and nodemasks.
 *
242 243
 * Accessing a task's cpuset should be done in accordance with the
 * guidelines for accessing subsystem state in kernel/cgroup.c
L
Linus Torvalds 已提交
244 245
 */

246
static DEFINE_MUTEX(cpuset_mutex);
247
static DEFINE_MUTEX(callback_mutex);
248

249 250 251 252 253 254 255 256 257 258 259
/*
 * cpuset_buffer_lock protects both the cpuset_name and cpuset_nodelist
 * buffers.  They are statically allocated to prevent using excess stack
 * when calling cpuset_print_task_mems_allowed().
 */
#define CPUSET_NAME_LEN		(128)
#define	CPUSET_NODELIST_LEN	(256)
static char cpuset_name[CPUSET_NAME_LEN];
static char cpuset_nodelist[CPUSET_NODELIST_LEN];
static DEFINE_SPINLOCK(cpuset_buffer_lock);

260 261 262
/*
 * CPU / memory hotplug is handled asynchronously.
 */
263 264
static struct workqueue_struct *cpuset_propagate_hotplug_wq;

265
static void cpuset_hotplug_workfn(struct work_struct *work);
266
static void cpuset_propagate_hotplug_workfn(struct work_struct *work);
267
static void schedule_cpuset_propagate_hotplug(struct cpuset *cs);
268 269 270

static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);

271 272
/*
 * This is ugly, but preserves the userspace API for existing cpuset
273
 * users. If someone tries to mount the "cpuset" filesystem, we
274 275
 * silently switch it to mount "cgroup" instead
 */
A
Al Viro 已提交
276 277
static struct dentry *cpuset_mount(struct file_system_type *fs_type,
			 int flags, const char *unused_dev_name, void *data)
L
Linus Torvalds 已提交
278
{
279
	struct file_system_type *cgroup_fs = get_fs_type("cgroup");
A
Al Viro 已提交
280
	struct dentry *ret = ERR_PTR(-ENODEV);
281 282 283 284
	if (cgroup_fs) {
		char mountopts[] =
			"cpuset,noprefix,"
			"release_agent=/sbin/cpuset_release_agent";
A
Al Viro 已提交
285 286
		ret = cgroup_fs->mount(cgroup_fs, flags,
					   unused_dev_name, mountopts);
287 288 289
		put_filesystem(cgroup_fs);
	}
	return ret;
L
Linus Torvalds 已提交
290 291 292 293
}

static struct file_system_type cpuset_fs_type = {
	.name = "cpuset",
A
Al Viro 已提交
294
	.mount = cpuset_mount,
L
Linus Torvalds 已提交
295 296 297
};

/*
298
 * Return in pmask the portion of a cpusets's cpus_allowed that
L
Linus Torvalds 已提交
299 300 301
 * are online.  If none are online, walk up the cpuset hierarchy
 * until we find one that does have some online cpus.  If we get
 * all the way to the top and still haven't found any online cpus,
302 303
 * return cpu_online_mask.  Or if passed a NULL cs from an exit'ing
 * task, return cpu_online_mask.
L
Linus Torvalds 已提交
304 305
 *
 * One way or another, we guarantee to return some non-empty subset
306
 * of cpu_online_mask.
L
Linus Torvalds 已提交
307
 *
308
 * Call with callback_mutex held.
L
Linus Torvalds 已提交
309 310
 */

311 312
static void guarantee_online_cpus(const struct cpuset *cs,
				  struct cpumask *pmask)
L
Linus Torvalds 已提交
313
{
314
	while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
L
Linus Torvalds 已提交
315 316
		cs = cs->parent;
	if (cs)
317
		cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);
L
Linus Torvalds 已提交
318
	else
319 320
		cpumask_copy(pmask, cpu_online_mask);
	BUG_ON(!cpumask_intersects(pmask, cpu_online_mask));
L
Linus Torvalds 已提交
321 322 323 324
}

/*
 * Return in *pmask the portion of a cpusets's mems_allowed that
325 326 327
 * are online, with memory.  If none are online with memory, walk
 * up the cpuset hierarchy until we find one that does have some
 * online mems.  If we get all the way to the top and still haven't
328
 * found any online mems, return node_states[N_MEMORY].
L
Linus Torvalds 已提交
329 330
 *
 * One way or another, we guarantee to return some non-empty subset
331
 * of node_states[N_MEMORY].
L
Linus Torvalds 已提交
332
 *
333
 * Call with callback_mutex held.
L
Linus Torvalds 已提交
334 335 336 337
 */

static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask)
{
338
	while (cs && !nodes_intersects(cs->mems_allowed,
339
					node_states[N_MEMORY]))
L
Linus Torvalds 已提交
340 341
		cs = cs->parent;
	if (cs)
342
		nodes_and(*pmask, cs->mems_allowed,
343
					node_states[N_MEMORY]);
L
Linus Torvalds 已提交
344
	else
345 346
		*pmask = node_states[N_MEMORY];
	BUG_ON(!nodes_intersects(*pmask, node_states[N_MEMORY]));
L
Linus Torvalds 已提交
347 348
}

349 350 351
/*
 * update task's spread flag if cpuset's page/slab spread flag is set
 *
352
 * Called with callback_mutex/cpuset_mutex held
353 354 355 356 357 358 359 360 361 362 363 364 365 366
 */
static void cpuset_update_task_spread_flag(struct cpuset *cs,
					struct task_struct *tsk)
{
	if (is_spread_page(cs))
		tsk->flags |= PF_SPREAD_PAGE;
	else
		tsk->flags &= ~PF_SPREAD_PAGE;
	if (is_spread_slab(cs))
		tsk->flags |= PF_SPREAD_SLAB;
	else
		tsk->flags &= ~PF_SPREAD_SLAB;
}

L
Linus Torvalds 已提交
367 368 369 370 371
/*
 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
 *
 * One cpuset is a subset of another if all its allowed CPUs and
 * Memory Nodes are a subset of the other, and its exclusive flags
372
 * are only set if the other's are set.  Call holding cpuset_mutex.
L
Linus Torvalds 已提交
373 374 375 376
 */

static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
{
377
	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
L
Linus Torvalds 已提交
378 379 380 381 382
		nodes_subset(p->mems_allowed, q->mems_allowed) &&
		is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
		is_mem_exclusive(p) <= is_mem_exclusive(q);
}

383 384 385 386 387 388
/**
 * alloc_trial_cpuset - allocate a trial cpuset
 * @cs: the cpuset that the trial cpuset duplicates
 */
static struct cpuset *alloc_trial_cpuset(const struct cpuset *cs)
{
389 390 391 392 393 394 395 396 397 398 399 400 401
	struct cpuset *trial;

	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
	if (!trial)
		return NULL;

	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {
		kfree(trial);
		return NULL;
	}
	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);

	return trial;
402 403 404 405 406 407 408 409
}

/**
 * free_trial_cpuset - free the trial cpuset
 * @trial: the trial cpuset to be freed
 */
static void free_trial_cpuset(struct cpuset *trial)
{
410
	free_cpumask_var(trial->cpus_allowed);
411 412 413
	kfree(trial);
}

L
Linus Torvalds 已提交
414 415 416 417 418 419 420
/*
 * validate_change() - Used to validate that any proposed cpuset change
 *		       follows the structural rules for cpusets.
 *
 * If we replaced the flag and mask values of the current cpuset
 * (cur) with those values in the trial cpuset (trial), would
 * our various subset and exclusive rules still be valid?  Presumes
421
 * cpuset_mutex held.
L
Linus Torvalds 已提交
422 423 424 425 426 427 428 429 430 431 432 433 434 435
 *
 * 'cur' is the address of an actual, in-use cpuset.  Operations
 * such as list traversal that depend on the actual address of the
 * cpuset in the list must use cur below, not trial.
 *
 * 'trial' is the address of bulk structure copy of cur, with
 * perhaps one or more of the fields cpus_allowed, mems_allowed,
 * or flags changed to new, trial values.
 *
 * Return 0 if valid, -errno if not.
 */

static int validate_change(const struct cpuset *cur, const struct cpuset *trial)
{
436
	struct cgroup *cont;
L
Linus Torvalds 已提交
437
	struct cpuset *c, *par;
438 439 440
	int ret;

	rcu_read_lock();
L
Linus Torvalds 已提交
441 442

	/* Each of our child cpusets must be a subset of us */
443 444 445 446
	ret = -EBUSY;
	cpuset_for_each_child(c, cont, cur)
		if (!is_cpuset_subset(c, trial))
			goto out;
L
Linus Torvalds 已提交
447 448

	/* Remaining checks don't apply to root cpuset */
449
	ret = 0;
450
	if (cur == &top_cpuset)
451
		goto out;
L
Linus Torvalds 已提交
452

453 454
	par = cur->parent;

L
Linus Torvalds 已提交
455
	/* We must be a subset of our parent cpuset */
456
	ret = -EACCES;
L
Linus Torvalds 已提交
457
	if (!is_cpuset_subset(trial, par))
458
		goto out;
L
Linus Torvalds 已提交
459

460 461 462 463
	/*
	 * If either I or some sibling (!= me) is exclusive, we can't
	 * overlap
	 */
464 465
	ret = -EINVAL;
	cpuset_for_each_child(c, cont, par) {
L
Linus Torvalds 已提交
466 467
		if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
		    c != cur &&
468
		    cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
469
			goto out;
L
Linus Torvalds 已提交
470 471 472
		if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
		    c != cur &&
		    nodes_intersects(trial->mems_allowed, c->mems_allowed))
473
			goto out;
L
Linus Torvalds 已提交
474 475
	}

476 477 478 479
	/*
	 * Cpusets with tasks - existing or newly being attached - can't
	 * have empty cpus_allowed or mems_allowed.
	 */
480
	ret = -ENOSPC;
481
	if ((cgroup_task_count(cur->css.cgroup) || cur->attach_in_progress) &&
482 483 484
	    (cpumask_empty(trial->cpus_allowed) ||
	     nodes_empty(trial->mems_allowed)))
		goto out;
485

486 487 488 489
	ret = 0;
out:
	rcu_read_unlock();
	return ret;
L
Linus Torvalds 已提交
490 491
}

492
#ifdef CONFIG_SMP
P
Paul Jackson 已提交
493
/*
494
 * Helper routine for generate_sched_domains().
P
Paul Jackson 已提交
495 496 497 498
 * Do cpusets a, b have overlapping cpus_allowed masks?
 */
static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
{
499
	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);
P
Paul Jackson 已提交
500 501
}

502 503 504 505 506 507 508 509
static void
update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
{
	if (dattr->relax_domain_level < c->relax_domain_level)
		dattr->relax_domain_level = c->relax_domain_level;
	return;
}

510 511 512 513 514 515 516 517 518 519 520 521 522 523
static void
update_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c)
{
	LIST_HEAD(q);

	list_add(&c->stack_list, &q);
	while (!list_empty(&q)) {
		struct cpuset *cp;
		struct cgroup *cont;
		struct cpuset *child;

		cp = list_first_entry(&q, struct cpuset, stack_list);
		list_del(q.next);

524
		if (cpumask_empty(cp->cpus_allowed))
525 526 527 528 529
			continue;

		if (is_sched_load_balance(cp))
			update_domain_attr(dattr, cp);

530 531
		rcu_read_lock();
		cpuset_for_each_child(child, cont, cp)
532
			list_add_tail(&child->stack_list, &q);
533
		rcu_read_unlock();
534 535 536
	}
}

P
Paul Jackson 已提交
537
/*
538 539 540 541 542 543 544 545 546
 * generate_sched_domains()
 *
 * This function builds a partial partition of the systems CPUs
 * A 'partial partition' is a set of non-overlapping subsets whose
 * union is a subset of that set.
 * The output of this function needs to be passed to kernel/sched.c
 * partition_sched_domains() routine, which will rebuild the scheduler's
 * load balancing domains (sched domains) as specified by that partial
 * partition.
P
Paul Jackson 已提交
547
 *
L
Li Zefan 已提交
548
 * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
P
Paul Jackson 已提交
549 550 551 552 553 554 555
 * for a background explanation of this.
 *
 * Does not return errors, on the theory that the callers of this
 * routine would rather not worry about failures to rebuild sched
 * domains when operating in the severe memory shortage situations
 * that could cause allocation failures below.
 *
556
 * Must be called with cpuset_mutex held.
P
Paul Jackson 已提交
557 558
 *
 * The three key local variables below are:
559
 *    q  - a linked-list queue of cpuset pointers, used to implement a
P
Paul Jackson 已提交
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
 *	   top-down scan of all cpusets.  This scan loads a pointer
 *	   to each cpuset marked is_sched_load_balance into the
 *	   array 'csa'.  For our purposes, rebuilding the schedulers
 *	   sched domains, we can ignore !is_sched_load_balance cpusets.
 *  csa  - (for CpuSet Array) Array of pointers to all the cpusets
 *	   that need to be load balanced, for convenient iterative
 *	   access by the subsequent code that finds the best partition,
 *	   i.e the set of domains (subsets) of CPUs such that the
 *	   cpus_allowed of every cpuset marked is_sched_load_balance
 *	   is a subset of one of these domains, while there are as
 *	   many such domains as possible, each as small as possible.
 * doms  - Conversion of 'csa' to an array of cpumasks, for passing to
 *	   the kernel/sched.c routine partition_sched_domains() in a
 *	   convenient format, that can be easily compared to the prior
 *	   value to determine what partition elements (sched domains)
 *	   were changed (added or removed.)
 *
 * Finding the best partition (set of domains):
 *	The triple nested loops below over i, j, k scan over the
 *	load balanced cpusets (using the array of cpuset pointers in
 *	csa[]) looking for pairs of cpusets that have overlapping
 *	cpus_allowed, but which don't have the same 'pn' partition
 *	number and gives them in the same partition number.  It keeps
 *	looping on the 'restart' label until it can no longer find
 *	any such pairs.
 *
 *	The union of the cpus_allowed masks from the set of
 *	all cpusets having the same 'pn' value then form the one
 *	element of the partition (one sched domain) to be passed to
 *	partition_sched_domains().
 */
591
static int generate_sched_domains(cpumask_var_t **domains,
592
			struct sched_domain_attr **attributes)
P
Paul Jackson 已提交
593
{
594
	LIST_HEAD(q);		/* queue of cpusets to be scanned */
P
Paul Jackson 已提交
595 596 597 598
	struct cpuset *cp;	/* scans q */
	struct cpuset **csa;	/* array of all cpuset ptrs */
	int csn;		/* how many cpuset ptrs in csa so far */
	int i, j, k;		/* indices for partition finding loops */
599
	cpumask_var_t *doms;	/* resulting partition; i.e. sched domains */
600
	struct sched_domain_attr *dattr;  /* attributes for custom domains */
601
	int ndoms = 0;		/* number of sched domains in result */
602
	int nslot;		/* next empty doms[] struct cpumask slot */
P
Paul Jackson 已提交
603 604

	doms = NULL;
605
	dattr = NULL;
606
	csa = NULL;
P
Paul Jackson 已提交
607 608 609

	/* Special case for the 99% of systems with one, full, sched domain */
	if (is_sched_load_balance(&top_cpuset)) {
610 611
		ndoms = 1;
		doms = alloc_sched_domains(ndoms);
P
Paul Jackson 已提交
612
		if (!doms)
613 614
			goto done;

615 616 617
		dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
		if (dattr) {
			*dattr = SD_ATTR_INIT;
618
			update_domain_attr_tree(dattr, &top_cpuset);
619
		}
620
		cpumask_copy(doms[0], top_cpuset.cpus_allowed);
621 622

		goto done;
P
Paul Jackson 已提交
623 624 625 626 627 628 629
	}

	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
	if (!csa)
		goto done;
	csn = 0;

630 631
	list_add(&top_cpuset.stack_list, &q);
	while (!list_empty(&q)) {
P
Paul Jackson 已提交
632 633
		struct cgroup *cont;
		struct cpuset *child;   /* scans child cpusets of cp */
634

635 636 637
		cp = list_first_entry(&q, struct cpuset, stack_list);
		list_del(q.next);

638
		if (cpumask_empty(cp->cpus_allowed))
639 640
			continue;

641 642 643 644 645 646 647
		/*
		 * All child cpusets contain a subset of the parent's cpus, so
		 * just skip them, and then we call update_domain_attr_tree()
		 * to calc relax_domain_level of the corresponding sched
		 * domain.
		 */
		if (is_sched_load_balance(cp)) {
P
Paul Jackson 已提交
648
			csa[csn++] = cp;
649 650
			continue;
		}
651

652 653
		rcu_read_lock();
		cpuset_for_each_child(child, cont, cp)
654
			list_add_tail(&child->stack_list, &q);
655
		rcu_read_unlock();
P
Paul Jackson 已提交
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
  	}

	for (i = 0; i < csn; i++)
		csa[i]->pn = i;
	ndoms = csn;

restart:
	/* Find the best partition (set of sched domains) */
	for (i = 0; i < csn; i++) {
		struct cpuset *a = csa[i];
		int apn = a->pn;

		for (j = 0; j < csn; j++) {
			struct cpuset *b = csa[j];
			int bpn = b->pn;

			if (apn != bpn && cpusets_overlap(a, b)) {
				for (k = 0; k < csn; k++) {
					struct cpuset *c = csa[k];

					if (c->pn == bpn)
						c->pn = apn;
				}
				ndoms--;	/* one less element */
				goto restart;
			}
		}
	}

685 686 687 688
	/*
	 * Now we know how many domains to create.
	 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
	 */
689
	doms = alloc_sched_domains(ndoms);
690
	if (!doms)
691 692 693 694 695 696
		goto done;

	/*
	 * The rest of the code, including the scheduler, can deal with
	 * dattr==NULL case. No need to abort if alloc fails.
	 */
697
	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
P
Paul Jackson 已提交
698 699 700

	for (nslot = 0, i = 0; i < csn; i++) {
		struct cpuset *a = csa[i];
701
		struct cpumask *dp;
P
Paul Jackson 已提交
702 703
		int apn = a->pn;

704 705 706 707 708
		if (apn < 0) {
			/* Skip completed partitions */
			continue;
		}

709
		dp = doms[nslot];
710 711 712 713 714 715 716 717 718 719

		if (nslot == ndoms) {
			static int warnings = 10;
			if (warnings) {
				printk(KERN_WARNING
				 "rebuild_sched_domains confused:"
				  " nslot %d, ndoms %d, csn %d, i %d,"
				  " apn %d\n",
				  nslot, ndoms, csn, i, apn);
				warnings--;
P
Paul Jackson 已提交
720
			}
721 722
			continue;
		}
P
Paul Jackson 已提交
723

724
		cpumask_clear(dp);
725 726 727 728 729 730
		if (dattr)
			*(dattr + nslot) = SD_ATTR_INIT;
		for (j = i; j < csn; j++) {
			struct cpuset *b = csa[j];

			if (apn == b->pn) {
731
				cpumask_or(dp, dp, b->cpus_allowed);
732 733 734 735 736
				if (dattr)
					update_domain_attr_tree(dattr + nslot, b);

				/* Done with this partition */
				b->pn = -1;
P
Paul Jackson 已提交
737 738
			}
		}
739
		nslot++;
P
Paul Jackson 已提交
740 741 742
	}
	BUG_ON(nslot != ndoms);

743 744 745
done:
	kfree(csa);

746 747 748 749 750 751 752
	/*
	 * Fallback to the default domain if kmalloc() failed.
	 * See comments in partition_sched_domains().
	 */
	if (doms == NULL)
		ndoms = 1;

753 754 755 756 757 758 759 760
	*domains    = doms;
	*attributes = dattr;
	return ndoms;
}

/*
 * Rebuild scheduler domains.
 *
761 762 763 764 765
 * If the flag 'sched_load_balance' of any cpuset with non-empty
 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
 * which has that flag enabled, or if any cpuset with a non-empty
 * 'cpus' is removed, then call this routine to rebuild the
 * scheduler's dynamic sched domains.
766
 *
767
 * Call with cpuset_mutex held.  Takes get_online_cpus().
768
 */
769
static void rebuild_sched_domains_locked(void)
770 771
{
	struct sched_domain_attr *attr;
772
	cpumask_var_t *doms;
773 774
	int ndoms;

775
	lockdep_assert_held(&cpuset_mutex);
776
	get_online_cpus();
777 778 779 780 781 782 783

	/* Generate domain masks and attrs */
	ndoms = generate_sched_domains(&doms, &attr);

	/* Have scheduler rebuild the domains */
	partition_sched_domains(ndoms, doms, attr);

784
	put_online_cpus();
785
}
786
#else /* !CONFIG_SMP */
787
static void rebuild_sched_domains_locked(void)
788 789 790
{
}

791
static int generate_sched_domains(cpumask_var_t **domains,
792 793 794 795 796 797
			struct sched_domain_attr **attributes)
{
	*domains = NULL;
	return 1;
}
#endif /* CONFIG_SMP */
P
Paul Jackson 已提交
798

799 800
void rebuild_sched_domains(void)
{
801
	mutex_lock(&cpuset_mutex);
802
	rebuild_sched_domains_locked();
803
	mutex_unlock(&cpuset_mutex);
P
Paul Jackson 已提交
804 805
}

C
Cliff Wickman 已提交
806 807 808 809 810
/**
 * cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's
 * @tsk: task to test
 * @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
 *
811
 * Call with cpuset_mutex held.  May take callback_mutex during call.
C
Cliff Wickman 已提交
812 813 814
 * Called for each task in a cgroup by cgroup_scan_tasks().
 * Return nonzero if this tasks's cpus_allowed mask should be changed (in other
 * words, if its mask is not equal to its cpuset's mask).
815
 */
816 817
static int cpuset_test_cpumask(struct task_struct *tsk,
			       struct cgroup_scanner *scan)
C
Cliff Wickman 已提交
818
{
819
	return !cpumask_equal(&tsk->cpus_allowed,
C
Cliff Wickman 已提交
820 821
			(cgroup_cs(scan->cg))->cpus_allowed);
}
822

C
Cliff Wickman 已提交
823 824 825 826 827 828 829 830 831
/**
 * cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
 * @tsk: task to test
 * @scan: struct cgroup_scanner containing the cgroup of the task
 *
 * Called by cgroup_scan_tasks() for each task in a cgroup whose
 * cpus_allowed mask needs to be changed.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
832
 * holding cpuset_mutex at this point.
C
Cliff Wickman 已提交
833
 */
834 835
static void cpuset_change_cpumask(struct task_struct *tsk,
				  struct cgroup_scanner *scan)
C
Cliff Wickman 已提交
836
{
837
	set_cpus_allowed_ptr(tsk, ((cgroup_cs(scan->cg))->cpus_allowed));
C
Cliff Wickman 已提交
838 839
}

840 841 842
/**
 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
843
 * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
844
 *
845
 * Called with cpuset_mutex held
846 847 848 849
 *
 * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
850 851
 * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
 * if @heap != NULL.
852
 */
853
static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap)
854 855 856 857 858 859
{
	struct cgroup_scanner scan;

	scan.cg = cs->css.cgroup;
	scan.test_task = cpuset_test_cpumask;
	scan.process_task = cpuset_change_cpumask;
860 861
	scan.heap = heap;
	cgroup_scan_tasks(&scan);
862 863
}

C
Cliff Wickman 已提交
864 865 866 867 868
/**
 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
 * @cs: the cpuset to consider
 * @buf: buffer of cpu numbers written to this cpuset
 */
869 870
static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
			  const char *buf)
L
Linus Torvalds 已提交
871
{
872
	struct ptr_heap heap;
C
Cliff Wickman 已提交
873 874
	int retval;
	int is_load_balanced;
L
Linus Torvalds 已提交
875

876
	/* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
877 878 879
	if (cs == &top_cpuset)
		return -EACCES;

880
	/*
881
	 * An empty cpus_allowed is ok only if the cpuset has no tasks.
882 883 884
	 * Since cpulist_parse() fails on an empty mask, we special case
	 * that parsing.  The validate_change() call ensures that cpusets
	 * with tasks have cpus.
885
	 */
886
	if (!*buf) {
887
		cpumask_clear(trialcs->cpus_allowed);
888
	} else {
889
		retval = cpulist_parse(buf, trialcs->cpus_allowed);
890 891
		if (retval < 0)
			return retval;
892

893
		if (!cpumask_subset(trialcs->cpus_allowed, cpu_active_mask))
894
			return -EINVAL;
895
	}
896
	retval = validate_change(cs, trialcs);
897 898
	if (retval < 0)
		return retval;
P
Paul Jackson 已提交
899

P
Paul Menage 已提交
900
	/* Nothing to do if the cpus didn't change */
901
	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
P
Paul Menage 已提交
902
		return 0;
C
Cliff Wickman 已提交
903

904 905 906 907
	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (retval)
		return retval;

908
	is_load_balanced = is_sched_load_balance(trialcs);
P
Paul Jackson 已提交
909

910
	mutex_lock(&callback_mutex);
911
	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
912
	mutex_unlock(&callback_mutex);
P
Paul Jackson 已提交
913

P
Paul Menage 已提交
914 915
	/*
	 * Scan tasks in the cpuset, and update the cpumasks of any
C
Cliff Wickman 已提交
916
	 * that need an update.
P
Paul Menage 已提交
917
	 */
918 919 920
	update_tasks_cpumask(cs, &heap);

	heap_free(&heap);
C
Cliff Wickman 已提交
921

P
Paul Menage 已提交
922
	if (is_load_balanced)
923
		rebuild_sched_domains_locked();
924
	return 0;
L
Linus Torvalds 已提交
925 926
}

927 928 929 930 931 932 933 934
/*
 * cpuset_migrate_mm
 *
 *    Migrate memory region from one set of nodes to another.
 *
 *    Temporarilly set tasks mems_allowed to target nodes of migration,
 *    so that the migration code can allocate pages on these nodes.
 *
935
 *    Call holding cpuset_mutex, so current's cpuset won't change
936
 *    during this call, as manage_mutex holds off any cpuset_attach()
937 938
 *    calls.  Therefore we don't need to take task_lock around the
 *    call to guarantee_online_mems(), as we know no one is changing
939
 *    our task's cpuset.
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
 *
 *    While the mm_struct we are migrating is typically from some
 *    other task, the task_struct mems_allowed that we are hacking
 *    is for our current task, which must allocate new pages for that
 *    migrating memory region.
 */

static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
							const nodemask_t *to)
{
	struct task_struct *tsk = current;

	tsk->mems_allowed = *to;

	do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);

956
	guarantee_online_mems(task_cs(tsk),&tsk->mems_allowed);
957 958
}

959
/*
960 961 962 963 964 965 966 967 968 969 970
 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
 * @tsk: the task to change
 * @newmems: new nodes that the task will be set
 *
 * In order to avoid seeing no nodes if the old and new nodes are disjoint,
 * we structure updates as setting all new allowed nodes, then clearing newly
 * disallowed ones.
 */
static void cpuset_change_task_nodemask(struct task_struct *tsk,
					nodemask_t *newmems)
{
971
	bool need_loop;
972

973 974 975 976 977 978 979 980 981 982
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return;
	if (current->flags & PF_EXITING) /* Let dying task have memory */
		return;

	task_lock(tsk);
983 984 985 986 987 988 989 990
	/*
	 * Determine if a loop is necessary if another thread is doing
	 * get_mems_allowed().  If at least one node remains unchanged and
	 * tsk does not have a mempolicy, then an empty nodemask will not be
	 * possible when mems_allowed is larger than a word.
	 */
	need_loop = task_has_mempolicy(tsk) ||
			!nodes_intersects(*newmems, tsk->mems_allowed);
991

992 993
	if (need_loop)
		write_seqcount_begin(&tsk->mems_allowed_seq);
994

995 996
	nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);
997 998

	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
999
	tsk->mems_allowed = *newmems;
1000 1001 1002 1003

	if (need_loop)
		write_seqcount_end(&tsk->mems_allowed_seq);

1004
	task_unlock(tsk);
1005 1006 1007 1008 1009
}

/*
 * Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
 * of it to cpuset's new mems_allowed, and migrate pages to new nodes if
1010
 * memory_migrate flag is set. Called with cpuset_mutex held.
1011 1012 1013 1014 1015 1016 1017 1018
 */
static void cpuset_change_nodemask(struct task_struct *p,
				   struct cgroup_scanner *scan)
{
	struct mm_struct *mm;
	struct cpuset *cs;
	int migrate;
	const nodemask_t *oldmem = scan->data;
1019
	static nodemask_t newmems;	/* protected by cpuset_mutex */
1020 1021

	cs = cgroup_cs(scan->cg);
1022
	guarantee_online_mems(cs, &newmems);
1023

1024
	cpuset_change_task_nodemask(p, &newmems);
1025

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	mm = get_task_mm(p);
	if (!mm)
		return;

	migrate = is_memory_migrate(cs);

	mpol_rebind_mm(mm, &cs->mems_allowed);
	if (migrate)
		cpuset_migrate_mm(mm, oldmem, &cs->mems_allowed);
	mmput(mm);
}

1038 1039
static void *cpuset_being_rebound;

1040 1041 1042 1043
/**
 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
 * @oldmem: old mems_allowed of cpuset cs
1044
 * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
1045
 *
1046
 * Called with cpuset_mutex held
1047 1048
 * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
 * if @heap != NULL.
1049
 */
1050 1051
static void update_tasks_nodemask(struct cpuset *cs, const nodemask_t *oldmem,
				 struct ptr_heap *heap)
L
Linus Torvalds 已提交
1052
{
1053
	struct cgroup_scanner scan;
1054

1055
	cpuset_being_rebound = cs;		/* causes mpol_dup() rebind */
1056

1057 1058 1059
	scan.cg = cs->css.cgroup;
	scan.test_task = NULL;
	scan.process_task = cpuset_change_nodemask;
1060
	scan.heap = heap;
1061
	scan.data = (nodemask_t *)oldmem;
1062 1063

	/*
1064 1065 1066 1067
	 * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
	 * take while holding tasklist_lock.  Forks can happen - the
	 * mpol_dup() cpuset_being_rebound check will catch such forks,
	 * and rebind their vma mempolicies too.  Because we still hold
1068
	 * the global cpuset_mutex, we know that no other rebind effort
1069
	 * will be contending for the global variable cpuset_being_rebound.
1070
	 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
1071
	 * is idempotent.  Also migrate pages in each mm to new nodes.
1072
	 */
1073
	cgroup_scan_tasks(&scan);
1074

1075
	/* We're done rebinding vmas to this cpuset's new mems_allowed. */
1076
	cpuset_being_rebound = NULL;
L
Linus Torvalds 已提交
1077 1078
}

1079 1080 1081
/*
 * Handle user request to change the 'mems' memory placement
 * of a cpuset.  Needs to validate the request, update the
1082 1083 1084 1085
 * cpusets mems_allowed, and for each task in the cpuset,
 * update mems_allowed and rebind task's mempolicy and any vma
 * mempolicies and if the cpuset is marked 'memory_migrate',
 * migrate the tasks pages to the new memory.
1086
 *
1087
 * Call with cpuset_mutex held.  May take callback_mutex during call.
1088 1089 1090 1091
 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
 * their mempolicies to the cpusets new mems_allowed.
 */
1092 1093
static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
			   const char *buf)
1094
{
1095
	NODEMASK_ALLOC(nodemask_t, oldmem, GFP_KERNEL);
1096
	int retval;
1097
	struct ptr_heap heap;
1098

1099 1100 1101
	if (!oldmem)
		return -ENOMEM;

1102
	/*
1103
	 * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
1104 1105
	 * it's read-only
	 */
1106 1107 1108 1109
	if (cs == &top_cpuset) {
		retval = -EACCES;
		goto done;
	}
1110 1111 1112 1113 1114 1115 1116 1117

	/*
	 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
	 * Since nodelist_parse() fails on an empty mask, we special case
	 * that parsing.  The validate_change() call ensures that cpusets
	 * with tasks have memory.
	 */
	if (!*buf) {
1118
		nodes_clear(trialcs->mems_allowed);
1119
	} else {
1120
		retval = nodelist_parse(buf, trialcs->mems_allowed);
1121 1122 1123
		if (retval < 0)
			goto done;

1124
		if (!nodes_subset(trialcs->mems_allowed,
1125
				node_states[N_MEMORY])) {
1126 1127 1128
			retval =  -EINVAL;
			goto done;
		}
1129
	}
1130 1131
	*oldmem = cs->mems_allowed;
	if (nodes_equal(*oldmem, trialcs->mems_allowed)) {
1132 1133 1134
		retval = 0;		/* Too easy - nothing to do */
		goto done;
	}
1135
	retval = validate_change(cs, trialcs);
1136 1137 1138
	if (retval < 0)
		goto done;

1139 1140 1141 1142
	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (retval < 0)
		goto done;

1143
	mutex_lock(&callback_mutex);
1144
	cs->mems_allowed = trialcs->mems_allowed;
1145 1146
	mutex_unlock(&callback_mutex);

1147
	update_tasks_nodemask(cs, oldmem, &heap);
1148 1149

	heap_free(&heap);
1150
done:
1151
	NODEMASK_FREE(oldmem);
1152 1153 1154
	return retval;
}

1155 1156 1157 1158 1159
int current_cpuset_is_being_rebound(void)
{
	return task_cs(current) == cpuset_being_rebound;
}

1160
static int update_relax_domain_level(struct cpuset *cs, s64 val)
1161
{
1162
#ifdef CONFIG_SMP
1163
	if (val < -1 || val >= sched_domain_level_max)
1164
		return -EINVAL;
1165
#endif
1166 1167 1168

	if (val != cs->relax_domain_level) {
		cs->relax_domain_level = val;
1169 1170
		if (!cpumask_empty(cs->cpus_allowed) &&
		    is_sched_load_balance(cs))
1171
			rebuild_sched_domains_locked();
1172 1173 1174 1175 1176
	}

	return 0;
}

1177 1178 1179 1180 1181 1182 1183 1184
/*
 * cpuset_change_flag - make a task's spread flags the same as its cpuset's
 * @tsk: task to be updated
 * @scan: struct cgroup_scanner containing the cgroup of the task
 *
 * Called by cgroup_scan_tasks() for each task in a cgroup.
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
1185
 * holding cpuset_mutex at this point.
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
 */
static void cpuset_change_flag(struct task_struct *tsk,
				struct cgroup_scanner *scan)
{
	cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
}

/*
 * update_tasks_flags - update the spread flags of tasks in the cpuset.
 * @cs: the cpuset in which each task's spread flags needs to be changed
 * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
 *
1198
 * Called with cpuset_mutex held
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
 *
 * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 *
 * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
 * if @heap != NULL.
 */
static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
{
	struct cgroup_scanner scan;

	scan.cg = cs->css.cgroup;
	scan.test_task = NULL;
	scan.process_task = cpuset_change_flag;
	scan.heap = heap;
	cgroup_scan_tasks(&scan);
}

L
Linus Torvalds 已提交
1217 1218
/*
 * update_flag - read a 0 or a 1 in a file and update associated flag
1219 1220 1221
 * bit:		the bit to update (see cpuset_flagbits_t)
 * cs:		the cpuset to update
 * turning_on: 	whether the flag is being set or cleared
1222
 *
1223
 * Call with cpuset_mutex held.
L
Linus Torvalds 已提交
1224 1225
 */

1226 1227
static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
		       int turning_on)
L
Linus Torvalds 已提交
1228
{
1229
	struct cpuset *trialcs;
R
Rakib Mullick 已提交
1230
	int balance_flag_changed;
1231 1232 1233
	int spread_flag_changed;
	struct ptr_heap heap;
	int err;
L
Linus Torvalds 已提交
1234

1235 1236 1237 1238
	trialcs = alloc_trial_cpuset(cs);
	if (!trialcs)
		return -ENOMEM;

L
Linus Torvalds 已提交
1239
	if (turning_on)
1240
		set_bit(bit, &trialcs->flags);
L
Linus Torvalds 已提交
1241
	else
1242
		clear_bit(bit, &trialcs->flags);
L
Linus Torvalds 已提交
1243

1244
	err = validate_change(cs, trialcs);
1245
	if (err < 0)
1246
		goto out;
P
Paul Jackson 已提交
1247

1248 1249 1250 1251
	err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (err < 0)
		goto out;

P
Paul Jackson 已提交
1252
	balance_flag_changed = (is_sched_load_balance(cs) !=
1253
				is_sched_load_balance(trialcs));
P
Paul Jackson 已提交
1254

1255 1256 1257
	spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
			|| (is_spread_page(cs) != is_spread_page(trialcs)));

1258
	mutex_lock(&callback_mutex);
1259
	cs->flags = trialcs->flags;
1260
	mutex_unlock(&callback_mutex);
1261

1262
	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1263
		rebuild_sched_domains_locked();
P
Paul Jackson 已提交
1264

1265 1266 1267
	if (spread_flag_changed)
		update_tasks_flags(cs, &heap);
	heap_free(&heap);
1268 1269 1270
out:
	free_trial_cpuset(trialcs);
	return err;
L
Linus Torvalds 已提交
1271 1272
}

1273
/*
A
Adrian Bunk 已提交
1274
 * Frequency meter - How fast is some event occurring?
1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
 *
 * These routines manage a digitally filtered, constant time based,
 * event frequency meter.  There are four routines:
 *   fmeter_init() - initialize a frequency meter.
 *   fmeter_markevent() - called each time the event happens.
 *   fmeter_getrate() - returns the recent rate of such events.
 *   fmeter_update() - internal routine used to update fmeter.
 *
 * A common data structure is passed to each of these routines,
 * which is used to keep track of the state required to manage the
 * frequency meter and its digital filter.
 *
 * The filter works on the number of events marked per unit time.
 * The filter is single-pole low-pass recursive (IIR).  The time unit
 * is 1 second.  Arithmetic is done using 32-bit integers scaled to
 * simulate 3 decimal digits of precision (multiplied by 1000).
 *
 * With an FM_COEF of 933, and a time base of 1 second, the filter
 * has a half-life of 10 seconds, meaning that if the events quit
 * happening, then the rate returned from the fmeter_getrate()
 * will be cut in half each 10 seconds, until it converges to zero.
 *
 * It is not worth doing a real infinitely recursive filter.  If more
 * than FM_MAXTICKS ticks have elapsed since the last filter event,
 * just compute FM_MAXTICKS ticks worth, by which point the level
 * will be stable.
 *
 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
 * arithmetic overflow in the fmeter_update() routine.
 *
 * Given the simple 32 bit integer arithmetic used, this meter works
 * best for reporting rates between one per millisecond (msec) and
 * one per 32 (approx) seconds.  At constant rates faster than one
 * per msec it maxes out at values just under 1,000,000.  At constant
 * rates between one per msec, and one per second it will stabilize
 * to a value N*1000, where N is the rate of events per second.
 * At constant rates between one per second and one per 32 seconds,
 * it will be choppy, moving up on the seconds that have an event,
 * and then decaying until the next event.  At rates slower than
 * about one in 32 seconds, it decays all the way back to zero between
 * each event.
 */

#define FM_COEF 933		/* coefficient for half-life of 10 secs */
#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
#define FM_MAXCNT 1000000	/* limit cnt to avoid overflow */
#define FM_SCALE 1000		/* faux fixed point scale */

/* Initialize a frequency meter */
static void fmeter_init(struct fmeter *fmp)
{
	fmp->cnt = 0;
	fmp->val = 0;
	fmp->time = 0;
	spin_lock_init(&fmp->lock);
}

/* Internal meter update - process cnt events and update value */
static void fmeter_update(struct fmeter *fmp)
{
	time_t now = get_seconds();
	time_t ticks = now - fmp->time;

	if (ticks == 0)
		return;

	ticks = min(FM_MAXTICKS, ticks);
	while (ticks-- > 0)
		fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
	fmp->time = now;

	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
	fmp->cnt = 0;
}

/* Process any previous ticks, then bump cnt by one (times scale). */
static void fmeter_markevent(struct fmeter *fmp)
{
	spin_lock(&fmp->lock);
	fmeter_update(fmp);
	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
	spin_unlock(&fmp->lock);
}

/* Process any previous ticks, then return current value. */
static int fmeter_getrate(struct fmeter *fmp)
{
	int val;

	spin_lock(&fmp->lock);
	fmeter_update(fmp);
	val = fmp->val;
	spin_unlock(&fmp->lock);
	return val;
}

1371
/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
1372
static int cpuset_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
1373
{
1374
	struct cpuset *cs = cgroup_cs(cgrp);
1375 1376
	struct task_struct *task;
	int ret;
L
Linus Torvalds 已提交
1377

1378 1379 1380
	mutex_lock(&cpuset_mutex);

	ret = -ENOSPC;
1381
	if (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
1382
		goto out_unlock;
1383

1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
	cgroup_taskset_for_each(task, cgrp, tset) {
		/*
		 * Kthreads bound to specific cpus cannot be moved to a new
		 * cpuset; we cannot change their cpu affinity and
		 * isolating such threads by their set of allowed nodes is
		 * unnecessary.  Thus, cpusets are not applicable for such
		 * threads.  This prevents checking for success of
		 * set_cpus_allowed_ptr() on all attached tasks before
		 * cpus_allowed may be changed.
		 */
1394
		ret = -EINVAL;
1395
		if (task->flags & PF_THREAD_BOUND)
1396 1397 1398 1399
			goto out_unlock;
		ret = security_task_setscheduler(task);
		if (ret)
			goto out_unlock;
1400
	}
1401

1402 1403 1404 1405 1406
	/*
	 * Mark attach is in progress.  This makes validate_change() fail
	 * changes which zero cpus/mems_allowed.
	 */
	cs->attach_in_progress++;
1407 1408 1409 1410
	ret = 0;
out_unlock:
	mutex_unlock(&cpuset_mutex);
	return ret;
1411
}
L
Linus Torvalds 已提交
1412

1413 1414 1415
static void cpuset_cancel_attach(struct cgroup *cgrp,
				 struct cgroup_taskset *tset)
{
1416
	mutex_lock(&cpuset_mutex);
1417
	cgroup_cs(cgrp)->attach_in_progress--;
1418
	mutex_unlock(&cpuset_mutex);
1419 1420
}

1421
/*
1422
 * Protected by cpuset_mutex.  cpus_attach is used only by cpuset_attach()
1423 1424 1425 1426 1427
 * but we can't allocate it dynamically there.  Define it global and
 * allocate from cpuset_init().
 */
static cpumask_var_t cpus_attach;

1428
static void cpuset_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
1429
{
1430
	/* static bufs protected by cpuset_mutex */
1431 1432
	static nodemask_t cpuset_attach_nodemask_from;
	static nodemask_t cpuset_attach_nodemask_to;
1433
	struct mm_struct *mm;
1434 1435
	struct task_struct *task;
	struct task_struct *leader = cgroup_taskset_first(tset);
1436 1437 1438
	struct cgroup *oldcgrp = cgroup_taskset_cur_cgroup(tset);
	struct cpuset *cs = cgroup_cs(cgrp);
	struct cpuset *oldcs = cgroup_cs(oldcgrp);
1439

1440 1441
	mutex_lock(&cpuset_mutex);

1442 1443 1444 1445 1446 1447 1448 1449
	/* prepare for attach */
	if (cs == &top_cpuset)
		cpumask_copy(cpus_attach, cpu_possible_mask);
	else
		guarantee_online_cpus(cs, cpus_attach);

	guarantee_online_mems(cs, &cpuset_attach_nodemask_to);

1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	cgroup_taskset_for_each(task, cgrp, tset) {
		/*
		 * can_attach beforehand should guarantee that this doesn't
		 * fail.  TODO: have a better way to handle failure here
		 */
		WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));

		cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
		cpuset_update_task_spread_flag(cs, task);
	}
1460

1461 1462 1463 1464 1465 1466
	/*
	 * Change mm, possibly for multiple threads in a threadgroup. This is
	 * expensive and may sleep.
	 */
	cpuset_attach_nodemask_from = oldcs->mems_allowed;
	cpuset_attach_nodemask_to = cs->mems_allowed;
1467
	mm = get_task_mm(leader);
1468
	if (mm) {
1469
		mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
1470
		if (is_memory_migrate(cs))
1471 1472
			cpuset_migrate_mm(mm, &cpuset_attach_nodemask_from,
					  &cpuset_attach_nodemask_to);
1473 1474
		mmput(mm);
	}
1475 1476

	cs->attach_in_progress--;
1477 1478 1479 1480 1481 1482 1483 1484

	/*
	 * We may have raced with CPU/memory hotunplug.  Trigger hotplug
	 * propagation if @cs doesn't have any CPU or memory.  It will move
	 * the newly added tasks to the nearest parent which can execute.
	 */
	if (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
		schedule_cpuset_propagate_hotplug(cs);
1485 1486

	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
1487 1488 1489 1490 1491
}

/* The various types of files and directories in a cpuset file system */

typedef enum {
1492
	FILE_MEMORY_MIGRATE,
L
Linus Torvalds 已提交
1493 1494 1495 1496
	FILE_CPULIST,
	FILE_MEMLIST,
	FILE_CPU_EXCLUSIVE,
	FILE_MEM_EXCLUSIVE,
1497
	FILE_MEM_HARDWALL,
P
Paul Jackson 已提交
1498
	FILE_SCHED_LOAD_BALANCE,
1499
	FILE_SCHED_RELAX_DOMAIN_LEVEL,
1500 1501
	FILE_MEMORY_PRESSURE_ENABLED,
	FILE_MEMORY_PRESSURE,
1502 1503
	FILE_SPREAD_PAGE,
	FILE_SPREAD_SLAB,
L
Linus Torvalds 已提交
1504 1505
} cpuset_filetype_t;

1506 1507 1508 1509
static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
{
	struct cpuset *cs = cgroup_cs(cgrp);
	cpuset_filetype_t type = cft->private;
1510
	int retval = -ENODEV;
1511

1512 1513 1514
	mutex_lock(&cpuset_mutex);
	if (!is_cpuset_online(cs))
		goto out_unlock;
1515 1516

	switch (type) {
L
Linus Torvalds 已提交
1517
	case FILE_CPU_EXCLUSIVE:
1518
		retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
L
Linus Torvalds 已提交
1519 1520
		break;
	case FILE_MEM_EXCLUSIVE:
1521
		retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
L
Linus Torvalds 已提交
1522
		break;
1523 1524 1525
	case FILE_MEM_HARDWALL:
		retval = update_flag(CS_MEM_HARDWALL, cs, val);
		break;
P
Paul Jackson 已提交
1526
	case FILE_SCHED_LOAD_BALANCE:
1527
		retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1528
		break;
1529
	case FILE_MEMORY_MIGRATE:
1530
		retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1531
		break;
1532
	case FILE_MEMORY_PRESSURE_ENABLED:
1533
		cpuset_memory_pressure_enabled = !!val;
1534 1535 1536 1537
		break;
	case FILE_MEMORY_PRESSURE:
		retval = -EACCES;
		break;
1538
	case FILE_SPREAD_PAGE:
1539
		retval = update_flag(CS_SPREAD_PAGE, cs, val);
1540 1541
		break;
	case FILE_SPREAD_SLAB:
1542
		retval = update_flag(CS_SPREAD_SLAB, cs, val);
1543
		break;
L
Linus Torvalds 已提交
1544 1545
	default:
		retval = -EINVAL;
1546
		break;
L
Linus Torvalds 已提交
1547
	}
1548 1549
out_unlock:
	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
1550 1551 1552
	return retval;
}

1553 1554 1555 1556
static int cpuset_write_s64(struct cgroup *cgrp, struct cftype *cft, s64 val)
{
	struct cpuset *cs = cgroup_cs(cgrp);
	cpuset_filetype_t type = cft->private;
1557
	int retval = -ENODEV;
1558

1559 1560 1561
	mutex_lock(&cpuset_mutex);
	if (!is_cpuset_online(cs))
		goto out_unlock;
1562

1563 1564 1565 1566 1567 1568 1569 1570
	switch (type) {
	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
		retval = update_relax_domain_level(cs, val);
		break;
	default:
		retval = -EINVAL;
		break;
	}
1571 1572
out_unlock:
	mutex_unlock(&cpuset_mutex);
1573 1574 1575
	return retval;
}

1576 1577 1578 1579 1580 1581
/*
 * Common handling for a write to a "cpus" or "mems" file.
 */
static int cpuset_write_resmask(struct cgroup *cgrp, struct cftype *cft,
				const char *buf)
{
1582 1583
	struct cpuset *cs = cgroup_cs(cgrp);
	struct cpuset *trialcs;
1584
	int retval = -ENODEV;
1585

1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
	/*
	 * CPU or memory hotunplug may leave @cs w/o any execution
	 * resources, in which case the hotplug code asynchronously updates
	 * configuration and transfers all tasks to the nearest ancestor
	 * which can execute.
	 *
	 * As writes to "cpus" or "mems" may restore @cs's execution
	 * resources, wait for the previously scheduled operations before
	 * proceeding, so that we don't end up keep removing tasks added
	 * after execution capability is restored.
1596 1597 1598 1599
	 *
	 * Flushing cpuset_hotplug_work is enough to synchronize against
	 * hotplug hanlding; however, cpuset_attach() may schedule
	 * propagation work directly.  Flush the workqueue too.
1600 1601
	 */
	flush_work(&cpuset_hotplug_work);
1602
	flush_workqueue(cpuset_propagate_hotplug_wq);
1603

1604 1605 1606
	mutex_lock(&cpuset_mutex);
	if (!is_cpuset_online(cs))
		goto out_unlock;
1607

1608
	trialcs = alloc_trial_cpuset(cs);
1609 1610
	if (!trialcs) {
		retval = -ENOMEM;
1611
		goto out_unlock;
1612
	}
1613

1614 1615
	switch (cft->private) {
	case FILE_CPULIST:
1616
		retval = update_cpumask(cs, trialcs, buf);
1617 1618
		break;
	case FILE_MEMLIST:
1619
		retval = update_nodemask(cs, trialcs, buf);
1620 1621 1622 1623 1624
		break;
	default:
		retval = -EINVAL;
		break;
	}
1625 1626

	free_trial_cpuset(trialcs);
1627 1628
out_unlock:
	mutex_unlock(&cpuset_mutex);
1629 1630 1631
	return retval;
}

L
Linus Torvalds 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
/*
 * These ascii lists should be read in a single call, by using a user
 * buffer large enough to hold the entire map.  If read in smaller
 * chunks, there is no guarantee of atomicity.  Since the display format
 * used, list of ranges of sequential numbers, is variable length,
 * and since these maps can change value dynamically, one could read
 * gibberish by doing partial reads while a list was changing.
 * A single large read to a buffer that crosses a page boundary is
 * ok, because the result being copied to user land is not recomputed
 * across a page fault.
 */

1644
static size_t cpuset_sprintf_cpulist(char *page, struct cpuset *cs)
L
Linus Torvalds 已提交
1645
{
1646
	size_t count;
L
Linus Torvalds 已提交
1647

1648
	mutex_lock(&callback_mutex);
1649
	count = cpulist_scnprintf(page, PAGE_SIZE, cs->cpus_allowed);
1650
	mutex_unlock(&callback_mutex);
L
Linus Torvalds 已提交
1651

1652
	return count;
L
Linus Torvalds 已提交
1653 1654
}

1655
static size_t cpuset_sprintf_memlist(char *page, struct cpuset *cs)
L
Linus Torvalds 已提交
1656
{
1657
	size_t count;
L
Linus Torvalds 已提交
1658

1659
	mutex_lock(&callback_mutex);
1660
	count = nodelist_scnprintf(page, PAGE_SIZE, cs->mems_allowed);
1661
	mutex_unlock(&callback_mutex);
L
Linus Torvalds 已提交
1662

1663
	return count;
L
Linus Torvalds 已提交
1664 1665
}

1666 1667 1668 1669 1670
static ssize_t cpuset_common_file_read(struct cgroup *cont,
				       struct cftype *cft,
				       struct file *file,
				       char __user *buf,
				       size_t nbytes, loff_t *ppos)
L
Linus Torvalds 已提交
1671
{
1672
	struct cpuset *cs = cgroup_cs(cont);
L
Linus Torvalds 已提交
1673 1674 1675 1676 1677
	cpuset_filetype_t type = cft->private;
	char *page;
	ssize_t retval = 0;
	char *s;

1678
	if (!(page = (char *)__get_free_page(GFP_TEMPORARY)))
L
Linus Torvalds 已提交
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
		return -ENOMEM;

	s = page;

	switch (type) {
	case FILE_CPULIST:
		s += cpuset_sprintf_cpulist(s, cs);
		break;
	case FILE_MEMLIST:
		s += cpuset_sprintf_memlist(s, cs);
		break;
	default:
		retval = -EINVAL;
		goto out;
	}
	*s++ = '\n';

A
Al Viro 已提交
1696
	retval = simple_read_from_buffer(buf, nbytes, ppos, page, s - page);
L
Linus Torvalds 已提交
1697 1698 1699 1700 1701
out:
	free_page((unsigned long)page);
	return retval;
}

1702 1703 1704 1705 1706 1707 1708 1709 1710
static u64 cpuset_read_u64(struct cgroup *cont, struct cftype *cft)
{
	struct cpuset *cs = cgroup_cs(cont);
	cpuset_filetype_t type = cft->private;
	switch (type) {
	case FILE_CPU_EXCLUSIVE:
		return is_cpu_exclusive(cs);
	case FILE_MEM_EXCLUSIVE:
		return is_mem_exclusive(cs);
1711 1712
	case FILE_MEM_HARDWALL:
		return is_mem_hardwall(cs);
1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727
	case FILE_SCHED_LOAD_BALANCE:
		return is_sched_load_balance(cs);
	case FILE_MEMORY_MIGRATE:
		return is_memory_migrate(cs);
	case FILE_MEMORY_PRESSURE_ENABLED:
		return cpuset_memory_pressure_enabled;
	case FILE_MEMORY_PRESSURE:
		return fmeter_getrate(&cs->fmeter);
	case FILE_SPREAD_PAGE:
		return is_spread_page(cs);
	case FILE_SPREAD_SLAB:
		return is_spread_slab(cs);
	default:
		BUG();
	}
1728 1729 1730

	/* Unreachable but makes gcc happy */
	return 0;
1731
}
L
Linus Torvalds 已提交
1732

1733 1734 1735 1736 1737 1738 1739 1740 1741 1742
static s64 cpuset_read_s64(struct cgroup *cont, struct cftype *cft)
{
	struct cpuset *cs = cgroup_cs(cont);
	cpuset_filetype_t type = cft->private;
	switch (type) {
	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
		return cs->relax_domain_level;
	default:
		BUG();
	}
1743 1744 1745

	/* Unrechable but makes gcc happy */
	return 0;
1746 1747
}

L
Linus Torvalds 已提交
1748 1749 1750 1751 1752

/*
 * for the common functions, 'private' gives the type of file
 */

1753 1754 1755 1756
static struct cftype files[] = {
	{
		.name = "cpus",
		.read = cpuset_common_file_read,
1757 1758
		.write_string = cpuset_write_resmask,
		.max_write_len = (100U + 6 * NR_CPUS),
1759 1760 1761 1762 1763 1764
		.private = FILE_CPULIST,
	},

	{
		.name = "mems",
		.read = cpuset_common_file_read,
1765 1766
		.write_string = cpuset_write_resmask,
		.max_write_len = (100U + 6 * MAX_NUMNODES),
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
		.private = FILE_MEMLIST,
	},

	{
		.name = "cpu_exclusive",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_CPU_EXCLUSIVE,
	},

	{
		.name = "mem_exclusive",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEM_EXCLUSIVE,
	},

1784 1785 1786 1787 1788 1789 1790
	{
		.name = "mem_hardwall",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEM_HARDWALL,
	},

1791 1792 1793 1794 1795 1796 1797 1798 1799
	{
		.name = "sched_load_balance",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SCHED_LOAD_BALANCE,
	},

	{
		.name = "sched_relax_domain_level",
1800 1801
		.read_s64 = cpuset_read_s64,
		.write_s64 = cpuset_write_s64,
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
		.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
	},

	{
		.name = "memory_migrate",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_MIGRATE,
	},

	{
		.name = "memory_pressure",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_PRESSURE,
L
Li Zefan 已提交
1817
		.mode = S_IRUGO,
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
	},

	{
		.name = "memory_spread_page",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SPREAD_PAGE,
	},

	{
		.name = "memory_spread_slab",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SPREAD_SLAB,
	},
1833

1834 1835 1836 1837 1838 1839 1840
	{
		.name = "memory_pressure_enabled",
		.flags = CFTYPE_ONLY_ON_ROOT,
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_PRESSURE_ENABLED,
	},
L
Linus Torvalds 已提交
1841

1842 1843
	{ }	/* terminate */
};
L
Linus Torvalds 已提交
1844 1845

/*
1846
 *	cpuset_css_alloc - allocate a cpuset css
1847
 *	cont:	control group that the new cpuset will be part of
L
Linus Torvalds 已提交
1848 1849
 */

1850
static struct cgroup_subsys_state *cpuset_css_alloc(struct cgroup *cont)
L
Linus Torvalds 已提交
1851
{
T
Tejun Heo 已提交
1852
	struct cpuset *cs;
L
Linus Torvalds 已提交
1853

T
Tejun Heo 已提交
1854
	if (!cont->parent)
1855
		return &top_cpuset.css;
1856

T
Tejun Heo 已提交
1857
	cs = kzalloc(sizeof(*cs), GFP_KERNEL);
L
Linus Torvalds 已提交
1858
	if (!cs)
1859
		return ERR_PTR(-ENOMEM);
1860 1861 1862 1863
	if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
		kfree(cs);
		return ERR_PTR(-ENOMEM);
	}
L
Linus Torvalds 已提交
1864

P
Paul Jackson 已提交
1865
	set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
1866
	cpumask_clear(cs->cpus_allowed);
1867
	nodes_clear(cs->mems_allowed);
1868
	fmeter_init(&cs->fmeter);
1869
	INIT_WORK(&cs->hotplug_work, cpuset_propagate_hotplug_workfn);
1870
	cs->relax_domain_level = -1;
T
Tejun Heo 已提交
1871 1872 1873 1874 1875 1876 1877 1878 1879
	cs->parent = cgroup_cs(cont->parent);

	return &cs->css;
}

static int cpuset_css_online(struct cgroup *cgrp)
{
	struct cpuset *cs = cgroup_cs(cgrp);
	struct cpuset *parent = cs->parent;
1880 1881
	struct cpuset *tmp_cs;
	struct cgroup *pos_cg;
T
Tejun Heo 已提交
1882 1883 1884 1885

	if (!parent)
		return 0;

1886 1887
	mutex_lock(&cpuset_mutex);

T
Tejun Heo 已提交
1888
	set_bit(CS_ONLINE, &cs->flags);
T
Tejun Heo 已提交
1889 1890 1891 1892
	if (is_spread_page(parent))
		set_bit(CS_SPREAD_PAGE, &cs->flags);
	if (is_spread_slab(parent))
		set_bit(CS_SPREAD_SLAB, &cs->flags);
L
Linus Torvalds 已提交
1893

1894
	number_of_cpusets++;
1895

T
Tejun Heo 已提交
1896
	if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags))
1897
		goto out_unlock;
1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911

	/*
	 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
	 * set.  This flag handling is implemented in cgroup core for
	 * histrical reasons - the flag may be specified during mount.
	 *
	 * Currently, if any sibling cpusets have exclusive cpus or mem, we
	 * refuse to clone the configuration - thereby refusing the task to
	 * be entered, and as a result refusing the sys_unshare() or
	 * clone() which initiated it.  If this becomes a problem for some
	 * users who wish to allow that scenario, then this could be
	 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
	 * (and likewise for mems) to the new cgroup.
	 */
1912 1913 1914 1915
	rcu_read_lock();
	cpuset_for_each_child(tmp_cs, pos_cg, parent) {
		if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
			rcu_read_unlock();
1916
			goto out_unlock;
1917
		}
1918
	}
1919
	rcu_read_unlock();
1920 1921 1922 1923 1924

	mutex_lock(&callback_mutex);
	cs->mems_allowed = parent->mems_allowed;
	cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
	mutex_unlock(&callback_mutex);
1925 1926
out_unlock:
	mutex_unlock(&cpuset_mutex);
T
Tejun Heo 已提交
1927 1928 1929 1930 1931 1932 1933
	return 0;
}

static void cpuset_css_offline(struct cgroup *cgrp)
{
	struct cpuset *cs = cgroup_cs(cgrp);

1934
	mutex_lock(&cpuset_mutex);
T
Tejun Heo 已提交
1935 1936 1937 1938 1939

	if (is_sched_load_balance(cs))
		update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);

	number_of_cpusets--;
T
Tejun Heo 已提交
1940
	clear_bit(CS_ONLINE, &cs->flags);
T
Tejun Heo 已提交
1941

1942
	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
1943 1944
}

P
Paul Jackson 已提交
1945 1946 1947
/*
 * If the cpuset being removed has its flag 'sched_load_balance'
 * enabled, then simulate turning sched_load_balance off, which
1948
 * will call rebuild_sched_domains_locked().
P
Paul Jackson 已提交
1949 1950
 */

1951
static void cpuset_css_free(struct cgroup *cont)
L
Linus Torvalds 已提交
1952
{
1953
	struct cpuset *cs = cgroup_cs(cont);
L
Linus Torvalds 已提交
1954

1955
	free_cpumask_var(cs->cpus_allowed);
1956
	kfree(cs);
L
Linus Torvalds 已提交
1957 1958
}

1959 1960
struct cgroup_subsys cpuset_subsys = {
	.name = "cpuset",
1961
	.css_alloc = cpuset_css_alloc,
T
Tejun Heo 已提交
1962 1963
	.css_online = cpuset_css_online,
	.css_offline = cpuset_css_offline,
1964
	.css_free = cpuset_css_free,
1965
	.can_attach = cpuset_can_attach,
1966
	.cancel_attach = cpuset_cancel_attach,
1967 1968
	.attach = cpuset_attach,
	.subsys_id = cpuset_subsys_id,
1969
	.base_cftypes = files,
1970 1971 1972
	.early_init = 1,
};

L
Linus Torvalds 已提交
1973 1974 1975 1976 1977 1978 1979 1980
/**
 * cpuset_init - initialize cpusets at system boot
 *
 * Description: Initialize top_cpuset and the cpuset internal file system,
 **/

int __init cpuset_init(void)
{
1981
	int err = 0;
L
Linus Torvalds 已提交
1982

1983 1984 1985
	if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
		BUG();

1986
	cpumask_setall(top_cpuset.cpus_allowed);
1987
	nodes_setall(top_cpuset.mems_allowed);
L
Linus Torvalds 已提交
1988

1989
	fmeter_init(&top_cpuset.fmeter);
P
Paul Jackson 已提交
1990
	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
1991
	top_cpuset.relax_domain_level = -1;
L
Linus Torvalds 已提交
1992 1993 1994

	err = register_filesystem(&cpuset_fs_type);
	if (err < 0)
1995 1996
		return err;

1997 1998 1999
	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
		BUG();

2000
	number_of_cpusets = 1;
2001
	return 0;
L
Linus Torvalds 已提交
2002 2003
}

2004 2005 2006 2007 2008 2009 2010 2011
/**
 * cpuset_do_move_task - move a given task to another cpuset
 * @tsk: pointer to task_struct the task to move
 * @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
 *
 * Called by cgroup_scan_tasks() for each task in a cgroup.
 * Return nonzero to stop the walk through the tasks.
 */
2012 2013
static void cpuset_do_move_task(struct task_struct *tsk,
				struct cgroup_scanner *scan)
2014
{
2015
	struct cgroup *new_cgroup = scan->data;
2016

2017
	cgroup_lock();
2018
	cgroup_attach_task(new_cgroup, tsk);
2019
	cgroup_unlock();
2020 2021 2022 2023 2024 2025 2026
}

/**
 * move_member_tasks_to_cpuset - move tasks from one cpuset to another
 * @from: cpuset in which the tasks currently reside
 * @to: cpuset to which the tasks will be moved
 *
2027
 * Called with cpuset_mutex held
2028
 * callback_mutex must not be held, as cpuset_attach() will take it.
2029 2030 2031 2032 2033 2034
 *
 * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
 * calling callback functions for each.
 */
static void move_member_tasks_to_cpuset(struct cpuset *from, struct cpuset *to)
{
2035
	struct cgroup_scanner scan;
2036

2037 2038 2039 2040 2041
	scan.cg = from->css.cgroup;
	scan.test_task = NULL; /* select all tasks in cgroup */
	scan.process_task = cpuset_do_move_task;
	scan.heap = NULL;
	scan.data = to->css.cgroup;
2042

2043
	if (cgroup_scan_tasks(&scan))
2044 2045 2046 2047
		printk(KERN_ERR "move_member_tasks_to_cpuset: "
				"cgroup_scan_tasks failed\n");
}

2048
/*
2049
 * If CPU and/or memory hotplug handlers, below, unplug any CPUs
2050 2051
 * or memory nodes, we need to walk over the cpuset hierarchy,
 * removing that CPU or node from all cpusets.  If this removes the
2052 2053
 * last CPU or node from a cpuset, then move the tasks in the empty
 * cpuset to its next-highest non-empty parent.
2054
 */
2055 2056 2057 2058 2059 2060 2061 2062 2063
static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
{
	struct cpuset *parent;

	/*
	 * Find its next-highest non-empty parent, (top cpuset
	 * has online cpus, so can't be empty).
	 */
	parent = cs->parent;
2064
	while (cpumask_empty(parent->cpus_allowed) ||
2065
			nodes_empty(parent->mems_allowed))
2066 2067 2068 2069 2070
		parent = parent->parent;

	move_member_tasks_to_cpuset(cs, parent);
}

2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
/*
 * Helper function to traverse cpusets.
 * It can be used to walk the cpuset tree from top to bottom, completing
 * one layer before dropping down to the next (thus always processing a
 * node before any of its children).
 */
static struct cpuset *cpuset_next(struct list_head *queue)
{
	struct cpuset *cp;
	struct cpuset *child;	/* scans child cpusets of cp */
	struct cgroup *cont;

	if (list_empty(queue))
		return NULL;

	cp = list_first_entry(queue, struct cpuset, stack_list);
	list_del(queue->next);
2088 2089
	rcu_read_lock();
	cpuset_for_each_child(child, cont, cp)
2090
		list_add_tail(&child->stack_list, queue);
2091
	rcu_read_unlock();
2092 2093 2094 2095

	return cp;
}

2096
/**
2097
 * cpuset_propagate_hotplug_workfn - propagate CPU/memory hotplug to a cpuset
2098
 * @cs: cpuset in interest
2099
 *
2100 2101 2102
 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
 * offline, update @cs accordingly.  If @cs ends up with no CPU or memory,
 * all its tasks are moved to the nearest ancestor with both resources.
2103
 */
2104
static void cpuset_propagate_hotplug_workfn(struct work_struct *work)
2105
{
2106 2107
	static cpumask_t off_cpus;
	static nodemask_t off_mems, tmp_mems;
2108
	struct cpuset *cs = container_of(work, struct cpuset, hotplug_work);
2109
	bool is_empty;
2110

2111
	mutex_lock(&cpuset_mutex);
2112

2113 2114
	cpumask_andnot(&off_cpus, cs->cpus_allowed, top_cpuset.cpus_allowed);
	nodes_andnot(off_mems, cs->mems_allowed, top_cpuset.mems_allowed);
2115

2116 2117 2118 2119 2120 2121 2122
	/* remove offline cpus from @cs */
	if (!cpumask_empty(&off_cpus)) {
		mutex_lock(&callback_mutex);
		cpumask_andnot(cs->cpus_allowed, cs->cpus_allowed, &off_cpus);
		mutex_unlock(&callback_mutex);
		update_tasks_cpumask(cs, NULL);
	}
2123

2124 2125 2126 2127 2128 2129 2130
	/* remove offline mems from @cs */
	if (!nodes_empty(off_mems)) {
		tmp_mems = cs->mems_allowed;
		mutex_lock(&callback_mutex);
		nodes_andnot(cs->mems_allowed, cs->mems_allowed, off_mems);
		mutex_unlock(&callback_mutex);
		update_tasks_nodemask(cs, &tmp_mems, NULL);
2131
	}
2132

2133 2134
	is_empty = cpumask_empty(cs->cpus_allowed) ||
		nodes_empty(cs->mems_allowed);
2135

2136 2137 2138 2139 2140 2141 2142 2143 2144
	mutex_unlock(&cpuset_mutex);

	/*
	 * If @cs became empty, move tasks to the nearest ancestor with
	 * execution resources.  This is full cgroup operation which will
	 * also call back into cpuset.  Should be done outside any lock.
	 */
	if (is_empty)
		remove_tasks_in_empty_cpuset(cs);
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172

	/* the following may free @cs, should be the last operation */
	css_put(&cs->css);
}

/**
 * schedule_cpuset_propagate_hotplug - schedule hotplug propagation to a cpuset
 * @cs: cpuset of interest
 *
 * Schedule cpuset_propagate_hotplug_workfn() which will update CPU and
 * memory masks according to top_cpuset.
 */
static void schedule_cpuset_propagate_hotplug(struct cpuset *cs)
{
	/*
	 * Pin @cs.  The refcnt will be released when the work item
	 * finishes executing.
	 */
	if (!css_tryget(&cs->css))
		return;

	/*
	 * Queue @cs->hotplug_work.  If already pending, lose the css ref.
	 * cpuset_propagate_hotplug_wq is ordered and propagation will
	 * happen in the order this function is called.
	 */
	if (!queue_work(cpuset_propagate_hotplug_wq, &cs->hotplug_work))
		css_put(&cs->css);
2173 2174
}

2175
/**
2176
 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
2177
 *
2178 2179 2180 2181 2182
 * This function is called after either CPU or memory configuration has
 * changed and updates cpuset accordingly.  The top_cpuset is always
 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
 * order to make cpusets transparent (of no affect) on systems that are
 * actively using CPU hotplug but making no active use of cpusets.
2183
 *
2184 2185 2186
 * Non-root cpusets are only affected by offlining.  If any CPUs or memory
 * nodes have been taken down, cpuset_propagate_hotplug() is invoked on all
 * descendants.
2187
 *
2188 2189
 * Note that CPU offlining during suspend is ignored.  We don't modify
 * cpusets across suspend/resume cycles at all.
2190
 */
2191
static void cpuset_hotplug_workfn(struct work_struct *work)
2192
{
2193 2194 2195 2196
	static cpumask_t new_cpus, tmp_cpus;
	static nodemask_t new_mems, tmp_mems;
	bool cpus_updated, mems_updated;
	bool cpus_offlined, mems_offlined;
2197

2198
	mutex_lock(&cpuset_mutex);
2199

2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
	/* fetch the available cpus/mems and find out which changed how */
	cpumask_copy(&new_cpus, cpu_active_mask);
	new_mems = node_states[N_MEMORY];

	cpus_updated = !cpumask_equal(top_cpuset.cpus_allowed, &new_cpus);
	cpus_offlined = cpumask_andnot(&tmp_cpus, top_cpuset.cpus_allowed,
				       &new_cpus);

	mems_updated = !nodes_equal(top_cpuset.mems_allowed, new_mems);
	nodes_andnot(tmp_mems, top_cpuset.mems_allowed, new_mems);
	mems_offlined = !nodes_empty(tmp_mems);

	/* synchronize cpus_allowed to cpu_active_mask */
	if (cpus_updated) {
		mutex_lock(&callback_mutex);
		cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
		mutex_unlock(&callback_mutex);
		/* we don't mess with cpumasks of tasks in top_cpuset */
	}

	/* synchronize mems_allowed to N_MEMORY */
	if (mems_updated) {
		tmp_mems = top_cpuset.mems_allowed;
		mutex_lock(&callback_mutex);
		top_cpuset.mems_allowed = new_mems;
		mutex_unlock(&callback_mutex);
		update_tasks_nodemask(&top_cpuset, &tmp_mems, NULL);
	}

	/* if cpus or mems went down, we need to propagate to descendants */
	if (cpus_offlined || mems_offlined) {
		struct cpuset *cs;
		LIST_HEAD(queue);

		list_add_tail(&top_cpuset.stack_list, &queue);
		while ((cs = cpuset_next(&queue)))
			if (cs != &top_cpuset)
2237
				schedule_cpuset_propagate_hotplug(cs);
2238
	}
2239

2240
	mutex_unlock(&cpuset_mutex);
2241

2242 2243 2244
	/* wait for propagations to finish */
	flush_workqueue(cpuset_propagate_hotplug_wq);

2245 2246 2247 2248 2249 2250
	/* rebuild sched domains if cpus_allowed has changed */
	if (cpus_updated) {
		struct sched_domain_attr *attr;
		cpumask_var_t *doms;
		int ndoms;

2251
		mutex_lock(&cpuset_mutex);
2252
		ndoms = generate_sched_domains(&doms, &attr);
2253
		mutex_unlock(&cpuset_mutex);
2254 2255 2256 2257 2258 2259 2260

		partition_sched_domains(ndoms, doms, attr);
	}
}

void cpuset_update_active_cpus(bool cpu_online)
{
2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272
	/*
	 * We're inside cpu hotplug critical region which usually nests
	 * inside cgroup synchronization.  Bounce actual hotplug processing
	 * to a work item to avoid reverse locking order.
	 *
	 * We still need to do partition_sched_domains() synchronously;
	 * otherwise, the scheduler will get confused and put tasks to the
	 * dead CPU.  Fall back to the default single domain.
	 * cpuset_hotplug_workfn() will rebuild it as necessary.
	 */
	partition_sched_domains(1, NULL, NULL);
	schedule_work(&cpuset_hotplug_work);
2273 2274
}

2275
#ifdef CONFIG_MEMORY_HOTPLUG
2276
/*
2277 2278
 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
 * Call this routine anytime after node_states[N_MEMORY] changes.
2279
 * See cpuset_update_active_cpus() for CPU hotplug handling.
2280
 */
2281 2282
static int cpuset_track_online_nodes(struct notifier_block *self,
				unsigned long action, void *arg)
2283
{
2284
	schedule_work(&cpuset_hotplug_work);
2285
	return NOTIFY_OK;
2286 2287 2288
}
#endif

L
Linus Torvalds 已提交
2289 2290 2291 2292 2293 2294 2295 2296
/**
 * cpuset_init_smp - initialize cpus_allowed
 *
 * Description: Finish top cpuset after cpu, node maps are initialized
 **/

void __init cpuset_init_smp(void)
{
2297
	cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
2298
	top_cpuset.mems_allowed = node_states[N_MEMORY];
2299

2300
	hotplug_memory_notifier(cpuset_track_online_nodes, 10);
2301 2302 2303 2304

	cpuset_propagate_hotplug_wq =
		alloc_ordered_workqueue("cpuset_hotplug", 0);
	BUG_ON(!cpuset_propagate_hotplug_wq);
L
Linus Torvalds 已提交
2305 2306 2307 2308 2309
}

/**
 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2310
 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
L
Linus Torvalds 已提交
2311
 *
2312
 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
L
Linus Torvalds 已提交
2313
 * attached to the specified @tsk.  Guaranteed to return some non-empty
2314
 * subset of cpu_online_mask, even if this means going outside the
L
Linus Torvalds 已提交
2315 2316 2317
 * tasks cpuset.
 **/

2318
void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
L
Linus Torvalds 已提交
2319
{
2320
	mutex_lock(&callback_mutex);
2321
	task_lock(tsk);
2322
	guarantee_online_cpus(task_cs(tsk), pmask);
2323
	task_unlock(tsk);
2324
	mutex_unlock(&callback_mutex);
L
Linus Torvalds 已提交
2325 2326
}

2327
void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
2328 2329 2330 2331 2332 2333
{
	const struct cpuset *cs;

	rcu_read_lock();
	cs = task_cs(tsk);
	if (cs)
2334
		do_set_cpus_allowed(tsk, cs->cpus_allowed);
2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
	rcu_read_unlock();

	/*
	 * We own tsk->cpus_allowed, nobody can change it under us.
	 *
	 * But we used cs && cs->cpus_allowed lockless and thus can
	 * race with cgroup_attach_task() or update_cpumask() and get
	 * the wrong tsk->cpus_allowed. However, both cases imply the
	 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
	 * which takes task_rq_lock().
	 *
	 * If we are called after it dropped the lock we must see all
	 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
	 * set any mask even if it is not right from task_cs() pov,
	 * the pending set_cpus_allowed_ptr() will fix things.
2350 2351 2352
	 *
	 * select_fallback_rq() will fix things ups and set cpu_possible_mask
	 * if required.
2353 2354 2355
	 */
}

L
Linus Torvalds 已提交
2356 2357
void cpuset_init_current_mems_allowed(void)
{
2358
	nodes_setall(current->mems_allowed);
L
Linus Torvalds 已提交
2359 2360
}

2361 2362 2363 2364 2365 2366
/**
 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
 *
 * Description: Returns the nodemask_t mems_allowed of the cpuset
 * attached to the specified @tsk.  Guaranteed to return some non-empty
2367
 * subset of node_states[N_MEMORY], even if this means going outside the
2368 2369 2370 2371 2372 2373 2374
 * tasks cpuset.
 **/

nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
{
	nodemask_t mask;

2375
	mutex_lock(&callback_mutex);
2376
	task_lock(tsk);
2377
	guarantee_online_mems(task_cs(tsk), &mask);
2378
	task_unlock(tsk);
2379
	mutex_unlock(&callback_mutex);
2380 2381 2382 2383

	return mask;
}

2384
/**
2385 2386
 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
 * @nodemask: the nodemask to be checked
2387
 *
2388
 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
L
Linus Torvalds 已提交
2389
 */
2390
int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
L
Linus Torvalds 已提交
2391
{
2392
	return nodes_intersects(*nodemask, current->mems_allowed);
L
Linus Torvalds 已提交
2393 2394
}

2395
/*
2396 2397 2398 2399
 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
 * mem_hardwall ancestor to the specified cpuset.  Call holding
 * callback_mutex.  If no ancestor is mem_exclusive or mem_hardwall
 * (an unusual configuration), then returns the root cpuset.
2400
 */
2401
static const struct cpuset *nearest_hardwall_ancestor(const struct cpuset *cs)
2402
{
2403
	while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && cs->parent)
2404 2405 2406 2407
		cs = cs->parent;
	return cs;
}

2408
/**
2409 2410
 * cpuset_node_allowed_softwall - Can we allocate on a memory node?
 * @node: is this an allowed node?
2411
 * @gfp_mask: memory allocation flags
2412
 *
2413 2414 2415 2416 2417 2418
 * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
 * set, yes, we can always allocate.  If node is in our task's mems_allowed,
 * yes.  If it's not a __GFP_HARDWALL request and this node is in the nearest
 * hardwalled cpuset ancestor to this task's cpuset, yes.  If the task has been
 * OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
 * flag, yes.
2419 2420
 * Otherwise, no.
 *
2421 2422 2423
 * If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
 * cpuset_node_allowed_hardwall().  Otherwise, cpuset_node_allowed_softwall()
 * might sleep, and might allow a node from an enclosing cpuset.
2424
 *
2425 2426
 * cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
 * cpusets, and never sleeps.
2427 2428 2429 2430 2431 2432 2433
 *
 * The __GFP_THISNODE placement logic is really handled elsewhere,
 * by forcibly using a zonelist starting at a specified node, and by
 * (in get_page_from_freelist()) refusing to consider the zones for
 * any node on the zonelist except the first.  By the time any such
 * calls get to this routine, we should just shut up and say 'yes'.
 *
2434
 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2435 2436
 * and do not allow allocations outside the current tasks cpuset
 * unless the task has been OOM killed as is marked TIF_MEMDIE.
2437
 * GFP_KERNEL allocations are not so marked, so can escape to the
2438
 * nearest enclosing hardwalled ancestor cpuset.
2439
 *
2440 2441 2442 2443 2444 2445 2446
 * Scanning up parent cpusets requires callback_mutex.  The
 * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
 * current tasks mems_allowed came up empty on the first pass over
 * the zonelist.  So only GFP_KERNEL allocations, if all nodes in the
 * cpuset are short of memory, might require taking the callback_mutex
 * mutex.
2447
 *
2448
 * The first call here from mm/page_alloc:get_page_from_freelist()
2449 2450 2451
 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
 * so no allocation on a node outside the cpuset is allowed (unless
 * in interrupt, of course).
2452 2453 2454 2455 2456 2457
 *
 * The second pass through get_page_from_freelist() doesn't even call
 * here for GFP_ATOMIC calls.  For those calls, the __alloc_pages()
 * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
 * in alloc_flags.  That logic and the checks below have the combined
 * affect that:
2458 2459
 *	in_interrupt - any node ok (current task context irrelevant)
 *	GFP_ATOMIC   - any node ok
2460
 *	TIF_MEMDIE   - any node ok
2461
 *	GFP_KERNEL   - any node in enclosing hardwalled cpuset ok
2462
 *	GFP_USER     - only nodes in current tasks mems allowed ok.
2463 2464
 *
 * Rule:
2465
 *    Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
2466 2467
 *    pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
 *    the code that might scan up ancestor cpusets and sleep.
2468
 */
2469
int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
L
Linus Torvalds 已提交
2470
{
2471
	const struct cpuset *cs;	/* current cpuset ancestors */
2472
	int allowed;			/* is allocation in zone z allowed? */
2473

2474
	if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
2475
		return 1;
2476
	might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
2477 2478
	if (node_isset(node, current->mems_allowed))
		return 1;
2479 2480 2481 2482 2483 2484
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return 1;
2485 2486 2487
	if (gfp_mask & __GFP_HARDWALL)	/* If hardwall request, stop here */
		return 0;

2488 2489 2490
	if (current->flags & PF_EXITING) /* Let dying task have memory */
		return 1;

2491
	/* Not hardwall and node outside mems_allowed: scan up cpusets */
2492
	mutex_lock(&callback_mutex);
2493 2494

	task_lock(current);
2495
	cs = nearest_hardwall_ancestor(task_cs(current));
2496 2497
	task_unlock(current);

2498
	allowed = node_isset(node, cs->mems_allowed);
2499
	mutex_unlock(&callback_mutex);
2500
	return allowed;
L
Linus Torvalds 已提交
2501 2502
}

2503
/*
2504 2505
 * cpuset_node_allowed_hardwall - Can we allocate on a memory node?
 * @node: is this an allowed node?
2506 2507
 * @gfp_mask: memory allocation flags
 *
2508 2509 2510 2511 2512
 * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
 * set, yes, we can always allocate.  If node is in our task's mems_allowed,
 * yes.  If the task has been OOM killed and has access to memory reserves as
 * specified by the TIF_MEMDIE flag, yes.
 * Otherwise, no.
2513 2514 2515 2516 2517 2518 2519
 *
 * The __GFP_THISNODE placement logic is really handled elsewhere,
 * by forcibly using a zonelist starting at a specified node, and by
 * (in get_page_from_freelist()) refusing to consider the zones for
 * any node on the zonelist except the first.  By the time any such
 * calls get to this routine, we should just shut up and say 'yes'.
 *
2520 2521
 * Unlike the cpuset_node_allowed_softwall() variant, above,
 * this variant requires that the node be in the current task's
2522 2523 2524 2525
 * mems_allowed or that we're in interrupt.  It does not scan up the
 * cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
 * It never sleeps.
 */
2526
int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
2527 2528 2529 2530 2531
{
	if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
		return 1;
	if (node_isset(node, current->mems_allowed))
		return 1;
D
Daniel Walker 已提交
2532 2533 2534 2535 2536 2537
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return 1;
2538 2539 2540
	return 0;
}

2541
/**
2542 2543
 * cpuset_mem_spread_node() - On which node to begin search for a file page
 * cpuset_slab_spread_node() - On which node to begin search for a slab page
2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
 *
 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
 * tasks in a cpuset with is_spread_page or is_spread_slab set),
 * and if the memory allocation used cpuset_mem_spread_node()
 * to determine on which node to start looking, as it will for
 * certain page cache or slab cache pages such as used for file
 * system buffers and inode caches, then instead of starting on the
 * local node to look for a free page, rather spread the starting
 * node around the tasks mems_allowed nodes.
 *
 * We don't have to worry about the returned node being offline
 * because "it can't happen", and even if it did, it would be ok.
 *
 * The routines calling guarantee_online_mems() are careful to
 * only set nodes in task->mems_allowed that are online.  So it
 * should not be possible for the following code to return an
 * offline node.  But if it did, that would be ok, as this routine
 * is not returning the node where the allocation must be, only
 * the node where the search should start.  The zonelist passed to
 * __alloc_pages() will include all nodes.  If the slab allocator
 * is passed an offline node, it will fall back to the local node.
 * See kmem_cache_alloc_node().
 */

2568
static int cpuset_spread_node(int *rotor)
2569 2570 2571
{
	int node;

2572
	node = next_node(*rotor, current->mems_allowed);
2573 2574
	if (node == MAX_NUMNODES)
		node = first_node(current->mems_allowed);
2575
	*rotor = node;
2576 2577
	return node;
}
2578 2579 2580

int cpuset_mem_spread_node(void)
{
2581 2582 2583 2584
	if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
		current->cpuset_mem_spread_rotor =
			node_random(&current->mems_allowed);

2585 2586 2587 2588 2589
	return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
}

int cpuset_slab_spread_node(void)
{
2590 2591 2592 2593
	if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
		current->cpuset_slab_spread_rotor =
			node_random(&current->mems_allowed);

2594 2595 2596
	return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
}

2597 2598
EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);

2599
/**
2600 2601 2602 2603 2604 2605 2606 2607
 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
 * @tsk1: pointer to task_struct of some task.
 * @tsk2: pointer to task_struct of some other task.
 *
 * Description: Return true if @tsk1's mems_allowed intersects the
 * mems_allowed of @tsk2.  Used by the OOM killer to determine if
 * one of the task's memory usage might impact the memory available
 * to the other.
2608 2609
 **/

2610 2611
int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
				   const struct task_struct *tsk2)
2612
{
2613
	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
2614 2615
}

2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638
/**
 * cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
 * @task: pointer to task_struct of some task.
 *
 * Description: Prints @task's name, cpuset name, and cached copy of its
 * mems_allowed to the kernel log.  Must hold task_lock(task) to allow
 * dereferencing task_cs(task).
 */
void cpuset_print_task_mems_allowed(struct task_struct *tsk)
{
	struct dentry *dentry;

	dentry = task_cs(tsk)->css.cgroup->dentry;
	spin_lock(&cpuset_buffer_lock);
	snprintf(cpuset_name, CPUSET_NAME_LEN,
		 dentry ? (const char *)dentry->d_name.name : "/");
	nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
			   tsk->mems_allowed);
	printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",
	       tsk->comm, cpuset_name, cpuset_nodelist);
	spin_unlock(&cpuset_buffer_lock);
}

2639 2640 2641 2642 2643 2644
/*
 * Collection of memory_pressure is suppressed unless
 * this flag is enabled by writing "1" to the special
 * cpuset file 'memory_pressure_enabled' in the root cpuset.
 */

2645
int cpuset_memory_pressure_enabled __read_mostly;
2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667

/**
 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
 *
 * Keep a running average of the rate of synchronous (direct)
 * page reclaim efforts initiated by tasks in each cpuset.
 *
 * This represents the rate at which some task in the cpuset
 * ran low on memory on all nodes it was allowed to use, and
 * had to enter the kernels page reclaim code in an effort to
 * create more free memory by tossing clean pages or swapping
 * or writing dirty pages.
 *
 * Display to user space in the per-cpuset read-only file
 * "memory_pressure".  Value displayed is an integer
 * representing the recent rate of entry into the synchronous
 * (direct) page reclaim by any task attached to the cpuset.
 **/

void __cpuset_memory_pressure_bump(void)
{
	task_lock(current);
2668
	fmeter_markevent(&task_cs(current)->fmeter);
2669 2670 2671
	task_unlock(current);
}

2672
#ifdef CONFIG_PROC_PID_CPUSET
L
Linus Torvalds 已提交
2673 2674 2675 2676
/*
 * proc_cpuset_show()
 *  - Print tasks cpuset path into seq_file.
 *  - Used for /proc/<pid>/cpuset.
2677 2678
 *  - No need to task_lock(tsk) on this tsk->cpuset reference, as it
 *    doesn't really matter if tsk->cpuset changes after we read it,
2679
 *    and we take cpuset_mutex, keeping cpuset_attach() from changing it
2680
 *    anyway.
L
Linus Torvalds 已提交
2681
 */
P
Paul Jackson 已提交
2682
static int proc_cpuset_show(struct seq_file *m, void *unused_v)
L
Linus Torvalds 已提交
2683
{
2684
	struct pid *pid;
L
Linus Torvalds 已提交
2685 2686
	struct task_struct *tsk;
	char *buf;
2687
	struct cgroup_subsys_state *css;
2688
	int retval;
L
Linus Torvalds 已提交
2689

2690
	retval = -ENOMEM;
L
Linus Torvalds 已提交
2691 2692
	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
2693 2694 2695
		goto out;

	retval = -ESRCH;
2696 2697
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
2698 2699
	if (!tsk)
		goto out_free;
L
Linus Torvalds 已提交
2700

2701
	retval = -EINVAL;
2702
	mutex_lock(&cpuset_mutex);
2703 2704
	css = task_subsys_state(tsk, cpuset_subsys_id);
	retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
L
Linus Torvalds 已提交
2705
	if (retval < 0)
2706
		goto out_unlock;
L
Linus Torvalds 已提交
2707 2708
	seq_puts(m, buf);
	seq_putc(m, '\n');
2709
out_unlock:
2710
	mutex_unlock(&cpuset_mutex);
2711 2712
	put_task_struct(tsk);
out_free:
L
Linus Torvalds 已提交
2713
	kfree(buf);
2714
out:
L
Linus Torvalds 已提交
2715 2716 2717 2718 2719
	return retval;
}

static int cpuset_open(struct inode *inode, struct file *file)
{
2720 2721
	struct pid *pid = PROC_I(inode)->pid;
	return single_open(file, proc_cpuset_show, pid);
L
Linus Torvalds 已提交
2722 2723
}

2724
const struct file_operations proc_cpuset_operations = {
L
Linus Torvalds 已提交
2725 2726 2727 2728 2729
	.open		= cpuset_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};
2730
#endif /* CONFIG_PROC_PID_CPUSET */
L
Linus Torvalds 已提交
2731

2732
/* Display task mems_allowed in /proc/<pid>/status file. */
2733 2734 2735
void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
{
	seq_printf(m, "Mems_allowed:\t");
2736
	seq_nodemask(m, &task->mems_allowed);
2737
	seq_printf(m, "\n");
2738
	seq_printf(m, "Mems_allowed_list:\t");
2739
	seq_nodemask_list(m, &task->mems_allowed);
2740
	seq_printf(m, "\n");
L
Linus Torvalds 已提交
2741
}