cpuset.c 76.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6
/*
 *  kernel/cpuset.c
 *
 *  Processor and Memory placement constraints for sets of tasks.
 *
 *  Copyright (C) 2003 BULL SA.
P
Paul Jackson 已提交
7
 *  Copyright (C) 2004-2007 Silicon Graphics, Inc.
8
 *  Copyright (C) 2006 Google, Inc
L
Linus Torvalds 已提交
9 10 11 12
 *
 *  Portions derived from Patrick Mochel's sysfs code.
 *  sysfs is Copyright (c) 2001-3 Patrick Mochel
 *
13
 *  2003-10-10 Written by Simon Derr.
L
Linus Torvalds 已提交
14
 *  2003-10-22 Updates by Stephen Hemminger.
15
 *  2004 May-July Rework by Paul Jackson.
16
 *  2006 Rework by Paul Menage to use generic cgroups
17 18
 *  2008 Rework of the scheduler domains and CPU hotplug handling
 *       by Max Krasnyansky
L
Linus Torvalds 已提交
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of the Linux
 *  distribution for more details.
 */

#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/cpuset.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/list.h>
37
#include <linux/mempolicy.h>
L
Linus Torvalds 已提交
38
#include <linux/mm.h>
39
#include <linux/memory.h>
40
#include <linux/export.h>
L
Linus Torvalds 已提交
41 42 43 44
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/pagemap.h>
#include <linux/proc_fs.h>
45
#include <linux/rcupdate.h>
L
Linus Torvalds 已提交
46 47
#include <linux/sched.h>
#include <linux/seq_file.h>
48
#include <linux/security.h>
L
Linus Torvalds 已提交
49 50 51 52 53 54 55 56 57
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/stat.h>
#include <linux/string.h>
#include <linux/time.h>
#include <linux/backing-dev.h>
#include <linux/sort.h>

#include <asm/uaccess.h>
A
Arun Sharma 已提交
58
#include <linux/atomic.h>
59
#include <linux/mutex.h>
60 61
#include <linux/workqueue.h>
#include <linux/cgroup.h>
62
#include <linux/wait.h>
L
Linus Torvalds 已提交
63

64 65 66 67 68
/*
 * Tracks how many cpusets are currently defined in system.
 * When there is only one cpuset (the root cpuset) we can
 * short circuit some hooks.
 */
69
int number_of_cpusets __read_mostly;
70

71 72 73 74 75 76 77 78 79
/* See "Frequency meter" comments, below. */

struct fmeter {
	int cnt;		/* unprocessed events count */
	int val;		/* most recent output value */
	time_t time;		/* clock (secs) when val computed */
	spinlock_t lock;	/* guards read or write of above */
};

L
Linus Torvalds 已提交
80
struct cpuset {
81 82
	struct cgroup_subsys_state css;

L
Linus Torvalds 已提交
83
	unsigned long flags;		/* "unsigned long" so bitops work */
84
	cpumask_var_t cpus_allowed;	/* CPUs allowed to tasks in cpuset */
L
Linus Torvalds 已提交
85 86
	nodemask_t mems_allowed;	/* Memory Nodes allowed to tasks */

87 88 89 90 91 92 93 94 95 96 97 98
	/*
	 * This is old Memory Nodes tasks took on.
	 *
	 * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
	 * - A new cpuset's old_mems_allowed is initialized when some
	 *   task is moved into it.
	 * - old_mems_allowed is used in cpuset_migrate_mm() when we change
	 *   cpuset.mems_allowed and have tasks' nodemask updated, and
	 *   then old_mems_allowed is updated to mems_allowed.
	 */
	nodemask_t old_mems_allowed;

99
	struct fmeter fmeter;		/* memory_pressure filter */
P
Paul Jackson 已提交
100

101 102 103 104 105 106
	/*
	 * Tasks are being attached to this cpuset.  Used to prevent
	 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
	 */
	int attach_in_progress;

P
Paul Jackson 已提交
107 108
	/* partition number for rebuild_sched_domains() */
	int pn;
109

110 111
	/* for custom sched domain */
	int relax_domain_level;
L
Linus Torvalds 已提交
112 113
};

114
static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
115
{
116
	return css ? container_of(css, struct cpuset, css) : NULL;
117 118 119 120 121
}

/* Retrieve the cpuset for a task */
static inline struct cpuset *task_cs(struct task_struct *task)
{
122
	return css_cs(task_css(task, cpuset_cgrp_id));
123 124
}

125
static inline struct cpuset *parent_cs(struct cpuset *cs)
T
Tejun Heo 已提交
126
{
T
Tejun Heo 已提交
127
	return css_cs(css_parent(&cs->css));
T
Tejun Heo 已提交
128 129
}

130 131 132 133 134 135 136 137 138 139 140 141 142
#ifdef CONFIG_NUMA
static inline bool task_has_mempolicy(struct task_struct *task)
{
	return task->mempolicy;
}
#else
static inline bool task_has_mempolicy(struct task_struct *task)
{
	return false;
}
#endif


L
Linus Torvalds 已提交
143 144
/* bits in struct cpuset flags field */
typedef enum {
T
Tejun Heo 已提交
145
	CS_ONLINE,
L
Linus Torvalds 已提交
146 147
	CS_CPU_EXCLUSIVE,
	CS_MEM_EXCLUSIVE,
148
	CS_MEM_HARDWALL,
149
	CS_MEMORY_MIGRATE,
P
Paul Jackson 已提交
150
	CS_SCHED_LOAD_BALANCE,
151 152
	CS_SPREAD_PAGE,
	CS_SPREAD_SLAB,
L
Linus Torvalds 已提交
153 154 155
} cpuset_flagbits_t;

/* convenient tests for these bits */
T
Tejun Heo 已提交
156 157 158 159 160
static inline bool is_cpuset_online(const struct cpuset *cs)
{
	return test_bit(CS_ONLINE, &cs->flags);
}

L
Linus Torvalds 已提交
161 162
static inline int is_cpu_exclusive(const struct cpuset *cs)
{
163
	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
L
Linus Torvalds 已提交
164 165 166 167
}

static inline int is_mem_exclusive(const struct cpuset *cs)
{
168
	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
L
Linus Torvalds 已提交
169 170
}

171 172 173 174 175
static inline int is_mem_hardwall(const struct cpuset *cs)
{
	return test_bit(CS_MEM_HARDWALL, &cs->flags);
}

P
Paul Jackson 已提交
176 177 178 179 180
static inline int is_sched_load_balance(const struct cpuset *cs)
{
	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
}

181 182
static inline int is_memory_migrate(const struct cpuset *cs)
{
183
	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
184 185
}

186 187 188 189 190 191 192 193 194 195
static inline int is_spread_page(const struct cpuset *cs)
{
	return test_bit(CS_SPREAD_PAGE, &cs->flags);
}

static inline int is_spread_slab(const struct cpuset *cs)
{
	return test_bit(CS_SPREAD_SLAB, &cs->flags);
}

L
Linus Torvalds 已提交
196
static struct cpuset top_cpuset = {
T
Tejun Heo 已提交
197 198
	.flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
		  (1 << CS_MEM_EXCLUSIVE)),
L
Linus Torvalds 已提交
199 200
};

201 202 203
/**
 * cpuset_for_each_child - traverse online children of a cpuset
 * @child_cs: loop cursor pointing to the current child
204
 * @pos_css: used for iteration
205 206 207 208 209
 * @parent_cs: target cpuset to walk children of
 *
 * Walk @child_cs through the online children of @parent_cs.  Must be used
 * with RCU read locked.
 */
210 211 212
#define cpuset_for_each_child(child_cs, pos_css, parent_cs)		\
	css_for_each_child((pos_css), &(parent_cs)->css)		\
		if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
213

214 215 216
/**
 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
 * @des_cs: loop cursor pointing to the current descendant
217
 * @pos_css: used for iteration
218 219 220
 * @root_cs: target cpuset to walk ancestor of
 *
 * Walk @des_cs through the online descendants of @root_cs.  Must be used
221
 * with RCU read locked.  The caller may modify @pos_css by calling
222 223
 * css_rightmost_descendant() to skip subtree.  @root_cs is included in the
 * iteration and the first node to be visited.
224
 */
225 226 227
#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs)	\
	css_for_each_descendant_pre((pos_css), &(root_cs)->css)		\
		if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
228

L
Linus Torvalds 已提交
229
/*
230 231 232 233 234 235 236 237 238 239 240 241 242 243
 * There are two global mutexes guarding cpuset structures - cpuset_mutex
 * and callback_mutex.  The latter may nest inside the former.  We also
 * require taking task_lock() when dereferencing a task's cpuset pointer.
 * See "The task_lock() exception", at the end of this comment.
 *
 * A task must hold both mutexes to modify cpusets.  If a task holds
 * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it
 * is the only task able to also acquire callback_mutex and be able to
 * modify cpusets.  It can perform various checks on the cpuset structure
 * first, knowing nothing will change.  It can also allocate memory while
 * just holding cpuset_mutex.  While it is performing these checks, various
 * callback routines can briefly acquire callback_mutex to query cpusets.
 * Once it is ready to make the changes, it takes callback_mutex, blocking
 * everyone else.
244 245
 *
 * Calls to the kernel memory allocator can not be made while holding
246
 * callback_mutex, as that would risk double tripping on callback_mutex
247 248 249
 * from one of the callbacks into the cpuset code from within
 * __alloc_pages().
 *
250
 * If a task is only holding callback_mutex, then it has read-only
251 252
 * access to cpusets.
 *
253 254 255
 * Now, the task_struct fields mems_allowed and mempolicy may be changed
 * by other task, we use alloc_lock in the task_struct fields to protect
 * them.
256
 *
257
 * The cpuset_common_file_read() handlers only hold callback_mutex across
258 259 260
 * small pieces of code, such as when reading out possibly multi-word
 * cpumasks and nodemasks.
 *
261 262
 * Accessing a task's cpuset should be done in accordance with the
 * guidelines for accessing subsystem state in kernel/cgroup.c
L
Linus Torvalds 已提交
263 264
 */

265
static DEFINE_MUTEX(cpuset_mutex);
266
static DEFINE_MUTEX(callback_mutex);
267

268 269 270 271 272 273
/*
 * CPU / memory hotplug is handled asynchronously.
 */
static void cpuset_hotplug_workfn(struct work_struct *work);
static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);

274 275
static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);

276 277
/*
 * This is ugly, but preserves the userspace API for existing cpuset
278
 * users. If someone tries to mount the "cpuset" filesystem, we
279 280
 * silently switch it to mount "cgroup" instead
 */
A
Al Viro 已提交
281 282
static struct dentry *cpuset_mount(struct file_system_type *fs_type,
			 int flags, const char *unused_dev_name, void *data)
L
Linus Torvalds 已提交
283
{
284
	struct file_system_type *cgroup_fs = get_fs_type("cgroup");
A
Al Viro 已提交
285
	struct dentry *ret = ERR_PTR(-ENODEV);
286 287 288 289
	if (cgroup_fs) {
		char mountopts[] =
			"cpuset,noprefix,"
			"release_agent=/sbin/cpuset_release_agent";
A
Al Viro 已提交
290 291
		ret = cgroup_fs->mount(cgroup_fs, flags,
					   unused_dev_name, mountopts);
292 293 294
		put_filesystem(cgroup_fs);
	}
	return ret;
L
Linus Torvalds 已提交
295 296 297 298
}

static struct file_system_type cpuset_fs_type = {
	.name = "cpuset",
A
Al Viro 已提交
299
	.mount = cpuset_mount,
L
Linus Torvalds 已提交
300 301 302
};

/*
303
 * Return in pmask the portion of a cpusets's cpus_allowed that
L
Linus Torvalds 已提交
304
 * are online.  If none are online, walk up the cpuset hierarchy
305 306
 * until we find one that does have some online cpus.  The top
 * cpuset always has some cpus online.
L
Linus Torvalds 已提交
307 308
 *
 * One way or another, we guarantee to return some non-empty subset
309
 * of cpu_online_mask.
L
Linus Torvalds 已提交
310
 *
311
 * Call with callback_mutex held.
L
Linus Torvalds 已提交
312
 */
313
static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
L
Linus Torvalds 已提交
314
{
315
	while (!cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
T
Tejun Heo 已提交
316
		cs = parent_cs(cs);
317
	cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);
L
Linus Torvalds 已提交
318 319 320 321
}

/*
 * Return in *pmask the portion of a cpusets's mems_allowed that
322 323
 * are online, with memory.  If none are online with memory, walk
 * up the cpuset hierarchy until we find one that does have some
324
 * online mems.  The top cpuset always has some mems online.
L
Linus Torvalds 已提交
325 326
 *
 * One way or another, we guarantee to return some non-empty subset
327
 * of node_states[N_MEMORY].
L
Linus Torvalds 已提交
328
 *
329
 * Call with callback_mutex held.
L
Linus Torvalds 已提交
330
 */
331
static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
L
Linus Torvalds 已提交
332
{
333
	while (!nodes_intersects(cs->mems_allowed, node_states[N_MEMORY]))
T
Tejun Heo 已提交
334
		cs = parent_cs(cs);
335
	nodes_and(*pmask, cs->mems_allowed, node_states[N_MEMORY]);
L
Linus Torvalds 已提交
336 337
}

338 339 340
/*
 * update task's spread flag if cpuset's page/slab spread flag is set
 *
341
 * Called with callback_mutex/cpuset_mutex held
342 343 344 345 346 347 348 349 350 351 352 353 354 355
 */
static void cpuset_update_task_spread_flag(struct cpuset *cs,
					struct task_struct *tsk)
{
	if (is_spread_page(cs))
		tsk->flags |= PF_SPREAD_PAGE;
	else
		tsk->flags &= ~PF_SPREAD_PAGE;
	if (is_spread_slab(cs))
		tsk->flags |= PF_SPREAD_SLAB;
	else
		tsk->flags &= ~PF_SPREAD_SLAB;
}

L
Linus Torvalds 已提交
356 357 358 359 360
/*
 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
 *
 * One cpuset is a subset of another if all its allowed CPUs and
 * Memory Nodes are a subset of the other, and its exclusive flags
361
 * are only set if the other's are set.  Call holding cpuset_mutex.
L
Linus Torvalds 已提交
362 363 364 365
 */

static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
{
366
	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
L
Linus Torvalds 已提交
367 368 369 370 371
		nodes_subset(p->mems_allowed, q->mems_allowed) &&
		is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
		is_mem_exclusive(p) <= is_mem_exclusive(q);
}

372 373 374 375
/**
 * alloc_trial_cpuset - allocate a trial cpuset
 * @cs: the cpuset that the trial cpuset duplicates
 */
376
static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
377
{
378 379 380 381 382 383 384 385 386 387 388 389 390
	struct cpuset *trial;

	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
	if (!trial)
		return NULL;

	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {
		kfree(trial);
		return NULL;
	}
	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);

	return trial;
391 392 393 394 395 396 397 398
}

/**
 * free_trial_cpuset - free the trial cpuset
 * @trial: the trial cpuset to be freed
 */
static void free_trial_cpuset(struct cpuset *trial)
{
399
	free_cpumask_var(trial->cpus_allowed);
400 401 402
	kfree(trial);
}

L
Linus Torvalds 已提交
403 404 405 406 407 408 409
/*
 * validate_change() - Used to validate that any proposed cpuset change
 *		       follows the structural rules for cpusets.
 *
 * If we replaced the flag and mask values of the current cpuset
 * (cur) with those values in the trial cpuset (trial), would
 * our various subset and exclusive rules still be valid?  Presumes
410
 * cpuset_mutex held.
L
Linus Torvalds 已提交
411 412 413 414 415 416 417 418 419 420 421 422
 *
 * 'cur' is the address of an actual, in-use cpuset.  Operations
 * such as list traversal that depend on the actual address of the
 * cpuset in the list must use cur below, not trial.
 *
 * 'trial' is the address of bulk structure copy of cur, with
 * perhaps one or more of the fields cpus_allowed, mems_allowed,
 * or flags changed to new, trial values.
 *
 * Return 0 if valid, -errno if not.
 */

423
static int validate_change(struct cpuset *cur, struct cpuset *trial)
L
Linus Torvalds 已提交
424
{
425
	struct cgroup_subsys_state *css;
L
Linus Torvalds 已提交
426
	struct cpuset *c, *par;
427 428 429
	int ret;

	rcu_read_lock();
L
Linus Torvalds 已提交
430 431

	/* Each of our child cpusets must be a subset of us */
432
	ret = -EBUSY;
433
	cpuset_for_each_child(c, css, cur)
434 435
		if (!is_cpuset_subset(c, trial))
			goto out;
L
Linus Torvalds 已提交
436 437

	/* Remaining checks don't apply to root cpuset */
438
	ret = 0;
439
	if (cur == &top_cpuset)
440
		goto out;
L
Linus Torvalds 已提交
441

T
Tejun Heo 已提交
442
	par = parent_cs(cur);
443

L
Linus Torvalds 已提交
444
	/* We must be a subset of our parent cpuset */
445
	ret = -EACCES;
L
Linus Torvalds 已提交
446
	if (!is_cpuset_subset(trial, par))
447
		goto out;
L
Linus Torvalds 已提交
448

449 450 451 452
	/*
	 * If either I or some sibling (!= me) is exclusive, we can't
	 * overlap
	 */
453
	ret = -EINVAL;
454
	cpuset_for_each_child(c, css, par) {
L
Linus Torvalds 已提交
455 456
		if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
		    c != cur &&
457
		    cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
458
			goto out;
L
Linus Torvalds 已提交
459 460 461
		if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
		    c != cur &&
		    nodes_intersects(trial->mems_allowed, c->mems_allowed))
462
			goto out;
L
Linus Torvalds 已提交
463 464
	}

465 466
	/*
	 * Cpusets with tasks - existing or newly being attached - can't
467
	 * be changed to have empty cpus_allowed or mems_allowed.
468
	 */
469
	ret = -ENOSPC;
470 471 472 473 474 475 476 477
	if ((cgroup_task_count(cur->css.cgroup) || cur->attach_in_progress)) {
		if (!cpumask_empty(cur->cpus_allowed) &&
		    cpumask_empty(trial->cpus_allowed))
			goto out;
		if (!nodes_empty(cur->mems_allowed) &&
		    nodes_empty(trial->mems_allowed))
			goto out;
	}
478

479 480 481 482
	ret = 0;
out:
	rcu_read_unlock();
	return ret;
L
Linus Torvalds 已提交
483 484
}

485
#ifdef CONFIG_SMP
P
Paul Jackson 已提交
486
/*
487
 * Helper routine for generate_sched_domains().
P
Paul Jackson 已提交
488 489 490 491
 * Do cpusets a, b have overlapping cpus_allowed masks?
 */
static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
{
492
	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);
P
Paul Jackson 已提交
493 494
}

495 496 497 498 499 500 501 502
static void
update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
{
	if (dattr->relax_domain_level < c->relax_domain_level)
		dattr->relax_domain_level = c->relax_domain_level;
	return;
}

503 504
static void update_domain_attr_tree(struct sched_domain_attr *dattr,
				    struct cpuset *root_cs)
505
{
506
	struct cpuset *cp;
507
	struct cgroup_subsys_state *pos_css;
508

509
	rcu_read_lock();
510
	cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
511 512 513
		if (cp == root_cs)
			continue;

514 515
		/* skip the whole subtree if @cp doesn't have any CPU */
		if (cpumask_empty(cp->cpus_allowed)) {
516
			pos_css = css_rightmost_descendant(pos_css);
517
			continue;
518
		}
519 520 521 522

		if (is_sched_load_balance(cp))
			update_domain_attr(dattr, cp);
	}
523
	rcu_read_unlock();
524 525
}

P
Paul Jackson 已提交
526
/*
527 528 529 530 531
 * generate_sched_domains()
 *
 * This function builds a partial partition of the systems CPUs
 * A 'partial partition' is a set of non-overlapping subsets whose
 * union is a subset of that set.
532
 * The output of this function needs to be passed to kernel/sched/core.c
533 534 535
 * partition_sched_domains() routine, which will rebuild the scheduler's
 * load balancing domains (sched domains) as specified by that partial
 * partition.
P
Paul Jackson 已提交
536
 *
L
Li Zefan 已提交
537
 * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
P
Paul Jackson 已提交
538 539 540 541 542 543 544
 * for a background explanation of this.
 *
 * Does not return errors, on the theory that the callers of this
 * routine would rather not worry about failures to rebuild sched
 * domains when operating in the severe memory shortage situations
 * that could cause allocation failures below.
 *
545
 * Must be called with cpuset_mutex held.
P
Paul Jackson 已提交
546 547
 *
 * The three key local variables below are:
548
 *    q  - a linked-list queue of cpuset pointers, used to implement a
P
Paul Jackson 已提交
549 550 551 552 553 554 555 556 557 558 559 560
 *	   top-down scan of all cpusets.  This scan loads a pointer
 *	   to each cpuset marked is_sched_load_balance into the
 *	   array 'csa'.  For our purposes, rebuilding the schedulers
 *	   sched domains, we can ignore !is_sched_load_balance cpusets.
 *  csa  - (for CpuSet Array) Array of pointers to all the cpusets
 *	   that need to be load balanced, for convenient iterative
 *	   access by the subsequent code that finds the best partition,
 *	   i.e the set of domains (subsets) of CPUs such that the
 *	   cpus_allowed of every cpuset marked is_sched_load_balance
 *	   is a subset of one of these domains, while there are as
 *	   many such domains as possible, each as small as possible.
 * doms  - Conversion of 'csa' to an array of cpumasks, for passing to
561
 *	   the kernel/sched/core.c routine partition_sched_domains() in a
P
Paul Jackson 已提交
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
 *	   convenient format, that can be easily compared to the prior
 *	   value to determine what partition elements (sched domains)
 *	   were changed (added or removed.)
 *
 * Finding the best partition (set of domains):
 *	The triple nested loops below over i, j, k scan over the
 *	load balanced cpusets (using the array of cpuset pointers in
 *	csa[]) looking for pairs of cpusets that have overlapping
 *	cpus_allowed, but which don't have the same 'pn' partition
 *	number and gives them in the same partition number.  It keeps
 *	looping on the 'restart' label until it can no longer find
 *	any such pairs.
 *
 *	The union of the cpus_allowed masks from the set of
 *	all cpusets having the same 'pn' value then form the one
 *	element of the partition (one sched domain) to be passed to
 *	partition_sched_domains().
 */
580
static int generate_sched_domains(cpumask_var_t **domains,
581
			struct sched_domain_attr **attributes)
P
Paul Jackson 已提交
582 583 584 585 586
{
	struct cpuset *cp;	/* scans q */
	struct cpuset **csa;	/* array of all cpuset ptrs */
	int csn;		/* how many cpuset ptrs in csa so far */
	int i, j, k;		/* indices for partition finding loops */
587
	cpumask_var_t *doms;	/* resulting partition; i.e. sched domains */
588
	struct sched_domain_attr *dattr;  /* attributes for custom domains */
589
	int ndoms = 0;		/* number of sched domains in result */
590
	int nslot;		/* next empty doms[] struct cpumask slot */
591
	struct cgroup_subsys_state *pos_css;
P
Paul Jackson 已提交
592 593

	doms = NULL;
594
	dattr = NULL;
595
	csa = NULL;
P
Paul Jackson 已提交
596 597 598

	/* Special case for the 99% of systems with one, full, sched domain */
	if (is_sched_load_balance(&top_cpuset)) {
599 600
		ndoms = 1;
		doms = alloc_sched_domains(ndoms);
P
Paul Jackson 已提交
601
		if (!doms)
602 603
			goto done;

604 605 606
		dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
		if (dattr) {
			*dattr = SD_ATTR_INIT;
607
			update_domain_attr_tree(dattr, &top_cpuset);
608
		}
609
		cpumask_copy(doms[0], top_cpuset.cpus_allowed);
610 611

		goto done;
P
Paul Jackson 已提交
612 613 614 615 616 617 618
	}

	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
	if (!csa)
		goto done;
	csn = 0;

619
	rcu_read_lock();
620
	cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
621 622
		if (cp == &top_cpuset)
			continue;
623
		/*
624 625 626 627 628 629
		 * Continue traversing beyond @cp iff @cp has some CPUs and
		 * isn't load balancing.  The former is obvious.  The
		 * latter: All child cpusets contain a subset of the
		 * parent's cpus, so just skip them, and then we call
		 * update_domain_attr_tree() to calc relax_domain_level of
		 * the corresponding sched domain.
630
		 */
631 632
		if (!cpumask_empty(cp->cpus_allowed) &&
		    !is_sched_load_balance(cp))
633
			continue;
634

635 636 637 638
		if (is_sched_load_balance(cp))
			csa[csn++] = cp;

		/* skip @cp's subtree */
639
		pos_css = css_rightmost_descendant(pos_css);
640 641
	}
	rcu_read_unlock();
P
Paul Jackson 已提交
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669

	for (i = 0; i < csn; i++)
		csa[i]->pn = i;
	ndoms = csn;

restart:
	/* Find the best partition (set of sched domains) */
	for (i = 0; i < csn; i++) {
		struct cpuset *a = csa[i];
		int apn = a->pn;

		for (j = 0; j < csn; j++) {
			struct cpuset *b = csa[j];
			int bpn = b->pn;

			if (apn != bpn && cpusets_overlap(a, b)) {
				for (k = 0; k < csn; k++) {
					struct cpuset *c = csa[k];

					if (c->pn == bpn)
						c->pn = apn;
				}
				ndoms--;	/* one less element */
				goto restart;
			}
		}
	}

670 671 672 673
	/*
	 * Now we know how many domains to create.
	 * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
	 */
674
	doms = alloc_sched_domains(ndoms);
675
	if (!doms)
676 677 678 679 680 681
		goto done;

	/*
	 * The rest of the code, including the scheduler, can deal with
	 * dattr==NULL case. No need to abort if alloc fails.
	 */
682
	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
P
Paul Jackson 已提交
683 684 685

	for (nslot = 0, i = 0; i < csn; i++) {
		struct cpuset *a = csa[i];
686
		struct cpumask *dp;
P
Paul Jackson 已提交
687 688
		int apn = a->pn;

689 690 691 692 693
		if (apn < 0) {
			/* Skip completed partitions */
			continue;
		}

694
		dp = doms[nslot];
695 696 697 698 699 700 701 702 703 704

		if (nslot == ndoms) {
			static int warnings = 10;
			if (warnings) {
				printk(KERN_WARNING
				 "rebuild_sched_domains confused:"
				  " nslot %d, ndoms %d, csn %d, i %d,"
				  " apn %d\n",
				  nslot, ndoms, csn, i, apn);
				warnings--;
P
Paul Jackson 已提交
705
			}
706 707
			continue;
		}
P
Paul Jackson 已提交
708

709
		cpumask_clear(dp);
710 711 712 713 714 715
		if (dattr)
			*(dattr + nslot) = SD_ATTR_INIT;
		for (j = i; j < csn; j++) {
			struct cpuset *b = csa[j];

			if (apn == b->pn) {
716
				cpumask_or(dp, dp, b->cpus_allowed);
717 718 719 720 721
				if (dattr)
					update_domain_attr_tree(dattr + nslot, b);

				/* Done with this partition */
				b->pn = -1;
P
Paul Jackson 已提交
722 723
			}
		}
724
		nslot++;
P
Paul Jackson 已提交
725 726 727
	}
	BUG_ON(nslot != ndoms);

728 729 730
done:
	kfree(csa);

731 732 733 734 735 736 737
	/*
	 * Fallback to the default domain if kmalloc() failed.
	 * See comments in partition_sched_domains().
	 */
	if (doms == NULL)
		ndoms = 1;

738 739 740 741 742 743 744 745
	*domains    = doms;
	*attributes = dattr;
	return ndoms;
}

/*
 * Rebuild scheduler domains.
 *
746 747 748 749 750
 * If the flag 'sched_load_balance' of any cpuset with non-empty
 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
 * which has that flag enabled, or if any cpuset with a non-empty
 * 'cpus' is removed, then call this routine to rebuild the
 * scheduler's dynamic sched domains.
751
 *
752
 * Call with cpuset_mutex held.  Takes get_online_cpus().
753
 */
754
static void rebuild_sched_domains_locked(void)
755 756
{
	struct sched_domain_attr *attr;
757
	cpumask_var_t *doms;
758 759
	int ndoms;

760
	lockdep_assert_held(&cpuset_mutex);
761
	get_online_cpus();
762

763 764 765 766 767 768 769 770
	/*
	 * We have raced with CPU hotplug. Don't do anything to avoid
	 * passing doms with offlined cpu to partition_sched_domains().
	 * Anyways, hotplug work item will rebuild sched domains.
	 */
	if (!cpumask_equal(top_cpuset.cpus_allowed, cpu_active_mask))
		goto out;

771 772 773 774 775
	/* Generate domain masks and attrs */
	ndoms = generate_sched_domains(&doms, &attr);

	/* Have scheduler rebuild the domains */
	partition_sched_domains(ndoms, doms, attr);
776
out:
777
	put_online_cpus();
778
}
779
#else /* !CONFIG_SMP */
780
static void rebuild_sched_domains_locked(void)
781 782 783
{
}
#endif /* CONFIG_SMP */
P
Paul Jackson 已提交
784

785 786
void rebuild_sched_domains(void)
{
787
	mutex_lock(&cpuset_mutex);
788
	rebuild_sched_domains_locked();
789
	mutex_unlock(&cpuset_mutex);
P
Paul Jackson 已提交
790 791
}

792 793 794
/*
 * effective_cpumask_cpuset - return nearest ancestor with non-empty cpus
 * @cs: the cpuset in interest
C
Cliff Wickman 已提交
795
 *
796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
 * A cpuset's effective cpumask is the cpumask of the nearest ancestor
 * with non-empty cpus. We use effective cpumask whenever:
 * - we update tasks' cpus_allowed. (they take on the ancestor's cpumask
 *   if the cpuset they reside in has no cpus)
 * - we want to retrieve task_cs(tsk)'s cpus_allowed.
 *
 * Called with cpuset_mutex held. cpuset_cpus_allowed_fallback() is an
 * exception. See comments there.
 */
static struct cpuset *effective_cpumask_cpuset(struct cpuset *cs)
{
	while (cpumask_empty(cs->cpus_allowed))
		cs = parent_cs(cs);
	return cs;
}

/*
 * effective_nodemask_cpuset - return nearest ancestor with non-empty mems
 * @cs: the cpuset in interest
 *
 * A cpuset's effective nodemask is the nodemask of the nearest ancestor
 * with non-empty memss. We use effective nodemask whenever:
 * - we update tasks' mems_allowed. (they take on the ancestor's nodemask
 *   if the cpuset they reside in has no mems)
 * - we want to retrieve task_cs(tsk)'s mems_allowed.
 *
 * Called with cpuset_mutex held.
823
 */
824
static struct cpuset *effective_nodemask_cpuset(struct cpuset *cs)
C
Cliff Wickman 已提交
825
{
826 827 828
	while (nodes_empty(cs->mems_allowed))
		cs = parent_cs(cs);
	return cs;
C
Cliff Wickman 已提交
829
}
830

C
Cliff Wickman 已提交
831 832 833
/**
 * cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
 * @tsk: task to test
T
Tejun Heo 已提交
834
 * @data: cpuset to @tsk belongs to
C
Cliff Wickman 已提交
835
 *
836 837
 * Called by css_scan_tasks() for each task in a cgroup whose cpus_allowed
 * mask needs to be changed.
C
Cliff Wickman 已提交
838 839
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
840
 * holding cpuset_mutex at this point.
C
Cliff Wickman 已提交
841
 */
T
Tejun Heo 已提交
842
static void cpuset_change_cpumask(struct task_struct *tsk, void *data)
C
Cliff Wickman 已提交
843
{
T
Tejun Heo 已提交
844 845
	struct cpuset *cs = data;
	struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
846 847

	set_cpus_allowed_ptr(tsk, cpus_cs->cpus_allowed);
C
Cliff Wickman 已提交
848 849
}

850 851 852
/**
 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
853
 * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
854
 *
855
 * Called with cpuset_mutex held
856
 *
857
 * The css_scan_tasks() function will scan all the tasks in a cgroup,
858 859
 * calling callback functions for each.
 *
860
 * No return value. It's guaranteed that css_scan_tasks() always returns 0
861
 * if @heap != NULL.
862
 */
863
static void update_tasks_cpumask(struct cpuset *cs, struct ptr_heap *heap)
864
{
865
	css_scan_tasks(&cs->css, NULL, cpuset_change_cpumask, cs, heap);
866 867
}

868 869 870 871
/*
 * update_tasks_cpumask_hier - Update the cpumasks of tasks in the hierarchy.
 * @root_cs: the root cpuset of the hierarchy
 * @update_root: update root cpuset or not?
872
 * @heap: the heap used by css_scan_tasks()
873 874 875 876 877 878 879 880 881 882
 *
 * This will update cpumasks of tasks in @root_cs and all other empty cpusets
 * which take on cpumask of @root_cs.
 *
 * Called with cpuset_mutex held
 */
static void update_tasks_cpumask_hier(struct cpuset *root_cs,
				      bool update_root, struct ptr_heap *heap)
{
	struct cpuset *cp;
883
	struct cgroup_subsys_state *pos_css;
884 885

	rcu_read_lock();
886
	cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
887 888 889 890 891 892 893 894 895
		if (cp == root_cs) {
			if (!update_root)
				continue;
		} else {
			/* skip the whole subtree if @cp have some CPU */
			if (!cpumask_empty(cp->cpus_allowed)) {
				pos_css = css_rightmost_descendant(pos_css);
				continue;
			}
896 897 898 899 900 901 902 903 904 905 906 907 908
		}
		if (!css_tryget(&cp->css))
			continue;
		rcu_read_unlock();

		update_tasks_cpumask(cp, heap);

		rcu_read_lock();
		css_put(&cp->css);
	}
	rcu_read_unlock();
}

C
Cliff Wickman 已提交
909 910 911 912 913
/**
 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
 * @cs: the cpuset to consider
 * @buf: buffer of cpu numbers written to this cpuset
 */
914 915
static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
			  const char *buf)
L
Linus Torvalds 已提交
916
{
917
	struct ptr_heap heap;
C
Cliff Wickman 已提交
918 919
	int retval;
	int is_load_balanced;
L
Linus Torvalds 已提交
920

921
	/* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
922 923 924
	if (cs == &top_cpuset)
		return -EACCES;

925
	/*
926
	 * An empty cpus_allowed is ok only if the cpuset has no tasks.
927 928 929
	 * Since cpulist_parse() fails on an empty mask, we special case
	 * that parsing.  The validate_change() call ensures that cpusets
	 * with tasks have cpus.
930
	 */
931
	if (!*buf) {
932
		cpumask_clear(trialcs->cpus_allowed);
933
	} else {
934
		retval = cpulist_parse(buf, trialcs->cpus_allowed);
935 936
		if (retval < 0)
			return retval;
937

938
		if (!cpumask_subset(trialcs->cpus_allowed, cpu_active_mask))
939
			return -EINVAL;
940
	}
P
Paul Jackson 已提交
941

P
Paul Menage 已提交
942
	/* Nothing to do if the cpus didn't change */
943
	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
P
Paul Menage 已提交
944
		return 0;
C
Cliff Wickman 已提交
945

946 947 948 949
	retval = validate_change(cs, trialcs);
	if (retval < 0)
		return retval;

950 951 952 953
	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (retval)
		return retval;

954
	is_load_balanced = is_sched_load_balance(trialcs);
P
Paul Jackson 已提交
955

956
	mutex_lock(&callback_mutex);
957
	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
958
	mutex_unlock(&callback_mutex);
P
Paul Jackson 已提交
959

960
	update_tasks_cpumask_hier(cs, true, &heap);
961 962

	heap_free(&heap);
C
Cliff Wickman 已提交
963

P
Paul Menage 已提交
964
	if (is_load_balanced)
965
		rebuild_sched_domains_locked();
966
	return 0;
L
Linus Torvalds 已提交
967 968
}

969 970 971 972 973 974 975 976
/*
 * cpuset_migrate_mm
 *
 *    Migrate memory region from one set of nodes to another.
 *
 *    Temporarilly set tasks mems_allowed to target nodes of migration,
 *    so that the migration code can allocate pages on these nodes.
 *
977
 *    Call holding cpuset_mutex, so current's cpuset won't change
978
 *    during this call, as manage_mutex holds off any cpuset_attach()
979 980
 *    calls.  Therefore we don't need to take task_lock around the
 *    call to guarantee_online_mems(), as we know no one is changing
981
 *    our task's cpuset.
982 983 984 985 986 987 988 989 990 991 992
 *
 *    While the mm_struct we are migrating is typically from some
 *    other task, the task_struct mems_allowed that we are hacking
 *    is for our current task, which must allocate new pages for that
 *    migrating memory region.
 */

static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
							const nodemask_t *to)
{
	struct task_struct *tsk = current;
993
	struct cpuset *mems_cs;
994 995 996 997 998

	tsk->mems_allowed = *to;

	do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);

999 1000
	mems_cs = effective_nodemask_cpuset(task_cs(tsk));
	guarantee_online_mems(mems_cs, &tsk->mems_allowed);
1001 1002
}

1003
/*
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
 * @tsk: the task to change
 * @newmems: new nodes that the task will be set
 *
 * In order to avoid seeing no nodes if the old and new nodes are disjoint,
 * we structure updates as setting all new allowed nodes, then clearing newly
 * disallowed ones.
 */
static void cpuset_change_task_nodemask(struct task_struct *tsk,
					nodemask_t *newmems)
{
1015
	bool need_loop;
1016

1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return;
	if (current->flags & PF_EXITING) /* Let dying task have memory */
		return;

	task_lock(tsk);
1027 1028 1029 1030 1031 1032 1033 1034
	/*
	 * Determine if a loop is necessary if another thread is doing
	 * get_mems_allowed().  If at least one node remains unchanged and
	 * tsk does not have a mempolicy, then an empty nodemask will not be
	 * possible when mems_allowed is larger than a word.
	 */
	need_loop = task_has_mempolicy(tsk) ||
			!nodes_intersects(*newmems, tsk->mems_allowed);
1035

1036 1037
	if (need_loop) {
		local_irq_disable();
1038
		write_seqcount_begin(&tsk->mems_allowed_seq);
1039
	}
1040

1041 1042
	nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);
1043 1044

	mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
1045
	tsk->mems_allowed = *newmems;
1046

1047
	if (need_loop) {
1048
		write_seqcount_end(&tsk->mems_allowed_seq);
1049 1050
		local_irq_enable();
	}
1051

1052
	task_unlock(tsk);
1053 1054
}

T
Tejun Heo 已提交
1055 1056 1057 1058 1059
struct cpuset_change_nodemask_arg {
	struct cpuset		*cs;
	nodemask_t		*newmems;
};

1060 1061 1062
/*
 * Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
 * of it to cpuset's new mems_allowed, and migrate pages to new nodes if
1063
 * memory_migrate flag is set. Called with cpuset_mutex held.
1064
 */
T
Tejun Heo 已提交
1065
static void cpuset_change_nodemask(struct task_struct *p, void *data)
1066
{
T
Tejun Heo 已提交
1067 1068
	struct cpuset_change_nodemask_arg *arg = data;
	struct cpuset *cs = arg->cs;
1069 1070
	struct mm_struct *mm;
	int migrate;
1071

T
Tejun Heo 已提交
1072
	cpuset_change_task_nodemask(p, arg->newmems);
1073

1074 1075 1076 1077 1078 1079 1080 1081
	mm = get_task_mm(p);
	if (!mm)
		return;

	migrate = is_memory_migrate(cs);

	mpol_rebind_mm(mm, &cs->mems_allowed);
	if (migrate)
T
Tejun Heo 已提交
1082
		cpuset_migrate_mm(mm, &cs->old_mems_allowed, arg->newmems);
1083 1084 1085
	mmput(mm);
}

1086 1087
static void *cpuset_being_rebound;

1088 1089 1090
/**
 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
1091
 * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
1092
 *
1093 1094
 * Called with cpuset_mutex held.  No return value. It's guaranteed that
 * css_scan_tasks() always returns 0 if @heap != NULL.
1095
 */
1096
static void update_tasks_nodemask(struct cpuset *cs, struct ptr_heap *heap)
L
Linus Torvalds 已提交
1097
{
1098
	static nodemask_t newmems;	/* protected by cpuset_mutex */
1099
	struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
T
Tejun Heo 已提交
1100 1101
	struct cpuset_change_nodemask_arg arg = { .cs = cs,
						  .newmems = &newmems };
1102

1103
	cpuset_being_rebound = cs;		/* causes mpol_dup() rebind */
1104

1105
	guarantee_online_mems(mems_cs, &newmems);
1106

1107
	/*
1108 1109 1110 1111
	 * The mpol_rebind_mm() call takes mmap_sem, which we couldn't
	 * take while holding tasklist_lock.  Forks can happen - the
	 * mpol_dup() cpuset_being_rebound check will catch such forks,
	 * and rebind their vma mempolicies too.  Because we still hold
1112
	 * the global cpuset_mutex, we know that no other rebind effort
1113
	 * will be contending for the global variable cpuset_being_rebound.
1114
	 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
1115
	 * is idempotent.  Also migrate pages in each mm to new nodes.
1116
	 */
1117
	css_scan_tasks(&cs->css, NULL, cpuset_change_nodemask, &arg, heap);
1118

1119 1120 1121 1122 1123 1124
	/*
	 * All the tasks' nodemasks have been updated, update
	 * cs->old_mems_allowed.
	 */
	cs->old_mems_allowed = newmems;

1125
	/* We're done rebinding vmas to this cpuset's new mems_allowed. */
1126
	cpuset_being_rebound = NULL;
L
Linus Torvalds 已提交
1127 1128
}

1129 1130 1131 1132
/*
 * update_tasks_nodemask_hier - Update the nodemasks of tasks in the hierarchy.
 * @cs: the root cpuset of the hierarchy
 * @update_root: update the root cpuset or not?
1133
 * @heap: the heap used by css_scan_tasks()
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
 *
 * This will update nodemasks of tasks in @root_cs and all other empty cpusets
 * which take on nodemask of @root_cs.
 *
 * Called with cpuset_mutex held
 */
static void update_tasks_nodemask_hier(struct cpuset *root_cs,
				       bool update_root, struct ptr_heap *heap)
{
	struct cpuset *cp;
1144
	struct cgroup_subsys_state *pos_css;
1145 1146

	rcu_read_lock();
1147
	cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
1148 1149 1150 1151 1152 1153 1154 1155 1156
		if (cp == root_cs) {
			if (!update_root)
				continue;
		} else {
			/* skip the whole subtree if @cp have some CPU */
			if (!nodes_empty(cp->mems_allowed)) {
				pos_css = css_rightmost_descendant(pos_css);
				continue;
			}
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
		}
		if (!css_tryget(&cp->css))
			continue;
		rcu_read_unlock();

		update_tasks_nodemask(cp, heap);

		rcu_read_lock();
		css_put(&cp->css);
	}
	rcu_read_unlock();
}

1170 1171 1172
/*
 * Handle user request to change the 'mems' memory placement
 * of a cpuset.  Needs to validate the request, update the
1173 1174 1175 1176
 * cpusets mems_allowed, and for each task in the cpuset,
 * update mems_allowed and rebind task's mempolicy and any vma
 * mempolicies and if the cpuset is marked 'memory_migrate',
 * migrate the tasks pages to the new memory.
1177
 *
1178
 * Call with cpuset_mutex held.  May take callback_mutex during call.
1179 1180 1181 1182
 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
 * their mempolicies to the cpusets new mems_allowed.
 */
1183 1184
static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
			   const char *buf)
1185 1186
{
	int retval;
1187
	struct ptr_heap heap;
1188 1189

	/*
1190
	 * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
1191 1192
	 * it's read-only
	 */
1193 1194 1195 1196
	if (cs == &top_cpuset) {
		retval = -EACCES;
		goto done;
	}
1197 1198 1199 1200 1201 1202 1203 1204

	/*
	 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
	 * Since nodelist_parse() fails on an empty mask, we special case
	 * that parsing.  The validate_change() call ensures that cpusets
	 * with tasks have memory.
	 */
	if (!*buf) {
1205
		nodes_clear(trialcs->mems_allowed);
1206
	} else {
1207
		retval = nodelist_parse(buf, trialcs->mems_allowed);
1208 1209 1210
		if (retval < 0)
			goto done;

1211
		if (!nodes_subset(trialcs->mems_allowed,
1212
				node_states[N_MEMORY])) {
1213 1214 1215
			retval =  -EINVAL;
			goto done;
		}
1216
	}
1217 1218

	if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
1219 1220 1221
		retval = 0;		/* Too easy - nothing to do */
		goto done;
	}
1222
	retval = validate_change(cs, trialcs);
1223 1224 1225
	if (retval < 0)
		goto done;

1226 1227 1228 1229
	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (retval < 0)
		goto done;

1230
	mutex_lock(&callback_mutex);
1231
	cs->mems_allowed = trialcs->mems_allowed;
1232 1233
	mutex_unlock(&callback_mutex);

1234
	update_tasks_nodemask_hier(cs, true, &heap);
1235 1236

	heap_free(&heap);
1237 1238 1239 1240
done:
	return retval;
}

1241 1242 1243 1244 1245
int current_cpuset_is_being_rebound(void)
{
	return task_cs(current) == cpuset_being_rebound;
}

1246
static int update_relax_domain_level(struct cpuset *cs, s64 val)
1247
{
1248
#ifdef CONFIG_SMP
1249
	if (val < -1 || val >= sched_domain_level_max)
1250
		return -EINVAL;
1251
#endif
1252 1253 1254

	if (val != cs->relax_domain_level) {
		cs->relax_domain_level = val;
1255 1256
		if (!cpumask_empty(cs->cpus_allowed) &&
		    is_sched_load_balance(cs))
1257
			rebuild_sched_domains_locked();
1258 1259 1260 1261 1262
	}

	return 0;
}

1263
/**
1264 1265
 * cpuset_change_flag - make a task's spread flags the same as its cpuset's
 * @tsk: task to be updated
T
Tejun Heo 已提交
1266
 * @data: cpuset to @tsk belongs to
1267
 *
1268
 * Called by css_scan_tasks() for each task in a cgroup.
1269 1270
 *
 * We don't need to re-check for the cgroup/cpuset membership, since we're
1271
 * holding cpuset_mutex at this point.
1272
 */
T
Tejun Heo 已提交
1273
static void cpuset_change_flag(struct task_struct *tsk, void *data)
1274
{
T
Tejun Heo 已提交
1275 1276 1277
	struct cpuset *cs = data;

	cpuset_update_task_spread_flag(cs, tsk);
1278 1279
}

1280
/**
1281 1282
 * update_tasks_flags - update the spread flags of tasks in the cpuset.
 * @cs: the cpuset in which each task's spread flags needs to be changed
1283
 * @heap: if NULL, defer allocating heap memory to css_scan_tasks()
1284
 *
1285
 * Called with cpuset_mutex held
1286
 *
1287
 * The css_scan_tasks() function will scan all the tasks in a cgroup,
1288 1289
 * calling callback functions for each.
 *
1290
 * No return value. It's guaranteed that css_scan_tasks() always returns 0
1291 1292 1293 1294
 * if @heap != NULL.
 */
static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
{
1295
	css_scan_tasks(&cs->css, NULL, cpuset_change_flag, cs, heap);
1296 1297
}

L
Linus Torvalds 已提交
1298 1299
/*
 * update_flag - read a 0 or a 1 in a file and update associated flag
1300 1301 1302
 * bit:		the bit to update (see cpuset_flagbits_t)
 * cs:		the cpuset to update
 * turning_on: 	whether the flag is being set or cleared
1303
 *
1304
 * Call with cpuset_mutex held.
L
Linus Torvalds 已提交
1305 1306
 */

1307 1308
static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
		       int turning_on)
L
Linus Torvalds 已提交
1309
{
1310
	struct cpuset *trialcs;
R
Rakib Mullick 已提交
1311
	int balance_flag_changed;
1312 1313 1314
	int spread_flag_changed;
	struct ptr_heap heap;
	int err;
L
Linus Torvalds 已提交
1315

1316 1317 1318 1319
	trialcs = alloc_trial_cpuset(cs);
	if (!trialcs)
		return -ENOMEM;

L
Linus Torvalds 已提交
1320
	if (turning_on)
1321
		set_bit(bit, &trialcs->flags);
L
Linus Torvalds 已提交
1322
	else
1323
		clear_bit(bit, &trialcs->flags);
L
Linus Torvalds 已提交
1324

1325
	err = validate_change(cs, trialcs);
1326
	if (err < 0)
1327
		goto out;
P
Paul Jackson 已提交
1328

1329 1330 1331 1332
	err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
	if (err < 0)
		goto out;

P
Paul Jackson 已提交
1333
	balance_flag_changed = (is_sched_load_balance(cs) !=
1334
				is_sched_load_balance(trialcs));
P
Paul Jackson 已提交
1335

1336 1337 1338
	spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
			|| (is_spread_page(cs) != is_spread_page(trialcs)));

1339
	mutex_lock(&callback_mutex);
1340
	cs->flags = trialcs->flags;
1341
	mutex_unlock(&callback_mutex);
1342

1343
	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1344
		rebuild_sched_domains_locked();
P
Paul Jackson 已提交
1345

1346 1347 1348
	if (spread_flag_changed)
		update_tasks_flags(cs, &heap);
	heap_free(&heap);
1349 1350 1351
out:
	free_trial_cpuset(trialcs);
	return err;
L
Linus Torvalds 已提交
1352 1353
}

1354
/*
A
Adrian Bunk 已提交
1355
 * Frequency meter - How fast is some event occurring?
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
 *
 * These routines manage a digitally filtered, constant time based,
 * event frequency meter.  There are four routines:
 *   fmeter_init() - initialize a frequency meter.
 *   fmeter_markevent() - called each time the event happens.
 *   fmeter_getrate() - returns the recent rate of such events.
 *   fmeter_update() - internal routine used to update fmeter.
 *
 * A common data structure is passed to each of these routines,
 * which is used to keep track of the state required to manage the
 * frequency meter and its digital filter.
 *
 * The filter works on the number of events marked per unit time.
 * The filter is single-pole low-pass recursive (IIR).  The time unit
 * is 1 second.  Arithmetic is done using 32-bit integers scaled to
 * simulate 3 decimal digits of precision (multiplied by 1000).
 *
 * With an FM_COEF of 933, and a time base of 1 second, the filter
 * has a half-life of 10 seconds, meaning that if the events quit
 * happening, then the rate returned from the fmeter_getrate()
 * will be cut in half each 10 seconds, until it converges to zero.
 *
 * It is not worth doing a real infinitely recursive filter.  If more
 * than FM_MAXTICKS ticks have elapsed since the last filter event,
 * just compute FM_MAXTICKS ticks worth, by which point the level
 * will be stable.
 *
 * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
 * arithmetic overflow in the fmeter_update() routine.
 *
 * Given the simple 32 bit integer arithmetic used, this meter works
 * best for reporting rates between one per millisecond (msec) and
 * one per 32 (approx) seconds.  At constant rates faster than one
 * per msec it maxes out at values just under 1,000,000.  At constant
 * rates between one per msec, and one per second it will stabilize
 * to a value N*1000, where N is the rate of events per second.
 * At constant rates between one per second and one per 32 seconds,
 * it will be choppy, moving up on the seconds that have an event,
 * and then decaying until the next event.  At rates slower than
 * about one in 32 seconds, it decays all the way back to zero between
 * each event.
 */

#define FM_COEF 933		/* coefficient for half-life of 10 secs */
#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
#define FM_MAXCNT 1000000	/* limit cnt to avoid overflow */
#define FM_SCALE 1000		/* faux fixed point scale */

/* Initialize a frequency meter */
static void fmeter_init(struct fmeter *fmp)
{
	fmp->cnt = 0;
	fmp->val = 0;
	fmp->time = 0;
	spin_lock_init(&fmp->lock);
}

/* Internal meter update - process cnt events and update value */
static void fmeter_update(struct fmeter *fmp)
{
	time_t now = get_seconds();
	time_t ticks = now - fmp->time;

	if (ticks == 0)
		return;

	ticks = min(FM_MAXTICKS, ticks);
	while (ticks-- > 0)
		fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
	fmp->time = now;

	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
	fmp->cnt = 0;
}

/* Process any previous ticks, then bump cnt by one (times scale). */
static void fmeter_markevent(struct fmeter *fmp)
{
	spin_lock(&fmp->lock);
	fmeter_update(fmp);
	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
	spin_unlock(&fmp->lock);
}

/* Process any previous ticks, then return current value. */
static int fmeter_getrate(struct fmeter *fmp)
{
	int val;

	spin_lock(&fmp->lock);
	fmeter_update(fmp);
	val = fmp->val;
	spin_unlock(&fmp->lock);
	return val;
}

1452
/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
1453 1454
static int cpuset_can_attach(struct cgroup_subsys_state *css,
			     struct cgroup_taskset *tset)
1455
{
1456
	struct cpuset *cs = css_cs(css);
1457 1458
	struct task_struct *task;
	int ret;
L
Linus Torvalds 已提交
1459

1460 1461
	mutex_lock(&cpuset_mutex);

1462 1463 1464 1465
	/*
	 * We allow to move tasks into an empty cpuset if sane_behavior
	 * flag is set.
	 */
1466
	ret = -ENOSPC;
1467
	if (!cgroup_sane_behavior(css->cgroup) &&
1468
	    (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)))
1469
		goto out_unlock;
1470

1471
	cgroup_taskset_for_each(task, css, tset) {
1472
		/*
1473 1474 1475 1476 1477 1478 1479
		 * Kthreads which disallow setaffinity shouldn't be moved
		 * to a new cpuset; we don't want to change their cpu
		 * affinity and isolating such threads by their set of
		 * allowed nodes is unnecessary.  Thus, cpusets are not
		 * applicable for such threads.  This prevents checking for
		 * success of set_cpus_allowed_ptr() on all attached tasks
		 * before cpus_allowed may be changed.
1480
		 */
1481
		ret = -EINVAL;
1482
		if (task->flags & PF_NO_SETAFFINITY)
1483 1484 1485 1486
			goto out_unlock;
		ret = security_task_setscheduler(task);
		if (ret)
			goto out_unlock;
1487
	}
1488

1489 1490 1491 1492 1493
	/*
	 * Mark attach is in progress.  This makes validate_change() fail
	 * changes which zero cpus/mems_allowed.
	 */
	cs->attach_in_progress++;
1494 1495 1496 1497
	ret = 0;
out_unlock:
	mutex_unlock(&cpuset_mutex);
	return ret;
1498
}
1499

1500
static void cpuset_cancel_attach(struct cgroup_subsys_state *css,
1501 1502
				 struct cgroup_taskset *tset)
{
1503
	mutex_lock(&cpuset_mutex);
1504
	css_cs(css)->attach_in_progress--;
1505
	mutex_unlock(&cpuset_mutex);
1506
}
L
Linus Torvalds 已提交
1507

1508
/*
1509
 * Protected by cpuset_mutex.  cpus_attach is used only by cpuset_attach()
1510 1511 1512 1513 1514
 * but we can't allocate it dynamically there.  Define it global and
 * allocate from cpuset_init().
 */
static cpumask_var_t cpus_attach;

1515 1516
static void cpuset_attach(struct cgroup_subsys_state *css,
			  struct cgroup_taskset *tset)
1517
{
1518
	/* static buf protected by cpuset_mutex */
1519
	static nodemask_t cpuset_attach_nodemask_to;
1520
	struct mm_struct *mm;
1521 1522
	struct task_struct *task;
	struct task_struct *leader = cgroup_taskset_first(tset);
1523
	struct cgroup_subsys_state *oldcss = cgroup_taskset_cur_css(tset,
1524
							cpuset_cgrp_id);
1525
	struct cpuset *cs = css_cs(css);
1526
	struct cpuset *oldcs = css_cs(oldcss);
1527 1528
	struct cpuset *cpus_cs = effective_cpumask_cpuset(cs);
	struct cpuset *mems_cs = effective_nodemask_cpuset(cs);
1529

1530 1531
	mutex_lock(&cpuset_mutex);

1532 1533 1534 1535
	/* prepare for attach */
	if (cs == &top_cpuset)
		cpumask_copy(cpus_attach, cpu_possible_mask);
	else
1536
		guarantee_online_cpus(cpus_cs, cpus_attach);
1537

1538
	guarantee_online_mems(mems_cs, &cpuset_attach_nodemask_to);
1539

1540
	cgroup_taskset_for_each(task, css, tset) {
1541 1542 1543 1544 1545 1546 1547 1548 1549
		/*
		 * can_attach beforehand should guarantee that this doesn't
		 * fail.  TODO: have a better way to handle failure here
		 */
		WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));

		cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
		cpuset_update_task_spread_flag(cs, task);
	}
1550

1551 1552 1553 1554 1555
	/*
	 * Change mm, possibly for multiple threads in a threadgroup. This is
	 * expensive and may sleep.
	 */
	cpuset_attach_nodemask_to = cs->mems_allowed;
1556
	mm = get_task_mm(leader);
1557
	if (mm) {
1558 1559
		struct cpuset *mems_oldcs = effective_nodemask_cpuset(oldcs);

1560
		mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570

		/*
		 * old_mems_allowed is the same with mems_allowed here, except
		 * if this task is being moved automatically due to hotplug.
		 * In that case @mems_allowed has been updated and is empty,
		 * so @old_mems_allowed is the right nodesets that we migrate
		 * mm from.
		 */
		if (is_memory_migrate(cs)) {
			cpuset_migrate_mm(mm, &mems_oldcs->old_mems_allowed,
1571
					  &cpuset_attach_nodemask_to);
1572
		}
1573 1574
		mmput(mm);
	}
1575

1576
	cs->old_mems_allowed = cpuset_attach_nodemask_to;
1577

1578
	cs->attach_in_progress--;
1579 1580
	if (!cs->attach_in_progress)
		wake_up(&cpuset_attach_wq);
1581 1582

	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
1583 1584 1585 1586 1587
}

/* The various types of files and directories in a cpuset file system */

typedef enum {
1588
	FILE_MEMORY_MIGRATE,
L
Linus Torvalds 已提交
1589 1590 1591 1592
	FILE_CPULIST,
	FILE_MEMLIST,
	FILE_CPU_EXCLUSIVE,
	FILE_MEM_EXCLUSIVE,
1593
	FILE_MEM_HARDWALL,
P
Paul Jackson 已提交
1594
	FILE_SCHED_LOAD_BALANCE,
1595
	FILE_SCHED_RELAX_DOMAIN_LEVEL,
1596 1597
	FILE_MEMORY_PRESSURE_ENABLED,
	FILE_MEMORY_PRESSURE,
1598 1599
	FILE_SPREAD_PAGE,
	FILE_SPREAD_SLAB,
L
Linus Torvalds 已提交
1600 1601
} cpuset_filetype_t;

1602 1603
static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
			    u64 val)
1604
{
1605
	struct cpuset *cs = css_cs(css);
1606
	cpuset_filetype_t type = cft->private;
1607
	int retval = 0;
1608

1609
	mutex_lock(&cpuset_mutex);
1610 1611
	if (!is_cpuset_online(cs)) {
		retval = -ENODEV;
1612
		goto out_unlock;
1613
	}
1614 1615

	switch (type) {
L
Linus Torvalds 已提交
1616
	case FILE_CPU_EXCLUSIVE:
1617
		retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
L
Linus Torvalds 已提交
1618 1619
		break;
	case FILE_MEM_EXCLUSIVE:
1620
		retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
L
Linus Torvalds 已提交
1621
		break;
1622 1623 1624
	case FILE_MEM_HARDWALL:
		retval = update_flag(CS_MEM_HARDWALL, cs, val);
		break;
P
Paul Jackson 已提交
1625
	case FILE_SCHED_LOAD_BALANCE:
1626
		retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1627
		break;
1628
	case FILE_MEMORY_MIGRATE:
1629
		retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1630
		break;
1631
	case FILE_MEMORY_PRESSURE_ENABLED:
1632
		cpuset_memory_pressure_enabled = !!val;
1633 1634 1635 1636
		break;
	case FILE_MEMORY_PRESSURE:
		retval = -EACCES;
		break;
1637
	case FILE_SPREAD_PAGE:
1638
		retval = update_flag(CS_SPREAD_PAGE, cs, val);
1639 1640
		break;
	case FILE_SPREAD_SLAB:
1641
		retval = update_flag(CS_SPREAD_SLAB, cs, val);
1642
		break;
L
Linus Torvalds 已提交
1643 1644
	default:
		retval = -EINVAL;
1645
		break;
L
Linus Torvalds 已提交
1646
	}
1647 1648
out_unlock:
	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
1649 1650 1651
	return retval;
}

1652 1653
static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
			    s64 val)
1654
{
1655
	struct cpuset *cs = css_cs(css);
1656
	cpuset_filetype_t type = cft->private;
1657
	int retval = -ENODEV;
1658

1659 1660 1661
	mutex_lock(&cpuset_mutex);
	if (!is_cpuset_online(cs))
		goto out_unlock;
1662

1663 1664 1665 1666 1667 1668 1669 1670
	switch (type) {
	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
		retval = update_relax_domain_level(cs, val);
		break;
	default:
		retval = -EINVAL;
		break;
	}
1671 1672
out_unlock:
	mutex_unlock(&cpuset_mutex);
1673 1674 1675
	return retval;
}

1676 1677 1678
/*
 * Common handling for a write to a "cpus" or "mems" file.
 */
1679 1680
static int cpuset_write_resmask(struct cgroup_subsys_state *css,
				struct cftype *cft, const char *buf)
1681
{
1682
	struct cpuset *cs = css_cs(css);
1683
	struct cpuset *trialcs;
1684
	int retval = -ENODEV;
1685

1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698
	/*
	 * CPU or memory hotunplug may leave @cs w/o any execution
	 * resources, in which case the hotplug code asynchronously updates
	 * configuration and transfers all tasks to the nearest ancestor
	 * which can execute.
	 *
	 * As writes to "cpus" or "mems" may restore @cs's execution
	 * resources, wait for the previously scheduled operations before
	 * proceeding, so that we don't end up keep removing tasks added
	 * after execution capability is restored.
	 */
	flush_work(&cpuset_hotplug_work);

1699 1700 1701
	mutex_lock(&cpuset_mutex);
	if (!is_cpuset_online(cs))
		goto out_unlock;
1702

1703
	trialcs = alloc_trial_cpuset(cs);
1704 1705
	if (!trialcs) {
		retval = -ENOMEM;
1706
		goto out_unlock;
1707
	}
1708

1709 1710
	switch (cft->private) {
	case FILE_CPULIST:
1711
		retval = update_cpumask(cs, trialcs, buf);
1712 1713
		break;
	case FILE_MEMLIST:
1714
		retval = update_nodemask(cs, trialcs, buf);
1715 1716 1717 1718 1719
		break;
	default:
		retval = -EINVAL;
		break;
	}
1720 1721

	free_trial_cpuset(trialcs);
1722 1723
out_unlock:
	mutex_unlock(&cpuset_mutex);
1724 1725 1726
	return retval;
}

L
Linus Torvalds 已提交
1727 1728 1729 1730 1731 1732 1733 1734
/*
 * These ascii lists should be read in a single call, by using a user
 * buffer large enough to hold the entire map.  If read in smaller
 * chunks, there is no guarantee of atomicity.  Since the display format
 * used, list of ranges of sequential numbers, is variable length,
 * and since these maps can change value dynamically, one could read
 * gibberish by doing partial reads while a list was changing.
 */
1735
static int cpuset_common_seq_show(struct seq_file *sf, void *v)
L
Linus Torvalds 已提交
1736
{
1737 1738
	struct cpuset *cs = css_cs(seq_css(sf));
	cpuset_filetype_t type = seq_cft(sf)->private;
1739 1740 1741
	ssize_t count;
	char *buf, *s;
	int ret = 0;
L
Linus Torvalds 已提交
1742

1743 1744
	count = seq_get_buf(sf, &buf);
	s = buf;
L
Linus Torvalds 已提交
1745

1746
	mutex_lock(&callback_mutex);
L
Linus Torvalds 已提交
1747 1748 1749

	switch (type) {
	case FILE_CPULIST:
1750
		s += cpulist_scnprintf(s, count, cs->cpus_allowed);
L
Linus Torvalds 已提交
1751 1752
		break;
	case FILE_MEMLIST:
1753
		s += nodelist_scnprintf(s, count, cs->mems_allowed);
L
Linus Torvalds 已提交
1754 1755
		break;
	default:
1756 1757
		ret = -EINVAL;
		goto out_unlock;
L
Linus Torvalds 已提交
1758 1759
	}

1760 1761 1762 1763 1764 1765 1766 1767 1768
	if (s < buf + count - 1) {
		*s++ = '\n';
		seq_commit(sf, s - buf);
	} else {
		seq_commit(sf, -1);
	}
out_unlock:
	mutex_unlock(&callback_mutex);
	return ret;
L
Linus Torvalds 已提交
1769 1770
}

1771
static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
1772
{
1773
	struct cpuset *cs = css_cs(css);
1774 1775 1776 1777 1778 1779
	cpuset_filetype_t type = cft->private;
	switch (type) {
	case FILE_CPU_EXCLUSIVE:
		return is_cpu_exclusive(cs);
	case FILE_MEM_EXCLUSIVE:
		return is_mem_exclusive(cs);
1780 1781
	case FILE_MEM_HARDWALL:
		return is_mem_hardwall(cs);
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796
	case FILE_SCHED_LOAD_BALANCE:
		return is_sched_load_balance(cs);
	case FILE_MEMORY_MIGRATE:
		return is_memory_migrate(cs);
	case FILE_MEMORY_PRESSURE_ENABLED:
		return cpuset_memory_pressure_enabled;
	case FILE_MEMORY_PRESSURE:
		return fmeter_getrate(&cs->fmeter);
	case FILE_SPREAD_PAGE:
		return is_spread_page(cs);
	case FILE_SPREAD_SLAB:
		return is_spread_slab(cs);
	default:
		BUG();
	}
1797 1798 1799

	/* Unreachable but makes gcc happy */
	return 0;
1800
}
L
Linus Torvalds 已提交
1801

1802
static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
1803
{
1804
	struct cpuset *cs = css_cs(css);
1805 1806 1807 1808 1809 1810 1811
	cpuset_filetype_t type = cft->private;
	switch (type) {
	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
		return cs->relax_domain_level;
	default:
		BUG();
	}
1812 1813 1814

	/* Unrechable but makes gcc happy */
	return 0;
1815 1816
}

L
Linus Torvalds 已提交
1817 1818 1819 1820 1821

/*
 * for the common functions, 'private' gives the type of file
 */

1822 1823 1824
static struct cftype files[] = {
	{
		.name = "cpus",
1825
		.seq_show = cpuset_common_seq_show,
1826 1827
		.write_string = cpuset_write_resmask,
		.max_write_len = (100U + 6 * NR_CPUS),
1828 1829 1830 1831 1832
		.private = FILE_CPULIST,
	},

	{
		.name = "mems",
1833
		.seq_show = cpuset_common_seq_show,
1834 1835
		.write_string = cpuset_write_resmask,
		.max_write_len = (100U + 6 * MAX_NUMNODES),
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852
		.private = FILE_MEMLIST,
	},

	{
		.name = "cpu_exclusive",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_CPU_EXCLUSIVE,
	},

	{
		.name = "mem_exclusive",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEM_EXCLUSIVE,
	},

1853 1854 1855 1856 1857 1858 1859
	{
		.name = "mem_hardwall",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEM_HARDWALL,
	},

1860 1861 1862 1863 1864 1865 1866 1867 1868
	{
		.name = "sched_load_balance",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SCHED_LOAD_BALANCE,
	},

	{
		.name = "sched_relax_domain_level",
1869 1870
		.read_s64 = cpuset_read_s64,
		.write_s64 = cpuset_write_s64,
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
		.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
	},

	{
		.name = "memory_migrate",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_MIGRATE,
	},

	{
		.name = "memory_pressure",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_PRESSURE,
L
Li Zefan 已提交
1886
		.mode = S_IRUGO,
1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
	},

	{
		.name = "memory_spread_page",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SPREAD_PAGE,
	},

	{
		.name = "memory_spread_slab",
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_SPREAD_SLAB,
	},
1902

1903 1904 1905 1906 1907 1908 1909
	{
		.name = "memory_pressure_enabled",
		.flags = CFTYPE_ONLY_ON_ROOT,
		.read_u64 = cpuset_read_u64,
		.write_u64 = cpuset_write_u64,
		.private = FILE_MEMORY_PRESSURE_ENABLED,
	},
L
Linus Torvalds 已提交
1910

1911 1912
	{ }	/* terminate */
};
L
Linus Torvalds 已提交
1913 1914

/*
1915
 *	cpuset_css_alloc - allocate a cpuset css
L
Li Zefan 已提交
1916
 *	cgrp:	control group that the new cpuset will be part of
L
Linus Torvalds 已提交
1917 1918
 */

1919 1920
static struct cgroup_subsys_state *
cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
L
Linus Torvalds 已提交
1921
{
T
Tejun Heo 已提交
1922
	struct cpuset *cs;
L
Linus Torvalds 已提交
1923

1924
	if (!parent_css)
1925
		return &top_cpuset.css;
1926

T
Tejun Heo 已提交
1927
	cs = kzalloc(sizeof(*cs), GFP_KERNEL);
L
Linus Torvalds 已提交
1928
	if (!cs)
1929
		return ERR_PTR(-ENOMEM);
1930 1931 1932 1933
	if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
		kfree(cs);
		return ERR_PTR(-ENOMEM);
	}
L
Linus Torvalds 已提交
1934

P
Paul Jackson 已提交
1935
	set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
1936
	cpumask_clear(cs->cpus_allowed);
1937
	nodes_clear(cs->mems_allowed);
1938
	fmeter_init(&cs->fmeter);
1939
	cs->relax_domain_level = -1;
L
Linus Torvalds 已提交
1940

T
Tejun Heo 已提交
1941 1942 1943
	return &cs->css;
}

1944
static int cpuset_css_online(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
1945
{
1946
	struct cpuset *cs = css_cs(css);
T
Tejun Heo 已提交
1947
	struct cpuset *parent = parent_cs(cs);
1948
	struct cpuset *tmp_cs;
1949
	struct cgroup_subsys_state *pos_css;
T
Tejun Heo 已提交
1950 1951 1952 1953

	if (!parent)
		return 0;

1954 1955
	mutex_lock(&cpuset_mutex);

T
Tejun Heo 已提交
1956
	set_bit(CS_ONLINE, &cs->flags);
T
Tejun Heo 已提交
1957 1958 1959 1960
	if (is_spread_page(parent))
		set_bit(CS_SPREAD_PAGE, &cs->flags);
	if (is_spread_slab(parent))
		set_bit(CS_SPREAD_SLAB, &cs->flags);
L
Linus Torvalds 已提交
1961

1962
	number_of_cpusets++;
1963

1964
	if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
1965
		goto out_unlock;
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979

	/*
	 * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
	 * set.  This flag handling is implemented in cgroup core for
	 * histrical reasons - the flag may be specified during mount.
	 *
	 * Currently, if any sibling cpusets have exclusive cpus or mem, we
	 * refuse to clone the configuration - thereby refusing the task to
	 * be entered, and as a result refusing the sys_unshare() or
	 * clone() which initiated it.  If this becomes a problem for some
	 * users who wish to allow that scenario, then this could be
	 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
	 * (and likewise for mems) to the new cgroup.
	 */
1980
	rcu_read_lock();
1981
	cpuset_for_each_child(tmp_cs, pos_css, parent) {
1982 1983
		if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
			rcu_read_unlock();
1984
			goto out_unlock;
1985
		}
1986
	}
1987
	rcu_read_unlock();
1988 1989 1990 1991 1992

	mutex_lock(&callback_mutex);
	cs->mems_allowed = parent->mems_allowed;
	cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
	mutex_unlock(&callback_mutex);
1993 1994
out_unlock:
	mutex_unlock(&cpuset_mutex);
T
Tejun Heo 已提交
1995 1996 1997
	return 0;
}

1998 1999 2000 2001 2002 2003
/*
 * If the cpuset being removed has its flag 'sched_load_balance'
 * enabled, then simulate turning sched_load_balance off, which
 * will call rebuild_sched_domains_locked().
 */

2004
static void cpuset_css_offline(struct cgroup_subsys_state *css)
T
Tejun Heo 已提交
2005
{
2006
	struct cpuset *cs = css_cs(css);
T
Tejun Heo 已提交
2007

2008
	mutex_lock(&cpuset_mutex);
T
Tejun Heo 已提交
2009 2010 2011 2012 2013

	if (is_sched_load_balance(cs))
		update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);

	number_of_cpusets--;
T
Tejun Heo 已提交
2014
	clear_bit(CS_ONLINE, &cs->flags);
T
Tejun Heo 已提交
2015

2016
	mutex_unlock(&cpuset_mutex);
L
Linus Torvalds 已提交
2017 2018
}

2019
static void cpuset_css_free(struct cgroup_subsys_state *css)
L
Linus Torvalds 已提交
2020
{
2021
	struct cpuset *cs = css_cs(css);
L
Linus Torvalds 已提交
2022

2023
	free_cpumask_var(cs->cpus_allowed);
2024
	kfree(cs);
L
Linus Torvalds 已提交
2025 2026
}

2027
struct cgroup_subsys cpuset_cgrp_subsys = {
2028
	.css_alloc = cpuset_css_alloc,
T
Tejun Heo 已提交
2029 2030
	.css_online = cpuset_css_online,
	.css_offline = cpuset_css_offline,
2031
	.css_free = cpuset_css_free,
2032
	.can_attach = cpuset_can_attach,
2033
	.cancel_attach = cpuset_cancel_attach,
2034
	.attach = cpuset_attach,
2035
	.base_cftypes = files,
2036 2037 2038
	.early_init = 1,
};

L
Linus Torvalds 已提交
2039 2040 2041 2042 2043 2044 2045 2046
/**
 * cpuset_init - initialize cpusets at system boot
 *
 * Description: Initialize top_cpuset and the cpuset internal file system,
 **/

int __init cpuset_init(void)
{
2047
	int err = 0;
L
Linus Torvalds 已提交
2048

2049 2050 2051
	if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
		BUG();

2052
	cpumask_setall(top_cpuset.cpus_allowed);
2053
	nodes_setall(top_cpuset.mems_allowed);
L
Linus Torvalds 已提交
2054

2055
	fmeter_init(&top_cpuset.fmeter);
P
Paul Jackson 已提交
2056
	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
2057
	top_cpuset.relax_domain_level = -1;
L
Linus Torvalds 已提交
2058 2059 2060

	err = register_filesystem(&cpuset_fs_type);
	if (err < 0)
2061 2062
		return err;

2063 2064 2065
	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
		BUG();

2066
	number_of_cpusets = 1;
2067
	return 0;
L
Linus Torvalds 已提交
2068 2069
}

2070
/*
2071
 * If CPU and/or memory hotplug handlers, below, unplug any CPUs
2072 2073
 * or memory nodes, we need to walk over the cpuset hierarchy,
 * removing that CPU or node from all cpusets.  If this removes the
2074 2075
 * last CPU or node from a cpuset, then move the tasks in the empty
 * cpuset to its next-highest non-empty parent.
2076
 */
2077 2078 2079 2080 2081 2082 2083 2084
static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
{
	struct cpuset *parent;

	/*
	 * Find its next-highest non-empty parent, (top cpuset
	 * has online cpus, so can't be empty).
	 */
T
Tejun Heo 已提交
2085
	parent = parent_cs(cs);
2086
	while (cpumask_empty(parent->cpus_allowed) ||
2087
			nodes_empty(parent->mems_allowed))
T
Tejun Heo 已提交
2088
		parent = parent_cs(parent);
2089

2090 2091 2092 2093 2094 2095
	if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
		rcu_read_lock();
		printk(KERN_ERR "cpuset: failed to transfer tasks out of empty cpuset %s\n",
		       cgroup_name(cs->css.cgroup));
		rcu_read_unlock();
	}
2096 2097
}

2098
/**
2099
 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
2100
 * @cs: cpuset in interest
2101
 *
2102 2103 2104
 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
 * offline, update @cs accordingly.  If @cs ends up with no CPU or memory,
 * all its tasks are moved to the nearest ancestor with both resources.
2105
 */
2106
static void cpuset_hotplug_update_tasks(struct cpuset *cs)
2107
{
2108
	static cpumask_t off_cpus;
2109
	static nodemask_t off_mems;
2110
	bool is_empty;
2111
	bool sane = cgroup_sane_behavior(cs->css.cgroup);
2112

2113 2114
retry:
	wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
2115

2116
	mutex_lock(&cpuset_mutex);
2117

2118 2119 2120 2121 2122 2123 2124 2125 2126
	/*
	 * We have raced with task attaching. We wait until attaching
	 * is finished, so we won't attach a task to an empty cpuset.
	 */
	if (cs->attach_in_progress) {
		mutex_unlock(&cpuset_mutex);
		goto retry;
	}

2127 2128
	cpumask_andnot(&off_cpus, cs->cpus_allowed, top_cpuset.cpus_allowed);
	nodes_andnot(off_mems, cs->mems_allowed, top_cpuset.mems_allowed);
2129

2130 2131 2132 2133 2134 2135
	mutex_lock(&callback_mutex);
	cpumask_andnot(cs->cpus_allowed, cs->cpus_allowed, &off_cpus);
	mutex_unlock(&callback_mutex);

	/*
	 * If sane_behavior flag is set, we need to update tasks' cpumask
2136 2137 2138
	 * for empty cpuset to take on ancestor's cpumask. Otherwise, don't
	 * call update_tasks_cpumask() if the cpuset becomes empty, as
	 * the tasks in it will be migrated to an ancestor.
2139 2140
	 */
	if ((sane && cpumask_empty(cs->cpus_allowed)) ||
2141
	    (!cpumask_empty(&off_cpus) && !cpumask_empty(cs->cpus_allowed)))
2142
		update_tasks_cpumask(cs, NULL);
2143

2144 2145 2146 2147 2148 2149
	mutex_lock(&callback_mutex);
	nodes_andnot(cs->mems_allowed, cs->mems_allowed, off_mems);
	mutex_unlock(&callback_mutex);

	/*
	 * If sane_behavior flag is set, we need to update tasks' nodemask
2150 2151 2152
	 * for empty cpuset to take on ancestor's nodemask. Otherwise, don't
	 * call update_tasks_nodemask() if the cpuset becomes empty, as
	 * the tasks in it will be migratd to an ancestor.
2153 2154
	 */
	if ((sane && nodes_empty(cs->mems_allowed)) ||
2155
	    (!nodes_empty(off_mems) && !nodes_empty(cs->mems_allowed)))
2156
		update_tasks_nodemask(cs, NULL);
2157

2158 2159
	is_empty = cpumask_empty(cs->cpus_allowed) ||
		nodes_empty(cs->mems_allowed);
2160

2161 2162 2163
	mutex_unlock(&cpuset_mutex);

	/*
2164 2165 2166 2167
	 * If sane_behavior flag is set, we'll keep tasks in empty cpusets.
	 *
	 * Otherwise move tasks to the nearest ancestor with execution
	 * resources.  This is full cgroup operation which will
2168 2169
	 * also call back into cpuset.  Should be done outside any lock.
	 */
2170
	if (!sane && is_empty)
2171
		remove_tasks_in_empty_cpuset(cs);
2172 2173
}

2174
/**
2175
 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
2176
 *
2177 2178 2179 2180 2181
 * This function is called after either CPU or memory configuration has
 * changed and updates cpuset accordingly.  The top_cpuset is always
 * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
 * order to make cpusets transparent (of no affect) on systems that are
 * actively using CPU hotplug but making no active use of cpusets.
2182
 *
2183
 * Non-root cpusets are only affected by offlining.  If any CPUs or memory
2184 2185
 * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
 * all descendants.
2186
 *
2187 2188
 * Note that CPU offlining during suspend is ignored.  We don't modify
 * cpusets across suspend/resume cycles at all.
2189
 */
2190
static void cpuset_hotplug_workfn(struct work_struct *work)
2191
{
2192 2193
	static cpumask_t new_cpus;
	static nodemask_t new_mems;
2194
	bool cpus_updated, mems_updated;
2195

2196
	mutex_lock(&cpuset_mutex);
2197

2198 2199 2200
	/* fetch the available cpus/mems and find out which changed how */
	cpumask_copy(&new_cpus, cpu_active_mask);
	new_mems = node_states[N_MEMORY];
2201

2202 2203
	cpus_updated = !cpumask_equal(top_cpuset.cpus_allowed, &new_cpus);
	mems_updated = !nodes_equal(top_cpuset.mems_allowed, new_mems);
2204

2205 2206 2207 2208 2209 2210 2211
	/* synchronize cpus_allowed to cpu_active_mask */
	if (cpus_updated) {
		mutex_lock(&callback_mutex);
		cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
		mutex_unlock(&callback_mutex);
		/* we don't mess with cpumasks of tasks in top_cpuset */
	}
2212

2213 2214 2215 2216 2217
	/* synchronize mems_allowed to N_MEMORY */
	if (mems_updated) {
		mutex_lock(&callback_mutex);
		top_cpuset.mems_allowed = new_mems;
		mutex_unlock(&callback_mutex);
2218
		update_tasks_nodemask(&top_cpuset, NULL);
2219
	}
2220

2221 2222
	mutex_unlock(&cpuset_mutex);

2223 2224
	/* if cpus or mems changed, we need to propagate to descendants */
	if (cpus_updated || mems_updated) {
2225
		struct cpuset *cs;
2226
		struct cgroup_subsys_state *pos_css;
2227

2228
		rcu_read_lock();
2229
		cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
2230
			if (cs == &top_cpuset || !css_tryget(&cs->css))
2231 2232
				continue;
			rcu_read_unlock();
2233

2234
			cpuset_hotplug_update_tasks(cs);
2235

2236 2237 2238 2239 2240
			rcu_read_lock();
			css_put(&cs->css);
		}
		rcu_read_unlock();
	}
2241

2242
	/* rebuild sched domains if cpus_allowed has changed */
2243 2244
	if (cpus_updated)
		rebuild_sched_domains();
2245 2246
}

2247
void cpuset_update_active_cpus(bool cpu_online)
2248
{
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
	/*
	 * We're inside cpu hotplug critical region which usually nests
	 * inside cgroup synchronization.  Bounce actual hotplug processing
	 * to a work item to avoid reverse locking order.
	 *
	 * We still need to do partition_sched_domains() synchronously;
	 * otherwise, the scheduler will get confused and put tasks to the
	 * dead CPU.  Fall back to the default single domain.
	 * cpuset_hotplug_workfn() will rebuild it as necessary.
	 */
	partition_sched_domains(1, NULL, NULL);
	schedule_work(&cpuset_hotplug_work);
2261 2262
}

2263
/*
2264 2265
 * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
 * Call this routine anytime after node_states[N_MEMORY] changes.
2266
 * See cpuset_update_active_cpus() for CPU hotplug handling.
2267
 */
2268 2269
static int cpuset_track_online_nodes(struct notifier_block *self,
				unsigned long action, void *arg)
2270
{
2271
	schedule_work(&cpuset_hotplug_work);
2272
	return NOTIFY_OK;
2273
}
2274 2275 2276 2277 2278

static struct notifier_block cpuset_track_online_nodes_nb = {
	.notifier_call = cpuset_track_online_nodes,
	.priority = 10,		/* ??! */
};
2279

L
Linus Torvalds 已提交
2280 2281 2282 2283
/**
 * cpuset_init_smp - initialize cpus_allowed
 *
 * Description: Finish top cpuset after cpu, node maps are initialized
2284
 */
L
Linus Torvalds 已提交
2285 2286
void __init cpuset_init_smp(void)
{
2287
	cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
2288
	top_cpuset.mems_allowed = node_states[N_MEMORY];
2289
	top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
2290

2291
	register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
L
Linus Torvalds 已提交
2292 2293 2294 2295 2296
}

/**
 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2297
 * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
L
Linus Torvalds 已提交
2298
 *
2299
 * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
L
Linus Torvalds 已提交
2300
 * attached to the specified @tsk.  Guaranteed to return some non-empty
2301
 * subset of cpu_online_mask, even if this means going outside the
L
Linus Torvalds 已提交
2302 2303 2304
 * tasks cpuset.
 **/

2305
void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
L
Linus Torvalds 已提交
2306
{
2307 2308
	struct cpuset *cpus_cs;

2309
	mutex_lock(&callback_mutex);
2310
	task_lock(tsk);
2311 2312
	cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
	guarantee_online_cpus(cpus_cs, pmask);
2313
	task_unlock(tsk);
2314
	mutex_unlock(&callback_mutex);
L
Linus Torvalds 已提交
2315 2316
}

2317
void cpuset_cpus_allowed_fallback(struct task_struct *tsk)
2318
{
2319
	struct cpuset *cpus_cs;
2320 2321

	rcu_read_lock();
2322 2323
	cpus_cs = effective_cpumask_cpuset(task_cs(tsk));
	do_set_cpus_allowed(tsk, cpus_cs->cpus_allowed);
2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
	rcu_read_unlock();

	/*
	 * We own tsk->cpus_allowed, nobody can change it under us.
	 *
	 * But we used cs && cs->cpus_allowed lockless and thus can
	 * race with cgroup_attach_task() or update_cpumask() and get
	 * the wrong tsk->cpus_allowed. However, both cases imply the
	 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
	 * which takes task_rq_lock().
	 *
	 * If we are called after it dropped the lock we must see all
	 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
	 * set any mask even if it is not right from task_cs() pov,
	 * the pending set_cpus_allowed_ptr() will fix things.
2339 2340 2341
	 *
	 * select_fallback_rq() will fix things ups and set cpu_possible_mask
	 * if required.
2342 2343 2344
	 */
}

L
Linus Torvalds 已提交
2345 2346
void cpuset_init_current_mems_allowed(void)
{
2347
	nodes_setall(current->mems_allowed);
L
Linus Torvalds 已提交
2348 2349
}

2350 2351 2352 2353 2354 2355
/**
 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
 *
 * Description: Returns the nodemask_t mems_allowed of the cpuset
 * attached to the specified @tsk.  Guaranteed to return some non-empty
2356
 * subset of node_states[N_MEMORY], even if this means going outside the
2357 2358 2359 2360 2361
 * tasks cpuset.
 **/

nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
{
2362
	struct cpuset *mems_cs;
2363 2364
	nodemask_t mask;

2365
	mutex_lock(&callback_mutex);
2366
	task_lock(tsk);
2367 2368
	mems_cs = effective_nodemask_cpuset(task_cs(tsk));
	guarantee_online_mems(mems_cs, &mask);
2369
	task_unlock(tsk);
2370
	mutex_unlock(&callback_mutex);
2371 2372 2373 2374

	return mask;
}

2375
/**
2376 2377
 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
 * @nodemask: the nodemask to be checked
2378
 *
2379
 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
L
Linus Torvalds 已提交
2380
 */
2381
int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
L
Linus Torvalds 已提交
2382
{
2383
	return nodes_intersects(*nodemask, current->mems_allowed);
L
Linus Torvalds 已提交
2384 2385
}

2386
/*
2387 2388 2389 2390
 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
 * mem_hardwall ancestor to the specified cpuset.  Call holding
 * callback_mutex.  If no ancestor is mem_exclusive or mem_hardwall
 * (an unusual configuration), then returns the root cpuset.
2391
 */
2392
static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
2393
{
T
Tejun Heo 已提交
2394 2395
	while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
		cs = parent_cs(cs);
2396 2397 2398
	return cs;
}

2399
/**
2400 2401
 * cpuset_node_allowed_softwall - Can we allocate on a memory node?
 * @node: is this an allowed node?
2402
 * @gfp_mask: memory allocation flags
2403
 *
2404 2405 2406 2407 2408 2409
 * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
 * set, yes, we can always allocate.  If node is in our task's mems_allowed,
 * yes.  If it's not a __GFP_HARDWALL request and this node is in the nearest
 * hardwalled cpuset ancestor to this task's cpuset, yes.  If the task has been
 * OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
 * flag, yes.
2410 2411
 * Otherwise, no.
 *
2412 2413 2414
 * If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
 * cpuset_node_allowed_hardwall().  Otherwise, cpuset_node_allowed_softwall()
 * might sleep, and might allow a node from an enclosing cpuset.
2415
 *
2416 2417
 * cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
 * cpusets, and never sleeps.
2418 2419 2420 2421 2422 2423 2424
 *
 * The __GFP_THISNODE placement logic is really handled elsewhere,
 * by forcibly using a zonelist starting at a specified node, and by
 * (in get_page_from_freelist()) refusing to consider the zones for
 * any node on the zonelist except the first.  By the time any such
 * calls get to this routine, we should just shut up and say 'yes'.
 *
2425
 * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2426 2427
 * and do not allow allocations outside the current tasks cpuset
 * unless the task has been OOM killed as is marked TIF_MEMDIE.
2428
 * GFP_KERNEL allocations are not so marked, so can escape to the
2429
 * nearest enclosing hardwalled ancestor cpuset.
2430
 *
2431 2432 2433 2434 2435 2436 2437
 * Scanning up parent cpusets requires callback_mutex.  The
 * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
 * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
 * current tasks mems_allowed came up empty on the first pass over
 * the zonelist.  So only GFP_KERNEL allocations, if all nodes in the
 * cpuset are short of memory, might require taking the callback_mutex
 * mutex.
2438
 *
2439
 * The first call here from mm/page_alloc:get_page_from_freelist()
2440 2441 2442
 * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
 * so no allocation on a node outside the cpuset is allowed (unless
 * in interrupt, of course).
2443 2444 2445 2446 2447 2448
 *
 * The second pass through get_page_from_freelist() doesn't even call
 * here for GFP_ATOMIC calls.  For those calls, the __alloc_pages()
 * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
 * in alloc_flags.  That logic and the checks below have the combined
 * affect that:
2449 2450
 *	in_interrupt - any node ok (current task context irrelevant)
 *	GFP_ATOMIC   - any node ok
2451
 *	TIF_MEMDIE   - any node ok
2452
 *	GFP_KERNEL   - any node in enclosing hardwalled cpuset ok
2453
 *	GFP_USER     - only nodes in current tasks mems allowed ok.
2454 2455
 *
 * Rule:
2456
 *    Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
2457 2458
 *    pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
 *    the code that might scan up ancestor cpusets and sleep.
2459
 */
2460
int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
L
Linus Torvalds 已提交
2461
{
2462
	struct cpuset *cs;		/* current cpuset ancestors */
2463
	int allowed;			/* is allocation in zone z allowed? */
2464

2465
	if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
2466
		return 1;
2467
	might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
2468 2469
	if (node_isset(node, current->mems_allowed))
		return 1;
2470 2471 2472 2473 2474 2475
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return 1;
2476 2477 2478
	if (gfp_mask & __GFP_HARDWALL)	/* If hardwall request, stop here */
		return 0;

2479 2480 2481
	if (current->flags & PF_EXITING) /* Let dying task have memory */
		return 1;

2482
	/* Not hardwall and node outside mems_allowed: scan up cpusets */
2483
	mutex_lock(&callback_mutex);
2484 2485

	task_lock(current);
2486
	cs = nearest_hardwall_ancestor(task_cs(current));
2487 2488
	task_unlock(current);

2489
	allowed = node_isset(node, cs->mems_allowed);
2490
	mutex_unlock(&callback_mutex);
2491
	return allowed;
L
Linus Torvalds 已提交
2492 2493
}

2494
/*
2495 2496
 * cpuset_node_allowed_hardwall - Can we allocate on a memory node?
 * @node: is this an allowed node?
2497 2498
 * @gfp_mask: memory allocation flags
 *
2499 2500 2501 2502 2503
 * If we're in interrupt, yes, we can always allocate.  If __GFP_THISNODE is
 * set, yes, we can always allocate.  If node is in our task's mems_allowed,
 * yes.  If the task has been OOM killed and has access to memory reserves as
 * specified by the TIF_MEMDIE flag, yes.
 * Otherwise, no.
2504 2505 2506 2507 2508 2509 2510
 *
 * The __GFP_THISNODE placement logic is really handled elsewhere,
 * by forcibly using a zonelist starting at a specified node, and by
 * (in get_page_from_freelist()) refusing to consider the zones for
 * any node on the zonelist except the first.  By the time any such
 * calls get to this routine, we should just shut up and say 'yes'.
 *
2511 2512
 * Unlike the cpuset_node_allowed_softwall() variant, above,
 * this variant requires that the node be in the current task's
2513 2514 2515 2516
 * mems_allowed or that we're in interrupt.  It does not scan up the
 * cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
 * It never sleeps.
 */
2517
int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
2518 2519 2520 2521 2522
{
	if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
		return 1;
	if (node_isset(node, current->mems_allowed))
		return 1;
D
Daniel Walker 已提交
2523 2524 2525 2526 2527 2528
	/*
	 * Allow tasks that have access to memory reserves because they have
	 * been OOM killed to get memory anywhere.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)))
		return 1;
2529 2530 2531
	return 0;
}

2532
/**
2533 2534
 * cpuset_mem_spread_node() - On which node to begin search for a file page
 * cpuset_slab_spread_node() - On which node to begin search for a slab page
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
 *
 * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
 * tasks in a cpuset with is_spread_page or is_spread_slab set),
 * and if the memory allocation used cpuset_mem_spread_node()
 * to determine on which node to start looking, as it will for
 * certain page cache or slab cache pages such as used for file
 * system buffers and inode caches, then instead of starting on the
 * local node to look for a free page, rather spread the starting
 * node around the tasks mems_allowed nodes.
 *
 * We don't have to worry about the returned node being offline
 * because "it can't happen", and even if it did, it would be ok.
 *
 * The routines calling guarantee_online_mems() are careful to
 * only set nodes in task->mems_allowed that are online.  So it
 * should not be possible for the following code to return an
 * offline node.  But if it did, that would be ok, as this routine
 * is not returning the node where the allocation must be, only
 * the node where the search should start.  The zonelist passed to
 * __alloc_pages() will include all nodes.  If the slab allocator
 * is passed an offline node, it will fall back to the local node.
 * See kmem_cache_alloc_node().
 */

2559
static int cpuset_spread_node(int *rotor)
2560 2561 2562
{
	int node;

2563
	node = next_node(*rotor, current->mems_allowed);
2564 2565
	if (node == MAX_NUMNODES)
		node = first_node(current->mems_allowed);
2566
	*rotor = node;
2567 2568
	return node;
}
2569 2570 2571

int cpuset_mem_spread_node(void)
{
2572 2573 2574 2575
	if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
		current->cpuset_mem_spread_rotor =
			node_random(&current->mems_allowed);

2576 2577 2578 2579 2580
	return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
}

int cpuset_slab_spread_node(void)
{
2581 2582 2583 2584
	if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
		current->cpuset_slab_spread_rotor =
			node_random(&current->mems_allowed);

2585 2586 2587
	return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
}

2588 2589
EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);

2590
/**
2591 2592 2593 2594 2595 2596 2597 2598
 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
 * @tsk1: pointer to task_struct of some task.
 * @tsk2: pointer to task_struct of some other task.
 *
 * Description: Return true if @tsk1's mems_allowed intersects the
 * mems_allowed of @tsk2.  Used by the OOM killer to determine if
 * one of the task's memory usage might impact the memory available
 * to the other.
2599 2600
 **/

2601 2602
int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
				   const struct task_struct *tsk2)
2603
{
2604
	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
2605 2606
}

2607 2608
#define CPUSET_NODELIST_LEN	(256)

2609 2610 2611 2612 2613 2614 2615 2616 2617 2618
/**
 * cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
 * @task: pointer to task_struct of some task.
 *
 * Description: Prints @task's name, cpuset name, and cached copy of its
 * mems_allowed to the kernel log.  Must hold task_lock(task) to allow
 * dereferencing task_cs(task).
 */
void cpuset_print_task_mems_allowed(struct task_struct *tsk)
{
2619 2620 2621
	 /* Statically allocated to prevent using excess stack. */
	static char cpuset_nodelist[CPUSET_NODELIST_LEN];
	static DEFINE_SPINLOCK(cpuset_buffer_lock);
2622

2623
	struct cgroup *cgrp = task_cs(tsk)->css.cgroup;
2624

2625
	rcu_read_lock();
2626
	spin_lock(&cpuset_buffer_lock);
2627

2628 2629 2630
	nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
			   tsk->mems_allowed);
	printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",
2631 2632
	       tsk->comm, cgroup_name(cgrp), cpuset_nodelist);

2633
	spin_unlock(&cpuset_buffer_lock);
2634
	rcu_read_unlock();
2635 2636
}

2637 2638 2639 2640 2641 2642
/*
 * Collection of memory_pressure is suppressed unless
 * this flag is enabled by writing "1" to the special
 * cpuset file 'memory_pressure_enabled' in the root cpuset.
 */

2643
int cpuset_memory_pressure_enabled __read_mostly;
2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665

/**
 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
 *
 * Keep a running average of the rate of synchronous (direct)
 * page reclaim efforts initiated by tasks in each cpuset.
 *
 * This represents the rate at which some task in the cpuset
 * ran low on memory on all nodes it was allowed to use, and
 * had to enter the kernels page reclaim code in an effort to
 * create more free memory by tossing clean pages or swapping
 * or writing dirty pages.
 *
 * Display to user space in the per-cpuset read-only file
 * "memory_pressure".  Value displayed is an integer
 * representing the recent rate of entry into the synchronous
 * (direct) page reclaim by any task attached to the cpuset.
 **/

void __cpuset_memory_pressure_bump(void)
{
	task_lock(current);
2666
	fmeter_markevent(&task_cs(current)->fmeter);
2667 2668 2669
	task_unlock(current);
}

2670
#ifdef CONFIG_PROC_PID_CPUSET
L
Linus Torvalds 已提交
2671 2672 2673 2674
/*
 * proc_cpuset_show()
 *  - Print tasks cpuset path into seq_file.
 *  - Used for /proc/<pid>/cpuset.
2675 2676
 *  - No need to task_lock(tsk) on this tsk->cpuset reference, as it
 *    doesn't really matter if tsk->cpuset changes after we read it,
2677
 *    and we take cpuset_mutex, keeping cpuset_attach() from changing it
2678
 *    anyway.
L
Linus Torvalds 已提交
2679
 */
2680
int proc_cpuset_show(struct seq_file *m, void *unused_v)
L
Linus Torvalds 已提交
2681
{
2682
	struct pid *pid;
L
Linus Torvalds 已提交
2683 2684
	struct task_struct *tsk;
	char *buf;
2685
	struct cgroup_subsys_state *css;
2686
	int retval;
L
Linus Torvalds 已提交
2687

2688
	retval = -ENOMEM;
L
Linus Torvalds 已提交
2689 2690
	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
	if (!buf)
2691 2692 2693
		goto out;

	retval = -ESRCH;
2694 2695
	pid = m->private;
	tsk = get_pid_task(pid, PIDTYPE_PID);
2696 2697
	if (!tsk)
		goto out_free;
L
Linus Torvalds 已提交
2698

L
Li Zefan 已提交
2699
	rcu_read_lock();
2700
	css = task_css(tsk, cpuset_cgrp_id);
2701
	retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
L
Li Zefan 已提交
2702
	rcu_read_unlock();
L
Linus Torvalds 已提交
2703
	if (retval < 0)
L
Li Zefan 已提交
2704
		goto out_put_task;
L
Linus Torvalds 已提交
2705 2706
	seq_puts(m, buf);
	seq_putc(m, '\n');
L
Li Zefan 已提交
2707
out_put_task:
2708 2709
	put_task_struct(tsk);
out_free:
L
Linus Torvalds 已提交
2710
	kfree(buf);
2711
out:
L
Linus Torvalds 已提交
2712 2713
	return retval;
}
2714
#endif /* CONFIG_PROC_PID_CPUSET */
L
Linus Torvalds 已提交
2715

2716
/* Display task mems_allowed in /proc/<pid>/status file. */
2717 2718 2719
void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
{
	seq_printf(m, "Mems_allowed:\t");
2720
	seq_nodemask(m, &task->mems_allowed);
2721
	seq_printf(m, "\n");
2722
	seq_printf(m, "Mems_allowed_list:\t");
2723
	seq_nodemask_list(m, &task->mems_allowed);
2724
	seq_printf(m, "\n");
L
Linus Torvalds 已提交
2725
}