memcontrol.c 132.6 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
B
Balbir Singh 已提交
13 14 15 16 17 18 19 20 21 22 23 24 25 26
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
27
#include <linux/mm.h>
28
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
29
#include <linux/pagemap.h>
30
#include <linux/smp.h>
31
#include <linux/page-flags.h>
32
#include <linux/backing-dev.h>
33 34
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
35
#include <linux/limits.h>
36
#include <linux/mutex.h>
37
#include <linux/rbtree.h>
38
#include <linux/slab.h>
39
#include <linux/swap.h>
40
#include <linux/swapops.h>
41
#include <linux/spinlock.h>
42 43
#include <linux/eventfd.h>
#include <linux/sort.h>
44
#include <linux/fs.h>
45
#include <linux/seq_file.h>
46
#include <linux/vmalloc.h>
47
#include <linux/mm_inline.h>
48
#include <linux/page_cgroup.h>
49
#include <linux/cpu.h>
50
#include <linux/oom.h>
K
KAMEZAWA Hiroyuki 已提交
51
#include "internal.h"
B
Balbir Singh 已提交
52

53 54
#include <asm/uaccess.h>

55 56
#include <trace/events/vmscan.h>

57 58
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
#define MEM_CGROUP_RECLAIM_RETRIES	5
59
struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
60

61
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
L
Li Zefan 已提交
62
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
63
int do_swap_account __read_mostly;
64 65 66 67 68 69 70 71

/* for remember boot option*/
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata = 0;
#endif

72 73 74 75 76
#else
#define do_swap_account		(0)
#endif


77 78 79 80 81 82 83 84
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
85
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as anon rss */
86
	MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
87
	MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
88
	MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
89
	MEM_CGROUP_ON_MOVE,	/* someone is moving account between groups */
90 91 92
	MEM_CGROUP_STAT_NSTATS,
};

93 94 95 96 97 98
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
	MEM_CGROUP_EVENTS_COUNT,	/* # of pages paged in/out */
	MEM_CGROUP_EVENTS_NSTATS,
};
99 100 101 102 103 104 105 106 107 108 109 110 111
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_NTARGETS,
};
#define THRESHOLDS_EVENTS_TARGET (128)
#define SOFTLIMIT_EVENTS_TARGET (1024)
112

113
struct mem_cgroup_stat_cpu {
114
	long count[MEM_CGROUP_STAT_NSTATS];
115
	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
116
	unsigned long targets[MEM_CGROUP_NTARGETS];
117 118
};

119 120 121 122
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
123 124 125
	/*
	 * spin_lock to protect the per cgroup LRU
	 */
126 127
	struct list_head	lists[NR_LRU_LISTS];
	unsigned long		count[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
128 129

	struct zone_reclaim_stat reclaim_stat;
130 131 132 133
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
134 135
	struct mem_cgroup	*mem;		/* Back pointer, we cannot */
						/* use container_of	   */
136 137 138 139 140 141 142 143 144 145 146 147
};
/* Macro for accessing counter */
#define MEM_CGROUP_ZSTAT(mz, idx)	((mz)->count[(idx)])

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
	struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
};

148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

168 169 170 171 172
struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	u64 threshold;
};

K
KAMEZAWA Hiroyuki 已提交
173
/* For threshold */
174 175
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below usage. */
176
	int current_threshold;
177 178 179 180 181
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};
182 183 184 185 186 187 188 189 190 191 192 193

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

K
KAMEZAWA Hiroyuki 已提交
194 195 196 197 198
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
199 200

static void mem_cgroup_threshold(struct mem_cgroup *mem);
K
KAMEZAWA Hiroyuki 已提交
201
static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
202

B
Balbir Singh 已提交
203 204 205 206 207 208 209
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
210 211 212
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
213 214 215 216 217 218 219
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
220 221 222 223
	/*
	 * the counter to account for mem+swap usage.
	 */
	struct res_counter memsw;
224 225 226 227
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 */
228
	struct mem_cgroup_lru_info info;
229
	/*
230
	 * While reclaiming in a hierarchy, we cache the last child we
K
KAMEZAWA Hiroyuki 已提交
231
	 * reclaimed from.
232
	 */
K
KAMEZAWA Hiroyuki 已提交
233
	int last_scanned_child;
234 235 236 237
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
K
KAMEZAWA Hiroyuki 已提交
238
	atomic_t	oom_lock;
239
	atomic_t	refcnt;
240

K
KOSAKI Motohiro 已提交
241
	unsigned int	swappiness;
242 243
	/* OOM-Killer disable */
	int		oom_kill_disable;
K
KOSAKI Motohiro 已提交
244

245 246 247
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

248 249 250 251
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
252
	struct mem_cgroup_thresholds thresholds;
253

254
	/* thresholds for mem+swap usage. RCU-protected */
255
	struct mem_cgroup_thresholds memsw_thresholds;
256

K
KAMEZAWA Hiroyuki 已提交
257 258 259
	/* For oom notifier event fd */
	struct list_head oom_notify;

260 261 262 263 264
	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long 	move_charge_at_immigrate;
265
	/*
266
	 * percpu counter.
267
	 */
268
	struct mem_cgroup_stat_cpu *stat;
269 270 271 272 273 274
	/*
	 * used when a cpu is offlined or other synchronizations
	 * See mem_cgroup_read_stat().
	 */
	struct mem_cgroup_stat_cpu nocpu_base;
	spinlock_t pcp_counter_lock;
B
Balbir Singh 已提交
275 276
};

277 278 279 280 281 282
/* Stuffs for move charges at task migration. */
/*
 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
 * left-shifted bitmap of these types.
 */
enum move_type {
283
	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */
284
	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */
285 286 287
	NR_MOVE_TYPE,
};

288 289
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
290
	spinlock_t	  lock; /* for from, to */
291 292 293
	struct mem_cgroup *from;
	struct mem_cgroup *to;
	unsigned long precharge;
294
	unsigned long moved_charge;
295
	unsigned long moved_swap;
296 297 298
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
299
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
300 301
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
302

D
Daisuke Nishimura 已提交
303 304 305 306 307 308
static bool move_anon(void)
{
	return test_bit(MOVE_CHARGE_TYPE_ANON,
					&mc.to->move_charge_at_immigrate);
}

309 310 311 312 313 314
static bool move_file(void)
{
	return test_bit(MOVE_CHARGE_TYPE_FILE,
					&mc.to->move_charge_at_immigrate);
}

315 316 317 318 319 320 321
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		(100)
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	(2)

322 323 324
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
	MEM_CGROUP_CHARGE_TYPE_MAPPED,
325
	MEM_CGROUP_CHARGE_TYPE_SHMEM,	/* used by page migration of shmem */
326
	MEM_CGROUP_CHARGE_TYPE_FORCE,	/* used by force_empty */
K
KAMEZAWA Hiroyuki 已提交
327
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
328
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
329 330 331
	NR_CHARGE_TYPE,
};

332 333 334
/* for encoding cft->private value on file */
#define _MEM			(0)
#define _MEMSWAP		(1)
K
KAMEZAWA Hiroyuki 已提交
335
#define _OOM_TYPE		(2)
336 337 338
#define MEMFILE_PRIVATE(x, val)	(((x) << 16) | (val))
#define MEMFILE_TYPE(val)	(((val) >> 16) & 0xffff)
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
339 340
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
341

342 343 344 345 346 347 348
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
349 350
#define MEM_CGROUP_RECLAIM_SOFT_BIT	0x2
#define MEM_CGROUP_RECLAIM_SOFT		(1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
351

352 353
static void mem_cgroup_get(struct mem_cgroup *mem);
static void mem_cgroup_put(struct mem_cgroup *mem);
354
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
355
static void drain_all_stock_async(void);
356

357 358 359 360 361 362
static struct mem_cgroup_per_zone *
mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
{
	return &mem->info.nodeinfo[nid]->zoneinfo[zid];
}

363 364 365 366 367
struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
{
	return &mem->css;
}

368
static struct mem_cgroup_per_zone *
369
page_cgroup_zoneinfo(struct mem_cgroup *mem, struct page *page)
370
{
371 372
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392

	return mem_cgroup_zoneinfo(mem, nid, zid);
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
393
__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
394
				struct mem_cgroup_per_zone *mz,
395 396
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
397 398 399 400 401 402 403 404
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

405 406 407
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
424 425 426 427 428 429 430 431 432 433 434 435 436
}

static void
__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

437 438 439 440 441 442
static void
mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
443
	__mem_cgroup_remove_exceeded(mem, mz, mctz);
444 445 446 447 448 449
	spin_unlock(&mctz->lock);
}


static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
{
450
	unsigned long long excess;
451 452
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
453 454
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
455 456 457
	mctz = soft_limit_tree_from_page(page);

	/*
458 459
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
460
	 */
461 462
	for (; mem; mem = parent_mem_cgroup(mem)) {
		mz = mem_cgroup_zoneinfo(mem, nid, zid);
463
		excess = res_counter_soft_limit_excess(&mem->res);
464 465 466 467
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
468
		if (excess || mz->on_tree) {
469 470 471 472 473
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
				__mem_cgroup_remove_exceeded(mem, mz, mctz);
			/*
474 475
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
476
			 */
477
			__mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
478 479
			spin_unlock(&mctz->lock);
		}
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

	for_each_node_state(node, N_POSSIBLE) {
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			mz = mem_cgroup_zoneinfo(mem, node, zone);
			mctz = soft_limit_tree_node_zone(node, zone);
			mem_cgroup_remove_exceeded(mem, mz, mctz);
		}
	}
}

498 499 500 501
static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
502
	struct mem_cgroup_per_zone *mz;
503 504

retry:
505
	mz = NULL;
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
	__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->mem->res) ||
		!css_tryget(&mz->mem->css))
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
554 555
static long mem_cgroup_read_stat(struct mem_cgroup *mem,
				 enum mem_cgroup_stat_index idx)
556
{
557
	long val = 0;
558 559
	int cpu;

560 561
	get_online_cpus();
	for_each_online_cpu(cpu)
562
		val += per_cpu(mem->stat->count[idx], cpu);
563 564 565 566 567 568
#ifdef CONFIG_HOTPLUG_CPU
	spin_lock(&mem->pcp_counter_lock);
	val += mem->nocpu_base.count[idx];
	spin_unlock(&mem->pcp_counter_lock);
#endif
	put_online_cpus();
569 570 571
	return val;
}

572
static long mem_cgroup_local_usage(struct mem_cgroup *mem)
573
{
574
	long ret;
575 576 577 578 579 580

	ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
	ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
	return ret;
}

581 582 583 584
static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
					 bool charge)
{
	int val = (charge) ? 1 : -1;
585
	this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
586 587
}

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
static unsigned long mem_cgroup_read_events(struct mem_cgroup *mem,
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

	for_each_online_cpu(cpu)
		val += per_cpu(mem->stat->events[idx], cpu);
#ifdef CONFIG_HOTPLUG_CPU
	spin_lock(&mem->pcp_counter_lock);
	val += mem->nocpu_base.events[idx];
	spin_unlock(&mem->pcp_counter_lock);
#endif
	return val;
}

604
static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
605
					 bool file, int nr_pages)
606
{
607 608
	preempt_disable();

609 610
	if (file)
		__this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
611
	else
612
		__this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
613

614 615
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
616
		__this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
617
	else {
618
		__this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
619 620
		nr_pages = -nr_pages; /* for event */
	}
621

622
	__this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
623

624
	preempt_enable();
625 626
}

K
KAMEZAWA Hiroyuki 已提交
627
static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
628
					enum lru_list idx)
629 630 631 632 633 634 635 636 637 638 639
{
	int nid, zid;
	struct mem_cgroup_per_zone *mz;
	u64 total = 0;

	for_each_online_node(nid)
		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			mz = mem_cgroup_zoneinfo(mem, nid, zid);
			total += MEM_CGROUP_ZSTAT(mz, idx);
		}
	return total;
640 641
}

642 643 644 645 646 647 648 649 650 651 652
static bool __memcg_event_check(struct mem_cgroup *mem, int target)
{
	unsigned long val, next;

	val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
	next = this_cpu_read(mem->stat->targets[target]);
	/* from time_after() in jiffies.h */
	return ((long)next - (long)val < 0);
}

static void __mem_cgroup_target_update(struct mem_cgroup *mem, int target)
653
{
654
	unsigned long val, next;
655

656
	val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
657

658 659 660 661 662 663 664 665 666 667 668 669
	switch (target) {
	case MEM_CGROUP_TARGET_THRESH:
		next = val + THRESHOLDS_EVENTS_TARGET;
		break;
	case MEM_CGROUP_TARGET_SOFTLIMIT:
		next = val + SOFTLIMIT_EVENTS_TARGET;
		break;
	default:
		return;
	}

	this_cpu_write(mem->stat->targets[target], next);
670 671 672 673 674 675 676 677 678
}

/*
 * Check events in order.
 *
 */
static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
{
	/* threshold event is triggered in finer grain than soft limit */
679
	if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_THRESH))) {
680
		mem_cgroup_threshold(mem);
681 682 683
		__mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_THRESH);
		if (unlikely(__memcg_event_check(mem,
			MEM_CGROUP_TARGET_SOFTLIMIT))){
684
			mem_cgroup_update_tree(mem, page);
685 686 687
			__mem_cgroup_target_update(mem,
				MEM_CGROUP_TARGET_SOFTLIMIT);
		}
688 689 690
	}
}

691
static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
692 693 694 695 696 697
{
	return container_of(cgroup_subsys_state(cont,
				mem_cgroup_subsys_id), struct mem_cgroup,
				css);
}

698
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
699
{
700 701 702 703 704 705 706 707
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

708 709 710 711
	return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
				struct mem_cgroup, css);
}

712 713 714
static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
{
	struct mem_cgroup *mem = NULL;
715 716 717

	if (!mm)
		return NULL;
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
		mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!mem))
			break;
	} while (!css_tryget(&mem->css));
	rcu_read_unlock();
	return mem;
}

K
KAMEZAWA Hiroyuki 已提交
733 734
/* The caller has to guarantee "mem" exists before calling this */
static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
K
KAMEZAWA Hiroyuki 已提交
735
{
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757
	struct cgroup_subsys_state *css;
	int found;

	if (!mem) /* ROOT cgroup has the smallest ID */
		return root_mem_cgroup; /*css_put/get against root is ignored*/
	if (!mem->use_hierarchy) {
		if (css_tryget(&mem->css))
			return mem;
		return NULL;
	}
	rcu_read_lock();
	/*
	 * searching a memory cgroup which has the smallest ID under given
	 * ROOT cgroup. (ID >= 1)
	 */
	css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
	if (css && css_tryget(css))
		mem = container_of(css, struct mem_cgroup, css);
	else
		mem = NULL;
	rcu_read_unlock();
	return mem;
K
KAMEZAWA Hiroyuki 已提交
758 759 760 761 762 763 764 765 766
}

static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
					struct mem_cgroup *root,
					bool cond)
{
	int nextid = css_id(&iter->css) + 1;
	int found;
	int hierarchy_used;
K
KAMEZAWA Hiroyuki 已提交
767 768
	struct cgroup_subsys_state *css;

K
KAMEZAWA Hiroyuki 已提交
769
	hierarchy_used = iter->use_hierarchy;
K
KAMEZAWA Hiroyuki 已提交
770

K
KAMEZAWA Hiroyuki 已提交
771
	css_put(&iter->css);
772 773
	/* If no ROOT, walk all, ignore hierarchy */
	if (!cond || (root && !hierarchy_used))
K
KAMEZAWA Hiroyuki 已提交
774
		return NULL;
K
KAMEZAWA Hiroyuki 已提交
775

776 777 778
	if (!root)
		root = root_mem_cgroup;

K
KAMEZAWA Hiroyuki 已提交
779 780
	do {
		iter = NULL;
K
KAMEZAWA Hiroyuki 已提交
781
		rcu_read_lock();
K
KAMEZAWA Hiroyuki 已提交
782 783 784

		css = css_get_next(&mem_cgroup_subsys, nextid,
				&root->css, &found);
K
KAMEZAWA Hiroyuki 已提交
785
		if (css && css_tryget(css))
K
KAMEZAWA Hiroyuki 已提交
786
			iter = container_of(css, struct mem_cgroup, css);
K
KAMEZAWA Hiroyuki 已提交
787
		rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
788
		/* If css is NULL, no more cgroups will be found */
K
KAMEZAWA Hiroyuki 已提交
789
		nextid = found + 1;
K
KAMEZAWA Hiroyuki 已提交
790
	} while (css && !iter);
K
KAMEZAWA Hiroyuki 已提交
791

K
KAMEZAWA Hiroyuki 已提交
792
	return iter;
K
KAMEZAWA Hiroyuki 已提交
793
}
K
KAMEZAWA Hiroyuki 已提交
794 795 796 797 798 799 800 801 802 803 804 805 806
/*
 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
 * be careful that "break" loop is not allowed. We have reference count.
 * Instead of that modify "cond" to be false and "continue" to exit the loop.
 */
#define for_each_mem_cgroup_tree_cond(iter, root, cond)	\
	for (iter = mem_cgroup_start_loop(root);\
	     iter != NULL;\
	     iter = mem_cgroup_get_next(iter, root, cond))

#define for_each_mem_cgroup_tree(iter, root) \
	for_each_mem_cgroup_tree_cond(iter, root, true)

807 808 809
#define for_each_mem_cgroup_all(iter) \
	for_each_mem_cgroup_tree_cond(iter, NULL, true)

K
KAMEZAWA Hiroyuki 已提交
810

811 812 813 814 815
static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
{
	return (mem == root_mem_cgroup);
}

K
KAMEZAWA Hiroyuki 已提交
816 817 818 819 820 821 822 823 824 825 826 827 828
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
829

K
KAMEZAWA Hiroyuki 已提交
830 831 832 833
void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
834

835
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
836 837 838
		return;
	pc = lookup_page_cgroup(page);
	/* can happen while we handle swapcache. */
839
	if (!TestClearPageCgroupAcctLRU(pc))
K
KAMEZAWA Hiroyuki 已提交
840
		return;
841
	VM_BUG_ON(!pc->mem_cgroup);
842 843 844 845
	/*
	 * We don't check PCG_USED bit. It's cleared when the "page" is finally
	 * removed from global LRU.
	 */
846
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
847 848
	/* huge page split is done under lru_lock. so, we have no races. */
	MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
849 850 851
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
	VM_BUG_ON(list_empty(&pc->lru));
K
KAMEZAWA Hiroyuki 已提交
852
	list_del_init(&pc->lru);
853 854
}

K
KAMEZAWA Hiroyuki 已提交
855
void mem_cgroup_del_lru(struct page *page)
856
{
K
KAMEZAWA Hiroyuki 已提交
857 858
	mem_cgroup_del_lru_list(page, page_lru(page));
}
859

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
/*
 * Writeback is about to end against a page which has been marked for immediate
 * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 * inactive list.
 */
void mem_cgroup_rotate_reclaimable_page(struct page *page)
{
	struct mem_cgroup_per_zone *mz;
	struct page_cgroup *pc;
	enum lru_list lru = page_lru(page);

	if (mem_cgroup_disabled())
		return;

	pc = lookup_page_cgroup(page);
	/* unused or root page is not rotated. */
	if (!PageCgroupUsed(pc))
		return;
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
882
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
883 884 885
	list_move_tail(&pc->lru, &mz->lists[lru]);
}

K
KAMEZAWA Hiroyuki 已提交
886 887 888 889
void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;
	struct page_cgroup *pc;
890

891
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
892
		return;
893

K
KAMEZAWA Hiroyuki 已提交
894
	pc = lookup_page_cgroup(page);
895
	/* unused or root page is not rotated. */
896 897 898 899 900
	if (!PageCgroupUsed(pc))
		return;
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
	if (mem_cgroup_is_root(pc->mem_cgroup))
K
KAMEZAWA Hiroyuki 已提交
901
		return;
902
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
K
KAMEZAWA Hiroyuki 已提交
903
	list_move(&pc->lru, &mz->lists[lru]);
904 905
}

K
KAMEZAWA Hiroyuki 已提交
906
void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
907
{
K
KAMEZAWA Hiroyuki 已提交
908 909
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
910

911
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
912 913
		return;
	pc = lookup_page_cgroup(page);
914
	VM_BUG_ON(PageCgroupAcctLRU(pc));
K
KAMEZAWA Hiroyuki 已提交
915
	if (!PageCgroupUsed(pc))
L
Lee Schermerhorn 已提交
916
		return;
917 918
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
919
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
920 921
	/* huge page split is done under lru_lock. so, we have no races. */
	MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
922 923 924
	SetPageCgroupAcctLRU(pc);
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
K
KAMEZAWA Hiroyuki 已提交
925 926
	list_add(&pc->lru, &mz->lists[lru]);
}
927

K
KAMEZAWA Hiroyuki 已提交
928
/*
929 930 931 932 933
 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
 * lru because the page may.be reused after it's fully uncharged (because of
 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
 * it again. This function is only used to charge SwapCache. It's done under
 * lock_page and expected that zone->lru_lock is never held.
K
KAMEZAWA Hiroyuki 已提交
934
 */
935
static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
K
KAMEZAWA Hiroyuki 已提交
936
{
937 938 939 940 941 942 943 944 945 946 947 948
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);

	spin_lock_irqsave(&zone->lru_lock, flags);
	/*
	 * Forget old LRU when this page_cgroup is *not* used. This Used bit
	 * is guarded by lock_page() because the page is SwapCache.
	 */
	if (!PageCgroupUsed(pc))
		mem_cgroup_del_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
K
KAMEZAWA Hiroyuki 已提交
949 950
}

951 952 953 954 955 956 957 958
static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
{
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);

	spin_lock_irqsave(&zone->lru_lock, flags);
	/* link when the page is linked to LRU but page_cgroup isn't */
959
	if (PageLRU(page) && !PageCgroupAcctLRU(pc))
960 961 962 963 964
		mem_cgroup_add_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
}


K
KAMEZAWA Hiroyuki 已提交
965 966 967
void mem_cgroup_move_lists(struct page *page,
			   enum lru_list from, enum lru_list to)
{
968
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
969 970 971
		return;
	mem_cgroup_del_lru_list(page, from);
	mem_cgroup_add_lru_list(page, to);
972 973
}

974 975 976
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
{
	int ret;
977
	struct mem_cgroup *curr = NULL;
978
	struct task_struct *p;
979

980 981 982 983 984
	p = find_lock_task_mm(task);
	if (!p)
		return 0;
	curr = try_get_mem_cgroup_from_mm(p->mm);
	task_unlock(p);
985 986
	if (!curr)
		return 0;
987 988 989 990 991 992 993
	/*
	 * We should check use_hierarchy of "mem" not "curr". Because checking
	 * use_hierarchy of "curr" here make this function true if hierarchy is
	 * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
	 * hierarchy(even if use_hierarchy is disabled in "mem").
	 */
	if (mem->use_hierarchy)
994 995 996 997
		ret = css_is_ancestor(&curr->css, &mem->css);
	else
		ret = (curr == mem);
	css_put(&curr->css);
998 999 1000
	return ret;
}

1001
static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
1002 1003 1004
{
	unsigned long active;
	unsigned long inactive;
1005 1006
	unsigned long gb;
	unsigned long inactive_ratio;
1007

K
KAMEZAWA Hiroyuki 已提交
1008 1009
	inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
1010

1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

	if (present_pages) {
		present_pages[0] = inactive;
		present_pages[1] = active;
	}

	return inactive_ratio;
}

int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
{
	unsigned long active;
	unsigned long inactive;
	unsigned long present_pages[2];
	unsigned long inactive_ratio;

	inactive_ratio = calc_inactive_ratio(memcg, present_pages);

	inactive = present_pages[0];
	active = present_pages[1];

	if (inactive * inactive_ratio < active)
1038 1039 1040 1041 1042
		return 1;

	return 0;
}

1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
{
	unsigned long active;
	unsigned long inactive;

	inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
	active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);

	return (active > inactive);
}

1054 1055 1056 1057
unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
				       struct zone *zone,
				       enum lru_list lru)
{
1058
	int nid = zone_to_nid(zone);
1059 1060 1061 1062 1063 1064
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

	return MEM_CGROUP_ZSTAT(mz, lru);
}

K
KOSAKI Motohiro 已提交
1065 1066 1067
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
						      struct zone *zone)
{
1068
	int nid = zone_to_nid(zone);
K
KOSAKI Motohiro 已提交
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

	return &mz->reclaim_stat;
}

struct zone_reclaim_stat *
mem_cgroup_get_reclaim_stat_from_page(struct page *page)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;

	if (mem_cgroup_disabled())
		return NULL;

	pc = lookup_page_cgroup(page);
1085 1086
	if (!PageCgroupUsed(pc))
		return NULL;
1087 1088
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
1089
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
K
KOSAKI Motohiro 已提交
1090 1091 1092
	return &mz->reclaim_stat;
}

1093 1094 1095 1096 1097
unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
					struct list_head *dst,
					unsigned long *scanned, int order,
					int mode, struct zone *z,
					struct mem_cgroup *mem_cont,
1098
					int active, int file)
1099 1100 1101 1102 1103 1104
{
	unsigned long nr_taken = 0;
	struct page *page;
	unsigned long scan;
	LIST_HEAD(pc_list);
	struct list_head *src;
1105
	struct page_cgroup *pc, *tmp;
1106
	int nid = zone_to_nid(z);
1107 1108
	int zid = zone_idx(z);
	struct mem_cgroup_per_zone *mz;
1109
	int lru = LRU_FILE * file + active;
1110
	int ret;
1111

1112
	BUG_ON(!mem_cont);
1113
	mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
1114
	src = &mz->lists[lru];
1115

1116 1117
	scan = 0;
	list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
H
Hugh Dickins 已提交
1118
		if (scan >= nr_to_scan)
1119
			break;
K
KAMEZAWA Hiroyuki 已提交
1120

1121 1122
		if (unlikely(!PageCgroupUsed(pc)))
			continue;
1123

1124
		page = lookup_cgroup_page(pc);
1125

H
Hugh Dickins 已提交
1126
		if (unlikely(!PageLRU(page)))
1127 1128
			continue;

H
Hugh Dickins 已提交
1129
		scan++;
1130 1131 1132
		ret = __isolate_lru_page(page, mode, file);
		switch (ret) {
		case 0:
1133
			list_move(&page->lru, dst);
1134
			mem_cgroup_del_lru(page);
1135
			nr_taken += hpage_nr_pages(page);
1136 1137 1138 1139 1140 1141 1142
			break;
		case -EBUSY:
			/* we don't affect global LRU but rotate in our LRU */
			mem_cgroup_rotate_lru_list(page, page_lru(page));
			break;
		default:
			break;
1143 1144 1145 1146
		}
	}

	*scanned = scan;
1147 1148 1149 1150

	trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
				      0, 0, 0, mode);

1151 1152 1153
	return nr_taken;
}

1154 1155 1156
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

1157
/**
1158 1159
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
 * @mem: the memory cgroup
1160
 *
1161
 * Returns the maximum amount of memory @mem can be charged with, in
1162
 * pages.
1163
 */
1164
static unsigned long mem_cgroup_margin(struct mem_cgroup *mem)
1165
{
1166 1167 1168 1169 1170
	unsigned long long margin;

	margin = res_counter_margin(&mem->res);
	if (do_swap_account)
		margin = min(margin, res_counter_margin(&mem->memsw));
1171
	return margin >> PAGE_SHIFT;
1172 1173
}

K
KOSAKI Motohiro 已提交
1174 1175 1176 1177 1178 1179 1180 1181
static unsigned int get_swappiness(struct mem_cgroup *memcg)
{
	struct cgroup *cgrp = memcg->css.cgroup;

	/* root ? */
	if (cgrp->parent == NULL)
		return vm_swappiness;

1182
	return memcg->swappiness;
K
KOSAKI Motohiro 已提交
1183 1184
}

1185 1186 1187
static void mem_cgroup_start_move(struct mem_cgroup *mem)
{
	int cpu;
1188 1189 1190 1191

	get_online_cpus();
	spin_lock(&mem->pcp_counter_lock);
	for_each_online_cpu(cpu)
1192
		per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
1193 1194 1195
	mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
	spin_unlock(&mem->pcp_counter_lock);
	put_online_cpus();
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205

	synchronize_rcu();
}

static void mem_cgroup_end_move(struct mem_cgroup *mem)
{
	int cpu;

	if (!mem)
		return;
1206 1207 1208
	get_online_cpus();
	spin_lock(&mem->pcp_counter_lock);
	for_each_online_cpu(cpu)
1209
		per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
1210 1211 1212
	mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
	spin_unlock(&mem->pcp_counter_lock);
	put_online_cpus();
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
}
/*
 * 2 routines for checking "mem" is under move_account() or not.
 *
 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
 *			  for avoiding race in accounting. If true,
 *			  pc->mem_cgroup may be overwritten.
 *
 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
 *			  under hierarchy of moving cgroups. This is for
 *			  waiting at hith-memory prressure caused by "move".
 */

static bool mem_cgroup_stealed(struct mem_cgroup *mem)
{
	VM_BUG_ON(!rcu_read_lock_held());
	return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
}
1231 1232 1233

static bool mem_cgroup_under_move(struct mem_cgroup *mem)
{
1234 1235
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1236
	bool ret = false;
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
	if (from == mem || to == mem
	    || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
	    || (mem->use_hierarchy && css_is_ancestor(&to->css,	&mem->css)))
		ret = true;
unlock:
	spin_unlock(&mc.lock);
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
	return ret;
}

static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
{
	if (mc.moving_task && current != mc.moving_task) {
		if (mem_cgroup_under_move(mem)) {
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1271
/**
1272
 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;

1291
	if (!memcg || !p)
1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
		return;


	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	printk(KERN_INFO "Task in %s killed", memcg_name);

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
	printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
done:

	printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
	printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
		"failcnt %llu\n",
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
}

1338 1339 1340 1341 1342 1343 1344
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
static int mem_cgroup_count_children(struct mem_cgroup *mem)
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1345 1346 1347 1348
	struct mem_cgroup *iter;

	for_each_mem_cgroup_tree(iter, mem)
		num++;
1349 1350 1351
	return num;
}

D
David Rientjes 已提交
1352 1353 1354 1355 1356 1357 1358 1359
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
{
	u64 limit;
	u64 memsw;

1360 1361 1362
	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	limit += total_swap_pages << PAGE_SHIFT;

D
David Rientjes 已提交
1363 1364 1365 1366 1367 1368 1369 1370
	memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	/*
	 * If memsw is finite and limits the amount of swap space available
	 * to this memcg, return that limit.
	 */
	return min(limit, memsw);
}

1371
/*
K
KAMEZAWA Hiroyuki 已提交
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
 * Visit the first child (need not be the first child as per the ordering
 * of the cgroup list, since we track last_scanned_child) of @mem and use
 * that to reclaim free pages from.
 */
static struct mem_cgroup *
mem_cgroup_select_victim(struct mem_cgroup *root_mem)
{
	struct mem_cgroup *ret = NULL;
	struct cgroup_subsys_state *css;
	int nextid, found;

	if (!root_mem->use_hierarchy) {
		css_get(&root_mem->css);
		ret = root_mem;
	}

	while (!ret) {
		rcu_read_lock();
		nextid = root_mem->last_scanned_child + 1;
		css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
				   &found);
		if (css && css_tryget(css))
			ret = container_of(css, struct mem_cgroup, css);

		rcu_read_unlock();
		/* Updates scanning parameter */
		if (!css) {
			/* this means start scan from ID:1 */
			root_mem->last_scanned_child = 0;
		} else
			root_mem->last_scanned_child = found;
	}

	return ret;
}

/*
 * Scan the hierarchy if needed to reclaim memory. We remember the last child
 * we reclaimed from, so that we don't end up penalizing one child extensively
 * based on its position in the children list.
1412 1413
 *
 * root_mem is the original ancestor that we've been reclaim from.
K
KAMEZAWA Hiroyuki 已提交
1414 1415 1416
 *
 * We give up and return to the caller when we visit root_mem twice.
 * (other groups can be removed while we're walking....)
1417 1418
 *
 * If shrink==true, for avoiding to free too much, this returns immedieately.
1419 1420
 */
static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
1421
						struct zone *zone,
1422 1423
						gfp_t gfp_mask,
						unsigned long reclaim_options)
1424
{
K
KAMEZAWA Hiroyuki 已提交
1425 1426 1427
	struct mem_cgroup *victim;
	int ret, total = 0;
	int loop = 0;
1428 1429
	bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
	bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1430
	bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1431 1432 1433
	unsigned long excess;

	excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT;
K
KAMEZAWA Hiroyuki 已提交
1434

1435 1436 1437 1438
	/* If memsw_is_minimum==1, swap-out is of-no-use. */
	if (root_mem->memsw_is_minimum)
		noswap = true;

1439
	while (1) {
K
KAMEZAWA Hiroyuki 已提交
1440
		victim = mem_cgroup_select_victim(root_mem);
1441
		if (victim == root_mem) {
K
KAMEZAWA Hiroyuki 已提交
1442
			loop++;
1443 1444
			if (loop >= 1)
				drain_all_stock_async();
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!check_soft || !total) {
					css_put(&victim->css);
					break;
				}
				/*
				 * We want to do more targetted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
					css_put(&victim->css);
					break;
				}
			}
		}
1468
		if (!mem_cgroup_local_usage(victim)) {
K
KAMEZAWA Hiroyuki 已提交
1469 1470
			/* this cgroup's local usage == 0 */
			css_put(&victim->css);
1471 1472
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
1473
		/* we use swappiness of local cgroup */
1474 1475
		if (check_soft)
			ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
1476
				noswap, get_swappiness(victim), zone);
1477 1478 1479
		else
			ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
						noswap, get_swappiness(victim));
K
KAMEZAWA Hiroyuki 已提交
1480
		css_put(&victim->css);
1481 1482 1483 1484 1485 1486 1487
		/*
		 * At shrinking usage, we can't check we should stop here or
		 * reclaim more. It's depends on callers. last_scanned_child
		 * will work enough for keeping fairness under tree.
		 */
		if (shrink)
			return ret;
K
KAMEZAWA Hiroyuki 已提交
1488
		total += ret;
1489
		if (check_soft) {
1490
			if (!res_counter_soft_limit_excess(&root_mem->res))
1491
				return total;
1492
		} else if (mem_cgroup_margin(root_mem))
K
KAMEZAWA Hiroyuki 已提交
1493
			return 1 + total;
1494
	}
K
KAMEZAWA Hiroyuki 已提交
1495
	return total;
1496 1497
}

K
KAMEZAWA Hiroyuki 已提交
1498 1499 1500 1501 1502 1503
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
 */
static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
{
K
KAMEZAWA Hiroyuki 已提交
1504 1505
	int x, lock_count = 0;
	struct mem_cgroup *iter;
1506

K
KAMEZAWA Hiroyuki 已提交
1507 1508 1509 1510
	for_each_mem_cgroup_tree(iter, mem) {
		x = atomic_inc_return(&iter->oom_lock);
		lock_count = max(x, lock_count);
	}
K
KAMEZAWA Hiroyuki 已提交
1511 1512 1513 1514

	if (lock_count == 1)
		return true;
	return false;
1515
}
1516

K
KAMEZAWA Hiroyuki 已提交
1517
static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
1518
{
K
KAMEZAWA Hiroyuki 已提交
1519 1520
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1521 1522 1523 1524 1525
	/*
	 * When a new child is created while the hierarchy is under oom,
	 * mem_cgroup_oom_lock() may not be called. We have to use
	 * atomic_add_unless() here.
	 */
K
KAMEZAWA Hiroyuki 已提交
1526 1527
	for_each_mem_cgroup_tree(iter, mem)
		atomic_add_unless(&iter->oom_lock, -1, 0);
1528 1529 1530
	return 0;
}

K
KAMEZAWA Hiroyuki 已提交
1531 1532 1533 1534

static DEFINE_MUTEX(memcg_oom_mutex);
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
struct oom_wait_info {
	struct mem_cgroup *mem;
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
	struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);

	if (oom_wait_info->mem == wake_mem)
		goto wakeup;
	/* if no hierarchy, no match */
	if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
		return 0;
	/*
	 * Both of oom_wait_info->mem and wake_mem are stable under us.
	 * Then we can use css_is_ancestor without taking care of RCU.
	 */
	if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
	    !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
		return 0;

wakeup:
	return autoremove_wake_function(wait, mode, sync, arg);
}

static void memcg_wakeup_oom(struct mem_cgroup *mem)
{
	/* for filtering, pass "mem" as argument. */
	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
}

1571 1572
static void memcg_oom_recover(struct mem_cgroup *mem)
{
1573
	if (mem && atomic_read(&mem->oom_lock))
1574 1575 1576
		memcg_wakeup_oom(mem);
}

K
KAMEZAWA Hiroyuki 已提交
1577 1578 1579 1580
/*
 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
 */
bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1581
{
K
KAMEZAWA Hiroyuki 已提交
1582
	struct oom_wait_info owait;
1583
	bool locked, need_to_kill;
K
KAMEZAWA Hiroyuki 已提交
1584

K
KAMEZAWA Hiroyuki 已提交
1585 1586 1587 1588 1589
	owait.mem = mem;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
1590
	need_to_kill = true;
K
KAMEZAWA Hiroyuki 已提交
1591 1592 1593 1594 1595 1596 1597 1598
	/* At first, try to OOM lock hierarchy under mem.*/
	mutex_lock(&memcg_oom_mutex);
	locked = mem_cgroup_oom_lock(mem);
	/*
	 * Even if signal_pending(), we can't quit charge() loop without
	 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
	 * under OOM is always welcomed, use TASK_KILLABLE here.
	 */
1599 1600 1601 1602
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
	if (!locked || mem->oom_kill_disable)
		need_to_kill = false;
	if (locked)
K
KAMEZAWA Hiroyuki 已提交
1603
		mem_cgroup_oom_notify(mem);
K
KAMEZAWA Hiroyuki 已提交
1604 1605
	mutex_unlock(&memcg_oom_mutex);

1606 1607
	if (need_to_kill) {
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
1608
		mem_cgroup_out_of_memory(mem, mask);
1609
	} else {
K
KAMEZAWA Hiroyuki 已提交
1610
		schedule();
K
KAMEZAWA Hiroyuki 已提交
1611
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
1612 1613 1614
	}
	mutex_lock(&memcg_oom_mutex);
	mem_cgroup_oom_unlock(mem);
K
KAMEZAWA Hiroyuki 已提交
1615
	memcg_wakeup_oom(mem);
K
KAMEZAWA Hiroyuki 已提交
1616 1617 1618 1619 1620 1621 1622
	mutex_unlock(&memcg_oom_mutex);

	if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
		return false;
	/* Give chance to dying process */
	schedule_timeout(1);
	return true;
1623 1624
}

1625 1626 1627
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
 *
 * Notes: Race condition
 *
 * We usually use page_cgroup_lock() for accessing page_cgroup member but
 * it tends to be costly. But considering some conditions, we doesn't need
 * to do so _always_.
 *
 * Considering "charge", lock_page_cgroup() is not required because all
 * file-stat operations happen after a page is attached to radix-tree. There
 * are no race with "charge".
 *
 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
 * if there are race with "uncharge". Statistics itself is properly handled
 * by flags.
 *
 * Considering "move", this is an only case we see a race. To make the race
 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
 * possibility of race condition. If there is, we take a lock.
1647
 */
1648

1649 1650
void mem_cgroup_update_page_stat(struct page *page,
				 enum mem_cgroup_page_stat_item idx, int val)
1651 1652
{
	struct mem_cgroup *mem;
1653 1654
	struct page_cgroup *pc = lookup_page_cgroup(page);
	bool need_unlock = false;
1655
	unsigned long uninitialized_var(flags);
1656 1657 1658 1659

	if (unlikely(!pc))
		return;

1660
	rcu_read_lock();
1661
	mem = pc->mem_cgroup;
1662 1663 1664
	if (unlikely(!mem || !PageCgroupUsed(pc)))
		goto out;
	/* pc->mem_cgroup is unstable ? */
1665
	if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
1666
		/* take a lock against to access pc->mem_cgroup */
1667
		move_lock_page_cgroup(pc, &flags);
1668 1669 1670 1671 1672
		need_unlock = true;
		mem = pc->mem_cgroup;
		if (!mem || !PageCgroupUsed(pc))
			goto out;
	}
1673 1674

	switch (idx) {
1675
	case MEMCG_NR_FILE_MAPPED:
1676 1677 1678
		if (val > 0)
			SetPageCgroupFileMapped(pc);
		else if (!page_mapped(page))
1679
			ClearPageCgroupFileMapped(pc);
1680
		idx = MEM_CGROUP_STAT_FILE_MAPPED;
1681 1682 1683
		break;
	default:
		BUG();
1684
	}
1685

1686 1687
	this_cpu_add(mem->stat->count[idx], val);

1688 1689
out:
	if (unlikely(need_unlock))
1690
		move_unlock_page_cgroup(pc, &flags);
1691 1692
	rcu_read_unlock();
	return;
1693
}
1694
EXPORT_SYMBOL(mem_cgroup_update_page_stat);
1695

1696 1697 1698 1699
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
1700
#define CHARGE_BATCH	32U
1701 1702
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
1703
	unsigned int nr_pages;
1704 1705 1706 1707 1708 1709
	struct work_struct work;
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
static atomic_t memcg_drain_count;

/*
1710
 * Try to consume stocked charge on this cpu. If success, one page is consumed
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
 * from local stock and true is returned. If the stock is 0 or charges from a
 * cgroup which is not current target, returns false. This stock will be
 * refilled.
 */
static bool consume_stock(struct mem_cgroup *mem)
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

	stock = &get_cpu_var(memcg_stock);
1721 1722
	if (mem == stock->cached && stock->nr_pages)
		stock->nr_pages--;
1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

1736 1737 1738 1739
	if (stock->nr_pages) {
		unsigned long bytes = stock->nr_pages * PAGE_SIZE;

		res_counter_uncharge(&old->res, bytes);
1740
		if (do_swap_account)
1741 1742
			res_counter_uncharge(&old->memsw, bytes);
		stock->nr_pages = 0;
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
}

/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
1759
 * This will be consumed by consume_stock() function, later.
1760
 */
1761
static void refill_stock(struct mem_cgroup *mem, unsigned int nr_pages)
1762 1763 1764 1765 1766 1767 1768
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

	if (stock->cached != mem) { /* reset if necessary */
		drain_stock(stock);
		stock->cached = mem;
	}
1769
	stock->nr_pages += nr_pages;
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
	put_cpu_var(memcg_stock);
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
static void drain_all_stock_async(void)
{
	int cpu;
	/* This function is for scheduling "drain" in asynchronous way.
	 * The result of "drain" is not directly handled by callers. Then,
	 * if someone is calling drain, we don't have to call drain more.
	 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
	 * there is a race. We just do loose check here.
	 */
	if (atomic_read(&memcg_drain_count))
		return;
	/* Notify other cpus that system-wide "drain" is running */
	atomic_inc(&memcg_drain_count);
	get_online_cpus();
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
		schedule_work_on(cpu, &stock->work);
	}
 	put_online_cpus();
	atomic_dec(&memcg_drain_count);
	/* We don't wait for flush_work */
}

/* This is a synchronous drain interface. */
static void drain_all_stock_sync(void)
{
	/* called when force_empty is called */
	atomic_inc(&memcg_drain_count);
	schedule_on_each_cpu(drain_local_stock);
	atomic_dec(&memcg_drain_count);
}

1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
/*
 * This function drains percpu counter value from DEAD cpu and
 * move it to local cpu. Note that this function can be preempted.
 */
static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
{
	int i;

	spin_lock(&mem->pcp_counter_lock);
	for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
1821
		long x = per_cpu(mem->stat->count[i], cpu);
1822 1823 1824 1825

		per_cpu(mem->stat->count[i], cpu) = 0;
		mem->nocpu_base.count[i] += x;
	}
1826 1827 1828 1829 1830 1831
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long x = per_cpu(mem->stat->events[i], cpu);

		per_cpu(mem->stat->events[i], cpu) = 0;
		mem->nocpu_base.events[i] += x;
	}
1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842
	/* need to clear ON_MOVE value, works as a kind of lock. */
	per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
	spin_unlock(&mem->pcp_counter_lock);
}

static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
{
	int idx = MEM_CGROUP_ON_MOVE;

	spin_lock(&mem->pcp_counter_lock);
	per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
1843 1844 1845 1846
	spin_unlock(&mem->pcp_counter_lock);
}

static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
1847 1848 1849 1850 1851
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;
1852
	struct mem_cgroup *iter;
1853

1854 1855 1856 1857 1858 1859
	if ((action == CPU_ONLINE)) {
		for_each_mem_cgroup_all(iter)
			synchronize_mem_cgroup_on_move(iter, cpu);
		return NOTIFY_OK;
	}

1860
	if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
1861
		return NOTIFY_OK;
1862 1863 1864 1865

	for_each_mem_cgroup_all(iter)
		mem_cgroup_drain_pcp_counter(iter, cpu);

1866 1867 1868 1869 1870
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

1871 1872 1873 1874 1875 1876 1877 1878 1879 1880

/* See __mem_cgroup_try_charge() for details */
enum {
	CHARGE_OK,		/* success */
	CHARGE_RETRY,		/* need to retry but retry is not bad */
	CHARGE_NOMEM,		/* we can't do more. return -ENOMEM */
	CHARGE_WOULDBLOCK,	/* GFP_WAIT wasn't set and no enough res. */
	CHARGE_OOM_DIE,		/* the current is killed because of OOM */
};

1881 1882
static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
				unsigned int nr_pages, bool oom_check)
1883
{
1884
	unsigned long csize = nr_pages * PAGE_SIZE;
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
	struct mem_cgroup *mem_over_limit;
	struct res_counter *fail_res;
	unsigned long flags = 0;
	int ret;

	ret = res_counter_charge(&mem->res, csize, &fail_res);

	if (likely(!ret)) {
		if (!do_swap_account)
			return CHARGE_OK;
		ret = res_counter_charge(&mem->memsw, csize, &fail_res);
		if (likely(!ret))
			return CHARGE_OK;

1899
		res_counter_uncharge(&mem->res, csize);
1900 1901 1902 1903
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
		flags |= MEM_CGROUP_RECLAIM_NOSWAP;
	} else
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1904
	/*
1905 1906
	 * nr_pages can be either a huge page (HPAGE_PMD_NR), a batch
	 * of regular pages (CHARGE_BATCH), or a single regular page (1).
1907 1908 1909 1910
	 *
	 * Never reclaim on behalf of optional batching, retry with a
	 * single page instead.
	 */
1911
	if (nr_pages == CHARGE_BATCH)
1912 1913 1914 1915 1916 1917
		return CHARGE_RETRY;

	if (!(gfp_mask & __GFP_WAIT))
		return CHARGE_WOULDBLOCK;

	ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1918
					      gfp_mask, flags);
1919
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1920
		return CHARGE_RETRY;
1921
	/*
1922 1923 1924 1925 1926 1927 1928
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
1929
	 */
1930
	if (nr_pages == 1 && ret)
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
		return CHARGE_RETRY;

	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		return CHARGE_RETRY;

	/* If we don't need to call oom-killer at el, return immediately */
	if (!oom_check)
		return CHARGE_NOMEM;
	/* check OOM */
	if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
		return CHARGE_OOM_DIE;

	return CHARGE_RETRY;
}

1950 1951 1952
/*
 * Unlike exported interface, "oom" parameter is added. if oom==true,
 * oom-killer can be invoked.
1953
 */
1954
static int __mem_cgroup_try_charge(struct mm_struct *mm,
A
Andrea Arcangeli 已提交
1955
				   gfp_t gfp_mask,
1956 1957 1958
				   unsigned int nr_pages,
				   struct mem_cgroup **memcg,
				   bool oom)
1959
{
1960
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
1961 1962 1963
	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
	struct mem_cgroup *mem = NULL;
	int ret;
1964

K
KAMEZAWA Hiroyuki 已提交
1965 1966 1967 1968 1969 1970 1971 1972
	/*
	 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
	 * in system level. So, allow to go ahead dying process in addition to
	 * MEMDIE process.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)
		     || fatal_signal_pending(current)))
		goto bypass;
1973

1974
	/*
1975 1976
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
1977 1978 1979
	 * thread group leader migrates. It's possible that mm is not
	 * set, if so charge the init_mm (happens for pagecache usage).
	 */
K
KAMEZAWA Hiroyuki 已提交
1980 1981 1982 1983
	if (!*memcg && !mm)
		goto bypass;
again:
	if (*memcg) { /* css should be a valid one */
1984
		mem = *memcg;
K
KAMEZAWA Hiroyuki 已提交
1985 1986 1987
		VM_BUG_ON(css_is_removed(&mem->css));
		if (mem_cgroup_is_root(mem))
			goto done;
1988
		if (nr_pages == 1 && consume_stock(mem))
K
KAMEZAWA Hiroyuki 已提交
1989
			goto done;
1990 1991
		css_get(&mem->css);
	} else {
K
KAMEZAWA Hiroyuki 已提交
1992
		struct task_struct *p;
1993

K
KAMEZAWA Hiroyuki 已提交
1994 1995 1996
		rcu_read_lock();
		p = rcu_dereference(mm->owner);
		/*
1997 1998 1999 2000 2001 2002 2003 2004
		 * Because we don't have task_lock(), "p" can exit.
		 * In that case, "mem" can point to root or p can be NULL with
		 * race with swapoff. Then, we have small risk of mis-accouning.
		 * But such kind of mis-account by race always happens because
		 * we don't have cgroup_mutex(). It's overkill and we allo that
		 * small race, here.
		 * (*) swapoff at el will charge against mm-struct not against
		 * task-struct. So, mm->owner can be NULL.
K
KAMEZAWA Hiroyuki 已提交
2005 2006
		 */
		mem = mem_cgroup_from_task(p);
2007
		if (!mem || mem_cgroup_is_root(mem)) {
K
KAMEZAWA Hiroyuki 已提交
2008 2009 2010
			rcu_read_unlock();
			goto done;
		}
2011
		if (nr_pages == 1 && consume_stock(mem)) {
K
KAMEZAWA Hiroyuki 已提交
2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
			/*
			 * It seems dagerous to access memcg without css_get().
			 * But considering how consume_stok works, it's not
			 * necessary. If consume_stock success, some charges
			 * from this memcg are cached on this cpu. So, we
			 * don't need to call css_get()/css_tryget() before
			 * calling consume_stock().
			 */
			rcu_read_unlock();
			goto done;
		}
		/* after here, we may be blocked. we need to get refcnt */
		if (!css_tryget(&mem->css)) {
			rcu_read_unlock();
			goto again;
		}
		rcu_read_unlock();
	}
2030

2031 2032
	do {
		bool oom_check;
2033

2034
		/* If killed, bypass charge */
K
KAMEZAWA Hiroyuki 已提交
2035 2036
		if (fatal_signal_pending(current)) {
			css_put(&mem->css);
2037
			goto bypass;
K
KAMEZAWA Hiroyuki 已提交
2038
		}
2039

2040 2041 2042 2043
		oom_check = false;
		if (oom && !nr_oom_retries) {
			oom_check = true;
			nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2044
		}
2045

2046
		ret = mem_cgroup_do_charge(mem, gfp_mask, batch, oom_check);
2047 2048 2049 2050
		switch (ret) {
		case CHARGE_OK:
			break;
		case CHARGE_RETRY: /* not in OOM situation but retry */
2051
			batch = nr_pages;
K
KAMEZAWA Hiroyuki 已提交
2052 2053 2054
			css_put(&mem->css);
			mem = NULL;
			goto again;
2055
		case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
K
KAMEZAWA Hiroyuki 已提交
2056
			css_put(&mem->css);
2057 2058
			goto nomem;
		case CHARGE_NOMEM: /* OOM routine works */
K
KAMEZAWA Hiroyuki 已提交
2059 2060
			if (!oom) {
				css_put(&mem->css);
K
KAMEZAWA Hiroyuki 已提交
2061
				goto nomem;
K
KAMEZAWA Hiroyuki 已提交
2062
			}
2063 2064 2065 2066
			/* If oom, we never return -ENOMEM */
			nr_oom_retries--;
			break;
		case CHARGE_OOM_DIE: /* Killed by OOM Killer */
K
KAMEZAWA Hiroyuki 已提交
2067
			css_put(&mem->css);
K
KAMEZAWA Hiroyuki 已提交
2068
			goto bypass;
2069
		}
2070 2071
	} while (ret != CHARGE_OK);

2072 2073
	if (batch > nr_pages)
		refill_stock(mem, batch - nr_pages);
K
KAMEZAWA Hiroyuki 已提交
2074
	css_put(&mem->css);
2075
done:
K
KAMEZAWA Hiroyuki 已提交
2076
	*memcg = mem;
2077 2078
	return 0;
nomem:
K
KAMEZAWA Hiroyuki 已提交
2079
	*memcg = NULL;
2080
	return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2081 2082 2083
bypass:
	*memcg = NULL;
	return 0;
2084
}
2085

2086 2087 2088 2089 2090
/*
 * Somemtimes we have to undo a charge we got by try_charge().
 * This function is for that and do uncharge, put css's refcnt.
 * gotten by try_charge().
 */
2091
static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2092
				       unsigned int nr_pages)
2093 2094
{
	if (!mem_cgroup_is_root(mem)) {
2095 2096 2097
		unsigned long bytes = nr_pages * PAGE_SIZE;

		res_counter_uncharge(&mem->res, bytes);
2098
		if (do_swap_account)
2099
			res_counter_uncharge(&mem->memsw, bytes);
2100
	}
2101 2102
}

2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
/*
 * A helper function to get mem_cgroup from ID. must be called under
 * rcu_read_lock(). The caller must check css_is_removed() or some if
 * it's concern. (dropping refcnt from swap can be called against removed
 * memcg.)
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
	return container_of(css, struct mem_cgroup, css);
}

2122
struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2123
{
2124
	struct mem_cgroup *mem = NULL;
2125
	struct page_cgroup *pc;
2126
	unsigned short id;
2127 2128
	swp_entry_t ent;

2129 2130 2131
	VM_BUG_ON(!PageLocked(page));

	pc = lookup_page_cgroup(page);
2132
	lock_page_cgroup(pc);
2133
	if (PageCgroupUsed(pc)) {
2134
		mem = pc->mem_cgroup;
2135 2136
		if (mem && !css_tryget(&mem->css))
			mem = NULL;
2137
	} else if (PageSwapCache(page)) {
2138
		ent.val = page_private(page);
2139 2140 2141 2142 2143 2144
		id = lookup_swap_cgroup(ent);
		rcu_read_lock();
		mem = mem_cgroup_lookup(id);
		if (mem && !css_tryget(&mem->css))
			mem = NULL;
		rcu_read_unlock();
2145
	}
2146
	unlock_page_cgroup(pc);
2147 2148 2149
	return mem;
}

2150
static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
2151
				       struct page *page,
2152
				       unsigned int nr_pages,
2153
				       struct page_cgroup *pc,
2154
				       enum charge_type ctype)
2155
{
2156 2157 2158
	lock_page_cgroup(pc);
	if (unlikely(PageCgroupUsed(pc))) {
		unlock_page_cgroup(pc);
2159
		__mem_cgroup_cancel_charge(mem, nr_pages);
2160 2161 2162 2163 2164 2165
		return;
	}
	/*
	 * we don't need page_cgroup_lock about tail pages, becase they are not
	 * accessed by any other context at this point.
	 */
2166
	pc->mem_cgroup = mem;
2167 2168 2169 2170 2171 2172 2173
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
2174
	smp_wmb();
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187
	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_CACHE:
	case MEM_CGROUP_CHARGE_TYPE_SHMEM:
		SetPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
		ClearPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	default:
		break;
	}
2188

2189
	mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
2190
	unlock_page_cgroup(pc);
2191 2192 2193 2194 2195
	/*
	 * "charge_statistics" updated event counter. Then, check it.
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
	 */
2196
	memcg_check_events(mem, page);
2197
}
2198

2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
			(1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
/*
 * Because tail pages are not marked as "used", set it. We're under
 * zone->lru_lock, 'splitting on pmd' and compund_lock.
 */
void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
{
	struct page_cgroup *head_pc = lookup_page_cgroup(head);
	struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
	unsigned long flags;

2213 2214
	if (mem_cgroup_disabled())
		return;
2215
	/*
2216
	 * We have no races with charge/uncharge but will have races with
2217 2218 2219 2220 2221 2222
	 * page state accounting.
	 */
	move_lock_page_cgroup(head_pc, &flags);

	tail_pc->mem_cgroup = head_pc->mem_cgroup;
	smp_wmb(); /* see __commit_charge() */
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232
	if (PageCgroupAcctLRU(head_pc)) {
		enum lru_list lru;
		struct mem_cgroup_per_zone *mz;

		/*
		 * LRU flags cannot be copied because we need to add tail
		 *.page to LRU by generic call and our hook will be called.
		 * We hold lru_lock, then, reduce counter directly.
		 */
		lru = page_lru(head);
2233
		mz = page_cgroup_zoneinfo(head_pc->mem_cgroup, head);
2234 2235
		MEM_CGROUP_ZSTAT(mz, lru) -= 1;
	}
2236 2237 2238 2239 2240
	tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
	move_unlock_page_cgroup(head_pc, &flags);
}
#endif

2241
/**
2242
 * mem_cgroup_move_account - move account of the page
2243
 * @page: the page
2244
 * @nr_pages: number of regular pages (>1 for huge pages)
2245 2246 2247
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
2248
 * @uncharge: whether we should call uncharge and css_put against @from.
2249 2250
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
2251
 * - page is not on LRU (isolate_page() is useful.)
2252
 * - compound_lock is held when nr_pages > 1
2253
 *
2254 2255 2256 2257
 * This function doesn't do "charge" nor css_get to new cgroup. It should be
 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
 * true, this function does "uncharge" from old cgroup, but it doesn't if
 * @uncharge is false, so a caller should do "uncharge".
2258
 */
2259 2260 2261 2262 2263 2264
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct page_cgroup *pc,
				   struct mem_cgroup *from,
				   struct mem_cgroup *to,
				   bool uncharge)
2265
{
2266 2267
	unsigned long flags;
	int ret;
2268

2269
	VM_BUG_ON(from == to);
2270
	VM_BUG_ON(PageLRU(page));
2271 2272 2273 2274 2275 2276 2277
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
2278
	if (nr_pages > 1 && !PageTransHuge(page))
2279 2280 2281 2282 2283 2284 2285 2286 2287
		goto out;

	lock_page_cgroup(pc);

	ret = -EINVAL;
	if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
		goto unlock;

	move_lock_page_cgroup(pc, &flags);
2288

2289
	if (PageCgroupFileMapped(pc)) {
2290 2291 2292 2293 2294
		/* Update mapped_file data for mem_cgroup */
		preempt_disable();
		__this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		__this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		preempt_enable();
2295
	}
2296
	mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
2297 2298
	if (uncharge)
		/* This is not "cancel", but cancel_charge does all we need. */
2299
		__mem_cgroup_cancel_charge(from, nr_pages);
2300

2301
	/* caller should have done css_get */
K
KAMEZAWA Hiroyuki 已提交
2302
	pc->mem_cgroup = to;
2303
	mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages);
2304 2305 2306
	/*
	 * We charges against "to" which may not have any tasks. Then, "to"
	 * can be under rmdir(). But in current implementation, caller of
2307 2308 2309
	 * this function is just force_empty() and move charge, so it's
	 * garanteed that "to" is never removed. So, we don't check rmdir
	 * status here.
2310
	 */
2311 2312 2313
	move_unlock_page_cgroup(pc, &flags);
	ret = 0;
unlock:
2314
	unlock_page_cgroup(pc);
2315 2316 2317
	/*
	 * check events
	 */
2318 2319
	memcg_check_events(to, page);
	memcg_check_events(from, page);
2320
out:
2321 2322 2323 2324 2325 2326 2327
	return ret;
}

/*
 * move charges to its parent.
 */

2328 2329
static int mem_cgroup_move_parent(struct page *page,
				  struct page_cgroup *pc,
2330 2331 2332 2333 2334 2335
				  struct mem_cgroup *child,
				  gfp_t gfp_mask)
{
	struct cgroup *cg = child->css.cgroup;
	struct cgroup *pcg = cg->parent;
	struct mem_cgroup *parent;
2336
	unsigned int nr_pages;
2337
	unsigned long uninitialized_var(flags);
2338 2339 2340 2341 2342 2343
	int ret;

	/* Is ROOT ? */
	if (!pcg)
		return -EINVAL;

2344 2345 2346 2347 2348
	ret = -EBUSY;
	if (!get_page_unless_zero(page))
		goto out;
	if (isolate_lru_page(page))
		goto put;
2349

2350
	nr_pages = hpage_nr_pages(page);
K
KAMEZAWA Hiroyuki 已提交
2351

2352
	parent = mem_cgroup_from_cont(pcg);
2353
	ret = __mem_cgroup_try_charge(NULL, gfp_mask, nr_pages, &parent, false);
2354
	if (ret || !parent)
2355
		goto put_back;
2356

2357
	if (nr_pages > 1)
2358 2359
		flags = compound_lock_irqsave(page);

2360
	ret = mem_cgroup_move_account(page, nr_pages, pc, child, parent, true);
2361
	if (ret)
2362
		__mem_cgroup_cancel_charge(parent, nr_pages);
2363

2364
	if (nr_pages > 1)
2365
		compound_unlock_irqrestore(page, flags);
2366
put_back:
K
KAMEZAWA Hiroyuki 已提交
2367
	putback_lru_page(page);
2368
put:
2369
	put_page(page);
2370
out:
2371 2372 2373
	return ret;
}

2374 2375 2376 2377 2378 2379 2380
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2381
				gfp_t gfp_mask, enum charge_type ctype)
2382
{
2383
	struct mem_cgroup *mem = NULL;
2384
	unsigned int nr_pages = 1;
2385
	struct page_cgroup *pc;
2386
	bool oom = true;
2387
	int ret;
A
Andrea Arcangeli 已提交
2388

A
Andrea Arcangeli 已提交
2389
	if (PageTransHuge(page)) {
2390
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
2391
		VM_BUG_ON(!PageTransHuge(page));
2392 2393 2394 2395 2396
		/*
		 * Never OOM-kill a process for a huge page.  The
		 * fault handler will fall back to regular pages.
		 */
		oom = false;
A
Andrea Arcangeli 已提交
2397
	}
2398 2399

	pc = lookup_page_cgroup(page);
2400
	BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */
2401

2402
	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &mem, oom);
2403
	if (ret || !mem)
2404 2405
		return ret;

2406
	__mem_cgroup_commit_charge(mem, page, nr_pages, pc, ctype);
2407 2408 2409
	return 0;
}

2410 2411
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
2412
{
2413
	if (mem_cgroup_disabled())
2414
		return 0;
2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425
	/*
	 * If already mapped, we don't have to account.
	 * If page cache, page->mapping has address_space.
	 * But page->mapping may have out-of-use anon_vma pointer,
	 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
	 * is NULL.
  	 */
	if (page_mapped(page) || (page->mapping && !PageAnon(page)))
		return 0;
	if (unlikely(!mm))
		mm = &init_mm;
2426
	return mem_cgroup_charge_common(page, mm, gfp_mask,
2427
				MEM_CGROUP_CHARGE_TYPE_MAPPED);
2428 2429
}

D
Daisuke Nishimura 已提交
2430 2431 2432 2433
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype);

2434 2435
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
2436
{
2437 2438
	int ret;

2439
	if (mem_cgroup_disabled())
2440
		return 0;
2441 2442
	if (PageCompound(page))
		return 0;
2443 2444 2445 2446 2447 2448 2449 2450
	/*
	 * Corner case handling. This is called from add_to_page_cache()
	 * in usual. But some FS (shmem) precharges this page before calling it
	 * and call add_to_page_cache() with GFP_NOWAIT.
	 *
	 * For GFP_NOWAIT case, the page may be pre-charged before calling
	 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
	 * charge twice. (It works but has to pay a bit larger cost.)
2451 2452
	 * And when the page is SwapCache, it should take swap information
	 * into account. This is under lock_page() now.
2453 2454 2455 2456
	 */
	if (!(gfp_mask & __GFP_WAIT)) {
		struct page_cgroup *pc;

2457 2458 2459 2460 2461 2462
		pc = lookup_page_cgroup(page);
		if (!pc)
			return 0;
		lock_page_cgroup(pc);
		if (PageCgroupUsed(pc)) {
			unlock_page_cgroup(pc);
2463 2464
			return 0;
		}
2465
		unlock_page_cgroup(pc);
2466 2467
	}

2468
	if (unlikely(!mm))
2469
		mm = &init_mm;
2470

2471 2472
	if (page_is_file_cache(page))
		return mem_cgroup_charge_common(page, mm, gfp_mask,
2473
				MEM_CGROUP_CHARGE_TYPE_CACHE);
2474

D
Daisuke Nishimura 已提交
2475 2476
	/* shmem */
	if (PageSwapCache(page)) {
2477
		struct mem_cgroup *mem;
2478

D
Daisuke Nishimura 已提交
2479 2480 2481 2482 2483 2484
		ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
		if (!ret)
			__mem_cgroup_commit_charge_swapin(page, mem,
					MEM_CGROUP_CHARGE_TYPE_SHMEM);
	} else
		ret = mem_cgroup_charge_common(page, mm, gfp_mask,
2485
					MEM_CGROUP_CHARGE_TYPE_SHMEM);
2486 2487

	return ret;
2488 2489
}

2490 2491 2492
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
2493
 * struct page_cgroup is acquired. This refcnt will be consumed by
2494 2495
 * "commit()" or removed by "cancel()"
 */
2496 2497 2498 2499 2500
int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
				 struct page *page,
				 gfp_t mask, struct mem_cgroup **ptr)
{
	struct mem_cgroup *mem;
2501
	int ret;
2502

2503 2504
	*ptr = NULL;

2505
	if (mem_cgroup_disabled())
2506 2507 2508 2509 2510 2511
		return 0;

	if (!do_swap_account)
		goto charge_cur_mm;
	/*
	 * A racing thread's fault, or swapoff, may have already updated
H
Hugh Dickins 已提交
2512 2513 2514
	 * the pte, and even removed page from swap cache: in those cases
	 * do_swap_page()'s pte_same() test will fail; but there's also a
	 * KSM case which does need to charge the page.
2515 2516
	 */
	if (!PageSwapCache(page))
H
Hugh Dickins 已提交
2517
		goto charge_cur_mm;
2518
	mem = try_get_mem_cgroup_from_page(page);
2519 2520
	if (!mem)
		goto charge_cur_mm;
2521
	*ptr = mem;
2522
	ret = __mem_cgroup_try_charge(NULL, mask, 1, ptr, true);
2523 2524
	css_put(&mem->css);
	return ret;
2525 2526 2527
charge_cur_mm:
	if (unlikely(!mm))
		mm = &init_mm;
2528
	return __mem_cgroup_try_charge(mm, mask, 1, ptr, true);
2529 2530
}

D
Daisuke Nishimura 已提交
2531 2532 2533
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype)
2534 2535 2536
{
	struct page_cgroup *pc;

2537
	if (mem_cgroup_disabled())
2538 2539 2540
		return;
	if (!ptr)
		return;
2541
	cgroup_exclude_rmdir(&ptr->css);
2542
	pc = lookup_page_cgroup(page);
2543
	mem_cgroup_lru_del_before_commit_swapcache(page);
2544
	__mem_cgroup_commit_charge(ptr, page, 1, pc, ctype);
2545
	mem_cgroup_lru_add_after_commit_swapcache(page);
2546 2547 2548
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
2549 2550 2551
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
2552
	 */
2553
	if (do_swap_account && PageSwapCache(page)) {
2554
		swp_entry_t ent = {.val = page_private(page)};
2555
		unsigned short id;
2556
		struct mem_cgroup *memcg;
2557 2558 2559 2560

		id = swap_cgroup_record(ent, 0);
		rcu_read_lock();
		memcg = mem_cgroup_lookup(id);
2561
		if (memcg) {
2562 2563 2564 2565
			/*
			 * This recorded memcg can be obsolete one. So, avoid
			 * calling css_tryget
			 */
2566
			if (!mem_cgroup_is_root(memcg))
2567
				res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2568
			mem_cgroup_swap_statistics(memcg, false);
2569 2570
			mem_cgroup_put(memcg);
		}
2571
		rcu_read_unlock();
2572
	}
2573 2574 2575 2576 2577 2578
	/*
	 * At swapin, we may charge account against cgroup which has no tasks.
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
	cgroup_release_and_wakeup_rmdir(&ptr->css);
2579 2580
}

D
Daisuke Nishimura 已提交
2581 2582 2583 2584 2585 2586
void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
{
	__mem_cgroup_commit_charge_swapin(page, ptr,
					MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

2587 2588
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
{
2589
	if (mem_cgroup_disabled())
2590 2591 2592
		return;
	if (!mem)
		return;
2593
	__mem_cgroup_cancel_charge(mem, 1);
2594 2595
}

2596 2597 2598
static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
				   unsigned int nr_pages,
				   const enum charge_type ctype)
2599 2600 2601
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
2602

2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
		batch->memcg = mem;
2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
	 * In those cases, all pages freed continously can be expected to be in
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */

	if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

2626
	if (nr_pages > 1)
A
Andrea Arcangeli 已提交
2627 2628
		goto direct_uncharge;

2629 2630 2631 2632 2633 2634 2635 2636
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
	if (batch->memcg != mem)
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
2637
	batch->nr_pages++;
2638
	if (uncharge_memsw)
2639
		batch->memsw_nr_pages++;
2640 2641
	return;
direct_uncharge:
2642
	res_counter_uncharge(&mem->res, nr_pages * PAGE_SIZE);
2643
	if (uncharge_memsw)
2644
		res_counter_uncharge(&mem->memsw, nr_pages * PAGE_SIZE);
2645 2646
	if (unlikely(batch->memcg != mem))
		memcg_oom_recover(mem);
2647 2648
	return;
}
2649

2650
/*
2651
 * uncharge if !page_mapped(page)
2652
 */
2653
static struct mem_cgroup *
2654
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2655
{
2656
	struct mem_cgroup *mem = NULL;
2657 2658
	unsigned int nr_pages = 1;
	struct page_cgroup *pc;
2659

2660
	if (mem_cgroup_disabled())
2661
		return NULL;
2662

K
KAMEZAWA Hiroyuki 已提交
2663
	if (PageSwapCache(page))
2664
		return NULL;
K
KAMEZAWA Hiroyuki 已提交
2665

A
Andrea Arcangeli 已提交
2666
	if (PageTransHuge(page)) {
2667
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
2668 2669
		VM_BUG_ON(!PageTransHuge(page));
	}
2670
	/*
2671
	 * Check if our page_cgroup is valid
2672
	 */
2673 2674
	pc = lookup_page_cgroup(page);
	if (unlikely(!pc || !PageCgroupUsed(pc)))
2675
		return NULL;
2676

2677
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
2678

2679 2680
	mem = pc->mem_cgroup;

K
KAMEZAWA Hiroyuki 已提交
2681 2682 2683 2684 2685
	if (!PageCgroupUsed(pc))
		goto unlock_out;

	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
K
KAMEZAWA Hiroyuki 已提交
2686
	case MEM_CGROUP_CHARGE_TYPE_DROP:
2687 2688
		/* See mem_cgroup_prepare_migration() */
		if (page_mapped(page) || PageCgroupMigration(pc))
K
KAMEZAWA Hiroyuki 已提交
2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
2700
	}
K
KAMEZAWA Hiroyuki 已提交
2701

2702
	mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -nr_pages);
K
KAMEZAWA Hiroyuki 已提交
2703

2704
	ClearPageCgroupUsed(pc);
2705 2706 2707 2708 2709 2710
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
2711

2712
	unlock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
2713 2714 2715 2716
	/*
	 * even after unlock, we have mem->res.usage here and this memcg
	 * will never be freed.
	 */
2717
	memcg_check_events(mem, page);
K
KAMEZAWA Hiroyuki 已提交
2718 2719 2720 2721 2722
	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
		mem_cgroup_swap_statistics(mem, true);
		mem_cgroup_get(mem);
	}
	if (!mem_cgroup_is_root(mem))
2723
		mem_cgroup_do_uncharge(mem, nr_pages, ctype);
2724

2725
	return mem;
K
KAMEZAWA Hiroyuki 已提交
2726 2727 2728

unlock_out:
	unlock_page_cgroup(pc);
2729
	return NULL;
2730 2731
}

2732 2733
void mem_cgroup_uncharge_page(struct page *page)
{
2734 2735 2736 2737 2738
	/* early check. */
	if (page_mapped(page))
		return;
	if (page->mapping && !PageAnon(page))
		return;
2739 2740 2741 2742 2743 2744
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
2745
	VM_BUG_ON(page->mapping);
2746 2747 2748
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
}

2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
2763 2764
		current->memcg_batch.nr_pages = 0;
		current->memcg_batch.memsw_nr_pages = 0;
2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
2785 2786 2787 2788 2789 2790
	if (batch->nr_pages)
		res_counter_uncharge(&batch->memcg->res,
				     batch->nr_pages * PAGE_SIZE);
	if (batch->memsw_nr_pages)
		res_counter_uncharge(&batch->memcg->memsw,
				     batch->memsw_nr_pages * PAGE_SIZE);
2791
	memcg_oom_recover(batch->memcg);
2792 2793 2794 2795
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

2796
#ifdef CONFIG_SWAP
2797
/*
2798
 * called after __delete_from_swap_cache() and drop "page" account.
2799 2800
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
2801 2802
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
2803 2804
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
2805 2806 2807 2808 2809 2810
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

	memcg = __mem_cgroup_uncharge_common(page, ctype);
2811

K
KAMEZAWA Hiroyuki 已提交
2812 2813 2814 2815 2816
	/*
	 * record memcg information,  if swapout && memcg != NULL,
	 * mem_cgroup_get() was called in uncharge().
	 */
	if (do_swap_account && swapout && memcg)
2817
		swap_cgroup_record(ent, css_id(&memcg->css));
2818
}
2819
#endif
2820 2821 2822 2823 2824 2825 2826

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
2827
{
2828
	struct mem_cgroup *memcg;
2829
	unsigned short id;
2830 2831 2832 2833

	if (!do_swap_account)
		return;

2834 2835 2836
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
2837
	if (memcg) {
2838 2839 2840 2841
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
2842
		if (!mem_cgroup_is_root(memcg))
2843
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2844
		mem_cgroup_swap_statistics(memcg, false);
2845 2846
		mem_cgroup_put(memcg);
	}
2847
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
2848
}
2849 2850 2851 2852 2853 2854

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
2855
 * @need_fixup: whether we should fixup res_counters and refcounts.
2856 2857 2858 2859 2860 2861 2862 2863 2864 2865
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
 * The caller must have charged to @to, IOW, called res_counter_charge() about
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
2866
		struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2867 2868 2869 2870 2871 2872 2873 2874
{
	unsigned short old_id, new_id;

	old_id = css_id(&from->css);
	new_id = css_id(&to->css);

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
2875
		mem_cgroup_swap_statistics(to, true);
2876
		/*
2877 2878 2879 2880 2881 2882
		 * This function is only called from task migration context now.
		 * It postpones res_counter and refcount handling till the end
		 * of task migration(mem_cgroup_clear_mc()) for performance
		 * improvement. But we cannot postpone mem_cgroup_get(to)
		 * because if the process that has been moved to @to does
		 * swap-in, the refcount of @to might be decreased to 0.
2883 2884
		 */
		mem_cgroup_get(to);
2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
		if (need_fixup) {
			if (!mem_cgroup_is_root(from))
				res_counter_uncharge(&from->memsw, PAGE_SIZE);
			mem_cgroup_put(from);
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			if (!mem_cgroup_is_root(to))
				res_counter_uncharge(&to->res, PAGE_SIZE);
		}
2896 2897 2898 2899 2900 2901
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2902
		struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2903 2904 2905
{
	return -EINVAL;
}
2906
#endif
K
KAMEZAWA Hiroyuki 已提交
2907

2908
/*
2909 2910
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
2911
 */
2912
int mem_cgroup_prepare_migration(struct page *page,
2913
	struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
2914
{
2915
	struct mem_cgroup *mem = NULL;
2916
	struct page_cgroup *pc;
2917
	enum charge_type ctype;
2918
	int ret = 0;
2919

2920 2921
	*ptr = NULL;

A
Andrea Arcangeli 已提交
2922
	VM_BUG_ON(PageTransHuge(page));
2923
	if (mem_cgroup_disabled())
2924 2925
		return 0;

2926 2927 2928
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
2929 2930
		mem = pc->mem_cgroup;
		css_get(&mem->css);
2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961
		/*
		 * At migrating an anonymous page, its mapcount goes down
		 * to 0 and uncharge() will be called. But, even if it's fully
		 * unmapped, migration may fail and this page has to be
		 * charged again. We set MIGRATION flag here and delay uncharge
		 * until end_migration() is called
		 *
		 * Corner Case Thinking
		 * A)
		 * When the old page was mapped as Anon and it's unmap-and-freed
		 * while migration was ongoing.
		 * If unmap finds the old page, uncharge() of it will be delayed
		 * until end_migration(). If unmap finds a new page, it's
		 * uncharged when it make mapcount to be 1->0. If unmap code
		 * finds swap_migration_entry, the new page will not be mapped
		 * and end_migration() will find it(mapcount==0).
		 *
		 * B)
		 * When the old page was mapped but migraion fails, the kernel
		 * remaps it. A charge for it is kept by MIGRATION flag even
		 * if mapcount goes down to 0. We can do remap successfully
		 * without charging it again.
		 *
		 * C)
		 * The "old" page is under lock_page() until the end of
		 * migration, so, the old page itself will not be swapped-out.
		 * If the new page is swapped out before end_migraton, our
		 * hook to usual swap-out path will catch the event.
		 */
		if (PageAnon(page))
			SetPageCgroupMigration(pc);
2962
	}
2963
	unlock_page_cgroup(pc);
2964 2965 2966 2967 2968 2969
	/*
	 * If the page is not charged at this point,
	 * we return here.
	 */
	if (!mem)
		return 0;
2970

A
Andrea Arcangeli 已提交
2971
	*ptr = mem;
2972
	ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, ptr, false);
2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
	css_put(&mem->css);/* drop extra refcnt */
	if (ret || *ptr == NULL) {
		if (PageAnon(page)) {
			lock_page_cgroup(pc);
			ClearPageCgroupMigration(pc);
			unlock_page_cgroup(pc);
			/*
			 * The old page may be fully unmapped while we kept it.
			 */
			mem_cgroup_uncharge_page(page);
		}
		return -ENOMEM;
2985
	}
2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998
	/*
	 * We charge new page before it's used/mapped. So, even if unlock_page()
	 * is called before end_migration, we can catch all events on this new
	 * page. In the case new page is migrated but not remapped, new page's
	 * mapcount will be finally 0 and we call uncharge in end_migration().
	 */
	pc = lookup_page_cgroup(newpage);
	if (PageAnon(page))
		ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
	else if (page_is_file_cache(page))
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
	else
		ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
2999
	__mem_cgroup_commit_charge(mem, page, 1, pc, ctype);
3000
	return ret;
3001
}
3002

3003
/* remove redundant charge if migration failed*/
3004
void mem_cgroup_end_migration(struct mem_cgroup *mem,
3005
	struct page *oldpage, struct page *newpage, bool migration_ok)
3006
{
3007
	struct page *used, *unused;
3008 3009 3010 3011
	struct page_cgroup *pc;

	if (!mem)
		return;
3012
	/* blocks rmdir() */
3013
	cgroup_exclude_rmdir(&mem->css);
3014
	if (!migration_ok) {
3015 3016
		used = oldpage;
		unused = newpage;
3017
	} else {
3018
		used = newpage;
3019 3020
		unused = oldpage;
	}
3021
	/*
3022 3023 3024
	 * We disallowed uncharge of pages under migration because mapcount
	 * of the page goes down to zero, temporarly.
	 * Clear the flag and check the page should be charged.
3025
	 */
3026 3027 3028 3029
	pc = lookup_page_cgroup(oldpage);
	lock_page_cgroup(pc);
	ClearPageCgroupMigration(pc);
	unlock_page_cgroup(pc);
3030

3031 3032
	__mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);

3033
	/*
3034 3035 3036 3037 3038 3039
	 * If a page is a file cache, radix-tree replacement is very atomic
	 * and we can skip this check. When it was an Anon page, its mapcount
	 * goes down to 0. But because we added MIGRATION flage, it's not
	 * uncharged yet. There are several case but page->mapcount check
	 * and USED bit check in mem_cgroup_uncharge_page() will do enough
	 * check. (see prepare_charge() also)
3040
	 */
3041 3042
	if (PageAnon(used))
		mem_cgroup_uncharge_page(used);
3043
	/*
3044 3045
	 * At migration, we may charge account against cgroup which has no
	 * tasks.
3046 3047 3048 3049
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
	cgroup_release_and_wakeup_rmdir(&mem->css);
3050
}
3051

3052
/*
3053 3054 3055 3056 3057 3058
 * A call to try to shrink memory usage on charge failure at shmem's swapin.
 * Calling hierarchical_reclaim is not enough because we should update
 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
 * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
 * not from the memcg which this page would be charged to.
 * try_charge_swapin does all of these works properly.
3059
 */
3060
int mem_cgroup_shmem_charge_fallback(struct page *page,
3061 3062
			    struct mm_struct *mm,
			    gfp_t gfp_mask)
3063
{
3064
	struct mem_cgroup *mem;
3065
	int ret;
3066

3067
	if (mem_cgroup_disabled())
3068
		return 0;
3069

3070 3071 3072
	ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
	if (!ret)
		mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
3073

3074
	return ret;
3075 3076
}

3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122
#ifdef CONFIG_DEBUG_VM
static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
	if (likely(pc) && PageCgroupUsed(pc))
		return pc;
	return NULL;
}

bool mem_cgroup_bad_page_check(struct page *page)
{
	if (mem_cgroup_disabled())
		return false;

	return lookup_page_cgroup_used(page) != NULL;
}

void mem_cgroup_print_bad_page(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup_used(page);
	if (pc) {
		int ret = -1;
		char *path;

		printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p",
		       pc, pc->flags, pc->mem_cgroup);

		path = kmalloc(PATH_MAX, GFP_KERNEL);
		if (path) {
			rcu_read_lock();
			ret = cgroup_path(pc->mem_cgroup->css.cgroup,
							path, PATH_MAX);
			rcu_read_unlock();
		}

		printk(KERN_CONT "(%s)\n",
				(ret < 0) ? "cannot get the path" : path);
		kfree(path);
	}
}
#endif

3123 3124
static DEFINE_MUTEX(set_limit_mutex);

3125
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
3126
				unsigned long long val)
3127
{
3128
	int retry_count;
3129
	u64 memswlimit, memlimit;
3130
	int ret = 0;
3131 3132
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;
3133
	int enlarge;
3134 3135 3136 3137 3138 3139 3140 3141 3142

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3143

3144
	enlarge = 0;
3145
	while (retry_count) {
3146 3147 3148 3149
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
3150 3151 3152 3153 3154 3155 3156 3157 3158 3159
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
		 * We have to guarantee mem->res.limit < mem->memsw.limit.
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
3160 3161
			break;
		}
3162 3163 3164 3165 3166

		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit < val)
			enlarge = 1;

3167
		ret = res_counter_set_limit(&memcg->res, val);
3168 3169 3170 3171 3172 3173
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3174 3175 3176 3177 3178
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3179
		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3180
						MEM_CGROUP_RECLAIM_SHRINK);
3181 3182 3183 3184 3185 3186
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
3187
	}
3188 3189
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3190

3191 3192 3193
	return ret;
}

L
Li Zefan 已提交
3194 3195
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
3196
{
3197
	int retry_count;
3198
	u64 memlimit, memswlimit, oldusage, curusage;
3199 3200
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
3201
	int enlarge = 0;
3202

3203 3204 3205
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
		 * We have to guarantee mem->res.limit < mem->memsw.limit.
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
3223 3224 3225
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val)
			enlarge = 1;
3226
		ret = res_counter_set_limit(&memcg->memsw, val);
3227 3228 3229 3230 3231 3232
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3233 3234 3235 3236 3237
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3238
		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3239 3240
						MEM_CGROUP_RECLAIM_NOSWAP |
						MEM_CGROUP_RECLAIM_SHRINK);
3241
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3242
		/* Usage is reduced ? */
3243
		if (curusage >= oldusage)
3244
			retry_count--;
3245 3246
		else
			oldusage = curusage;
3247
	}
3248 3249
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3250 3251 3252
	return ret;
}

3253
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3254
					    gfp_t gfp_mask)
3255 3256 3257 3258 3259 3260
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
3261
	unsigned long long excess;
3262 3263 3264 3265

	if (order > 0)
		return 0;

3266
	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
						gfp_mask,
						MEM_CGROUP_RECLAIM_SOFT);
		nr_reclaimed += reclaimed;
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
				if (next_mz == mz) {
					css_put(&next_mz->mem->css);
					next_mz = NULL;
				} else /* next_mz == NULL or other memcg */
					break;
			} while (1);
		}
		__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
3314
		excess = res_counter_soft_limit_excess(&mz->mem->res);
3315 3316 3317 3318 3319 3320 3321 3322
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
3323 3324
		/* If excess == 0, no tree ops */
		__mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342
		spin_unlock(&mctz->lock);
		css_put(&mz->mem->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->mem->css);
	return nr_reclaimed;
}

3343 3344 3345 3346
/*
 * This routine traverse page_cgroup in given list and drop them all.
 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
 */
3347
static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
K
KAMEZAWA Hiroyuki 已提交
3348
				int node, int zid, enum lru_list lru)
3349
{
K
KAMEZAWA Hiroyuki 已提交
3350 3351
	struct zone *zone;
	struct mem_cgroup_per_zone *mz;
3352
	struct page_cgroup *pc, *busy;
K
KAMEZAWA Hiroyuki 已提交
3353
	unsigned long flags, loop;
3354
	struct list_head *list;
3355
	int ret = 0;
3356

K
KAMEZAWA Hiroyuki 已提交
3357 3358
	zone = &NODE_DATA(node)->node_zones[zid];
	mz = mem_cgroup_zoneinfo(mem, node, zid);
3359
	list = &mz->lists[lru];
3360

3361 3362 3363 3364 3365
	loop = MEM_CGROUP_ZSTAT(mz, lru);
	/* give some margin against EBUSY etc...*/
	loop += 256;
	busy = NULL;
	while (loop--) {
3366 3367
		struct page *page;

3368
		ret = 0;
K
KAMEZAWA Hiroyuki 已提交
3369
		spin_lock_irqsave(&zone->lru_lock, flags);
3370
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
3371
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3372
			break;
3373 3374 3375 3376
		}
		pc = list_entry(list->prev, struct page_cgroup, lru);
		if (busy == pc) {
			list_move(&pc->lru, list);
3377
			busy = NULL;
K
KAMEZAWA Hiroyuki 已提交
3378
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3379 3380
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
3381
		spin_unlock_irqrestore(&zone->lru_lock, flags);
3382

3383
		page = lookup_cgroup_page(pc);
3384 3385

		ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL);
3386
		if (ret == -ENOMEM)
3387
			break;
3388 3389 3390 3391 3392 3393 3394

		if (ret == -EBUSY || ret == -EINVAL) {
			/* found lock contention or "pc" is obsolete. */
			busy = pc;
			cond_resched();
		} else
			busy = NULL;
3395
	}
K
KAMEZAWA Hiroyuki 已提交
3396

3397 3398 3399
	if (!ret && !list_empty(list))
		return -EBUSY;
	return ret;
3400 3401 3402 3403 3404 3405
}

/*
 * make mem_cgroup's charge to be 0 if there is no task.
 * This enables deleting this mem_cgroup.
 */
3406
static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
3407
{
3408 3409 3410
	int ret;
	int node, zid, shrink;
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
3411
	struct cgroup *cgrp = mem->css.cgroup;
3412

3413
	css_get(&mem->css);
3414 3415

	shrink = 0;
3416 3417 3418
	/* should free all ? */
	if (free_all)
		goto try_to_free;
3419
move_account:
3420
	do {
3421
		ret = -EBUSY;
3422 3423 3424 3425
		if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
			goto out;
		ret = -EINTR;
		if (signal_pending(current))
3426
			goto out;
3427 3428
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
3429
		drain_all_stock_sync();
3430
		ret = 0;
3431
		mem_cgroup_start_move(mem);
3432
		for_each_node_state(node, N_HIGH_MEMORY) {
3433
			for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
3434
				enum lru_list l;
3435 3436
				for_each_lru(l) {
					ret = mem_cgroup_force_empty_list(mem,
K
KAMEZAWA Hiroyuki 已提交
3437
							node, zid, l);
3438 3439 3440
					if (ret)
						break;
				}
3441
			}
3442 3443 3444
			if (ret)
				break;
		}
3445
		mem_cgroup_end_move(mem);
3446
		memcg_oom_recover(mem);
3447 3448 3449
		/* it seems parent cgroup doesn't have enough mem */
		if (ret == -ENOMEM)
			goto try_to_free;
3450
		cond_resched();
3451 3452
	/* "ret" should also be checked to ensure all lists are empty. */
	} while (mem->res.usage > 0 || ret);
3453 3454 3455
out:
	css_put(&mem->css);
	return ret;
3456 3457

try_to_free:
3458 3459
	/* returns EBUSY if there is a task or if we come here twice. */
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
3460 3461 3462
		ret = -EBUSY;
		goto out;
	}
3463 3464
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
3465 3466 3467 3468
	/* try to free all pages in this cgroup */
	shrink = 1;
	while (nr_retries && mem->res.usage > 0) {
		int progress;
3469 3470 3471 3472 3473

		if (signal_pending(current)) {
			ret = -EINTR;
			goto out;
		}
K
KOSAKI Motohiro 已提交
3474 3475
		progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
						false, get_swappiness(mem));
3476
		if (!progress) {
3477
			nr_retries--;
3478
			/* maybe some writeback is necessary */
3479
			congestion_wait(BLK_RW_ASYNC, HZ/10);
3480
		}
3481 3482

	}
K
KAMEZAWA Hiroyuki 已提交
3483
	lru_add_drain();
3484
	/* try move_account...there may be some *locked* pages. */
3485
	goto move_account;
3486 3487
}

3488 3489 3490 3491 3492 3493
int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
{
	return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
}


3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511
static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
{
	return mem_cgroup_from_cont(cont)->use_hierarchy;
}

static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
					u64 val)
{
	int retval = 0;
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
	struct cgroup *parent = cont->parent;
	struct mem_cgroup *parent_mem = NULL;

	if (parent)
		parent_mem = mem_cgroup_from_cont(parent);

	cgroup_lock();
	/*
3512
	 * If parent's use_hierarchy is set, we can't make any modifications
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
	if ((!parent_mem || !parent_mem->use_hierarchy) &&
				(val == 1 || val == 0)) {
		if (list_empty(&cont->children))
			mem->use_hierarchy = val;
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
	cgroup_unlock();

	return retval;
}

3532

3533 3534
static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *mem,
					       enum mem_cgroup_stat_index idx)
3535
{
K
KAMEZAWA Hiroyuki 已提交
3536
	struct mem_cgroup *iter;
3537
	long val = 0;
3538

3539
	/* Per-cpu values can be negative, use a signed accumulator */
K
KAMEZAWA Hiroyuki 已提交
3540 3541 3542 3543 3544 3545
	for_each_mem_cgroup_tree(iter, mem)
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
3546 3547
}

3548 3549
static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
{
K
KAMEZAWA Hiroyuki 已提交
3550
	u64 val;
3551 3552 3553 3554 3555 3556 3557 3558

	if (!mem_cgroup_is_root(mem)) {
		if (!swap)
			return res_counter_read_u64(&mem->res, RES_USAGE);
		else
			return res_counter_read_u64(&mem->memsw, RES_USAGE);
	}

3559 3560
	val = mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_CACHE);
	val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_RSS);
3561

K
KAMEZAWA Hiroyuki 已提交
3562
	if (swap)
3563
		val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
3564 3565 3566 3567

	return val << PAGE_SHIFT;
}

3568
static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
B
Balbir Singh 已提交
3569
{
3570
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3571
	u64 val;
3572 3573 3574 3575 3576 3577
	int type, name;

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (type) {
	case _MEM:
3578 3579 3580
		if (name == RES_USAGE)
			val = mem_cgroup_usage(mem, false);
		else
3581
			val = res_counter_read_u64(&mem->res, name);
3582 3583
		break;
	case _MEMSWAP:
3584 3585 3586
		if (name == RES_USAGE)
			val = mem_cgroup_usage(mem, true);
		else
3587
			val = res_counter_read_u64(&mem->memsw, name);
3588 3589 3590 3591 3592 3593
		break;
	default:
		BUG();
		break;
	}
	return val;
B
Balbir Singh 已提交
3594
}
3595 3596 3597 3598
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3599 3600
static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
			    const char *buffer)
B
Balbir Singh 已提交
3601
{
3602
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3603
	int type, name;
3604 3605 3606
	unsigned long long val;
	int ret;

3607 3608 3609
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (name) {
3610
	case RES_LIMIT:
3611 3612 3613 3614
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3615 3616
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
3617 3618 3619
		if (ret)
			break;
		if (type == _MEM)
3620
			ret = mem_cgroup_resize_limit(memcg, val);
3621 3622
		else
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
3623
		break;
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
3638 3639 3640 3641 3642
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
3643 3644
}

3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	struct cgroup *cgroup;
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	cgroup = memcg->css.cgroup;
	if (!memcg->use_hierarchy)
		goto out;

	while (cgroup->parent) {
		cgroup = cgroup->parent;
		memcg = mem_cgroup_from_cont(cgroup);
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
	return;
}

3673
static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
3674 3675
{
	struct mem_cgroup *mem;
3676
	int type, name;
3677 3678

	mem = mem_cgroup_from_cont(cont);
3679 3680 3681
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
	switch (name) {
3682
	case RES_MAX_USAGE:
3683 3684 3685 3686
		if (type == _MEM)
			res_counter_reset_max(&mem->res);
		else
			res_counter_reset_max(&mem->memsw);
3687 3688
		break;
	case RES_FAILCNT:
3689 3690 3691 3692
		if (type == _MEM)
			res_counter_reset_failcnt(&mem->res);
		else
			res_counter_reset_failcnt(&mem->memsw);
3693 3694
		break;
	}
3695

3696
	return 0;
3697 3698
}

3699 3700 3701 3702 3703 3704
static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
					struct cftype *cft)
{
	return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
}

3705
#ifdef CONFIG_MMU
3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);

	if (val >= (1 << NR_MOVE_TYPE))
		return -EINVAL;
	/*
	 * We check this value several times in both in can_attach() and
	 * attach(), so we need cgroup lock to prevent this value from being
	 * inconsistent.
	 */
	cgroup_lock();
	mem->move_charge_at_immigrate = val;
	cgroup_unlock();

	return 0;
}
3724 3725 3726 3727 3728 3729 3730
#else
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
3731

K
KAMEZAWA Hiroyuki 已提交
3732 3733 3734 3735 3736

/* For read statistics */
enum {
	MCS_CACHE,
	MCS_RSS,
3737
	MCS_FILE_MAPPED,
K
KAMEZAWA Hiroyuki 已提交
3738 3739
	MCS_PGPGIN,
	MCS_PGPGOUT,
3740
	MCS_SWAP,
K
KAMEZAWA Hiroyuki 已提交
3741 3742 3743 3744 3745 3746 3747 3748 3749 3750
	MCS_INACTIVE_ANON,
	MCS_ACTIVE_ANON,
	MCS_INACTIVE_FILE,
	MCS_ACTIVE_FILE,
	MCS_UNEVICTABLE,
	NR_MCS_STAT,
};

struct mcs_total_stat {
	s64 stat[NR_MCS_STAT];
3751 3752
};

K
KAMEZAWA Hiroyuki 已提交
3753 3754 3755 3756 3757 3758
struct {
	char *local_name;
	char *total_name;
} memcg_stat_strings[NR_MCS_STAT] = {
	{"cache", "total_cache"},
	{"rss", "total_rss"},
3759
	{"mapped_file", "total_mapped_file"},
K
KAMEZAWA Hiroyuki 已提交
3760 3761
	{"pgpgin", "total_pgpgin"},
	{"pgpgout", "total_pgpgout"},
3762
	{"swap", "total_swap"},
K
KAMEZAWA Hiroyuki 已提交
3763 3764 3765 3766 3767 3768 3769 3770
	{"inactive_anon", "total_inactive_anon"},
	{"active_anon", "total_active_anon"},
	{"inactive_file", "total_inactive_file"},
	{"active_file", "total_active_file"},
	{"unevictable", "total_unevictable"}
};


K
KAMEZAWA Hiroyuki 已提交
3771 3772
static void
mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
K
KAMEZAWA Hiroyuki 已提交
3773 3774 3775 3776
{
	s64 val;

	/* per cpu stat */
3777
	val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
K
KAMEZAWA Hiroyuki 已提交
3778
	s->stat[MCS_CACHE] += val * PAGE_SIZE;
3779
	val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
K
KAMEZAWA Hiroyuki 已提交
3780
	s->stat[MCS_RSS] += val * PAGE_SIZE;
3781
	val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
3782
	s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
3783
	val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGIN);
K
KAMEZAWA Hiroyuki 已提交
3784
	s->stat[MCS_PGPGIN] += val;
3785
	val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGOUT);
K
KAMEZAWA Hiroyuki 已提交
3786
	s->stat[MCS_PGPGOUT] += val;
3787
	if (do_swap_account) {
3788
		val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
3789 3790
		s->stat[MCS_SWAP] += val * PAGE_SIZE;
	}
K
KAMEZAWA Hiroyuki 已提交
3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807

	/* per zone stat */
	val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
	s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
	s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
	s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
	s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
	s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
}

static void
mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
{
K
KAMEZAWA Hiroyuki 已提交
3808 3809 3810 3811
	struct mem_cgroup *iter;

	for_each_mem_cgroup_tree(iter, mem)
		mem_cgroup_get_local_stat(iter, s);
K
KAMEZAWA Hiroyuki 已提交
3812 3813
}

3814 3815
static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
				 struct cgroup_map_cb *cb)
3816 3817
{
	struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
K
KAMEZAWA Hiroyuki 已提交
3818
	struct mcs_total_stat mystat;
3819 3820
	int i;

K
KAMEZAWA Hiroyuki 已提交
3821 3822
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_local_stat(mem_cont, &mystat);
3823

3824 3825 3826
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
3827
		cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
3828
	}
L
Lee Schermerhorn 已提交
3829

K
KAMEZAWA Hiroyuki 已提交
3830
	/* Hierarchical information */
3831 3832 3833 3834 3835 3836 3837
	{
		unsigned long long limit, memsw_limit;
		memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
		cb->fill(cb, "hierarchical_memory_limit", limit);
		if (do_swap_account)
			cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
	}
K
KOSAKI Motohiro 已提交
3838

K
KAMEZAWA Hiroyuki 已提交
3839 3840
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_total_stat(mem_cont, &mystat);
3841 3842 3843
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
3844
		cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
3845
	}
K
KAMEZAWA Hiroyuki 已提交
3846

K
KOSAKI Motohiro 已提交
3847
#ifdef CONFIG_DEBUG_VM
3848
	cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
K
KOSAKI Motohiro 已提交
3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875

	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
				mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);

				recent_rotated[0] +=
					mz->reclaim_stat.recent_rotated[0];
				recent_rotated[1] +=
					mz->reclaim_stat.recent_rotated[1];
				recent_scanned[0] +=
					mz->reclaim_stat.recent_scanned[0];
				recent_scanned[1] +=
					mz->reclaim_stat.recent_scanned[1];
			}
		cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
		cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
		cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
		cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
	}
#endif

3876 3877 3878
	return 0;
}

K
KOSAKI Motohiro 已提交
3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890
static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);

	return get_swappiness(memcg);
}

static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
				       u64 val)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;
3891

K
KOSAKI Motohiro 已提交
3892 3893 3894 3895 3896 3897 3898
	if (val > 100)
		return -EINVAL;

	if (cgrp->parent == NULL)
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);
3899 3900 3901

	cgroup_lock();

K
KOSAKI Motohiro 已提交
3902 3903
	/* If under hierarchy, only empty-root can set this value */
	if ((parent->use_hierarchy) ||
3904 3905
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
		cgroup_unlock();
K
KOSAKI Motohiro 已提交
3906
		return -EINVAL;
3907
	}
K
KOSAKI Motohiro 已提交
3908 3909 3910

	memcg->swappiness = val;

3911 3912
	cgroup_unlock();

K
KOSAKI Motohiro 已提交
3913 3914 3915
	return 0;
}

3916 3917 3918 3919 3920 3921 3922 3923
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
	u64 usage;
	int i;

	rcu_read_lock();
	if (!swap)
3924
		t = rcu_dereference(memcg->thresholds.primary);
3925
	else
3926
		t = rcu_dereference(memcg->memsw_thresholds.primary);
3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937

	if (!t)
		goto unlock;

	usage = mem_cgroup_usage(memcg, swap);

	/*
	 * current_threshold points to threshold just below usage.
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
3938
	i = t->current_threshold;
3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
3962
	t->current_threshold = i - 1;
3963 3964 3965 3966 3967 3968
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
3969 3970 3971 3972 3973 3974 3975
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

	return _a->threshold - _b->threshold;
}

K
KAMEZAWA Hiroyuki 已提交
3986
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
K
KAMEZAWA Hiroyuki 已提交
3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
{
	struct mem_cgroup_eventfd_list *ev;

	list_for_each_entry(ev, &mem->oom_notify, list)
		eventfd_signal(ev->eventfd, 1);
	return 0;
}

static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
{
K
KAMEZAWA Hiroyuki 已提交
3997 3998 3999 4000
	struct mem_cgroup *iter;

	for_each_mem_cgroup_tree(iter, mem)
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
4001 4002 4003 4004
}

static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4005 4006
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4007 4008
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4009 4010
	int type = MEMFILE_TYPE(cft->private);
	u64 threshold, usage;
4011
	int i, size, ret;
4012 4013 4014 4015 4016 4017

	ret = res_counter_memparse_write_strategy(args, &threshold);
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
4018

4019
	if (type == _MEM)
4020
		thresholds = &memcg->thresholds;
4021
	else if (type == _MEMSWAP)
4022
		thresholds = &memcg->memsw_thresholds;
4023 4024 4025 4026 4027 4028
	else
		BUG();

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before adding a new one */
4029
	if (thresholds->primary)
4030 4031
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

4032
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4033 4034

	/* Allocate memory for new array of thresholds */
4035
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
4036
			GFP_KERNEL);
4037
	if (!new) {
4038 4039 4040
		ret = -ENOMEM;
		goto unlock;
	}
4041
	new->size = size;
4042 4043

	/* Copy thresholds (if any) to new array */
4044 4045
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
4046
				sizeof(struct mem_cgroup_threshold));
4047 4048
	}

4049
	/* Add new threshold */
4050 4051
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
4052 4053

	/* Sort thresholds. Registering of new threshold isn't time-critical */
4054
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
4055 4056 4057
			compare_thresholds, NULL);

	/* Find current threshold */
4058
	new->current_threshold = -1;
4059
	for (i = 0; i < size; i++) {
4060
		if (new->entries[i].threshold < usage) {
4061
			/*
4062 4063
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
4064 4065
			 * it here.
			 */
4066
			++new->current_threshold;
4067 4068 4069
		}
	}

4070 4071 4072 4073 4074
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
4075

4076
	/* To be sure that nobody uses thresholds */
4077 4078 4079 4080 4081 4082 4083 4084
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

4085
static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4086
	struct cftype *cft, struct eventfd_ctx *eventfd)
4087 4088
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4089 4090
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4091 4092
	int type = MEMFILE_TYPE(cft->private);
	u64 usage;
4093
	int i, j, size;
4094 4095 4096

	mutex_lock(&memcg->thresholds_lock);
	if (type == _MEM)
4097
		thresholds = &memcg->thresholds;
4098
	else if (type == _MEMSWAP)
4099
		thresholds = &memcg->memsw_thresholds;
4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
	else
		BUG();

	/*
	 * Something went wrong if we trying to unregister a threshold
	 * if we don't have thresholds
	 */
	BUG_ON(!thresholds);

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
4115 4116 4117
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
4118 4119 4120
			size++;
	}

4121
	new = thresholds->spare;
4122

4123 4124
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
4125 4126
		kfree(new);
		new = NULL;
4127
		goto swap_buffers;
4128 4129
	}

4130
	new->size = size;
4131 4132

	/* Copy thresholds and find current threshold */
4133 4134 4135
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
4136 4137
			continue;

4138 4139
		new->entries[j] = thresholds->primary->entries[i];
		if (new->entries[j].threshold < usage) {
4140
			/*
4141
			 * new->current_threshold will not be used
4142 4143 4144
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
4145
			++new->current_threshold;
4146 4147 4148 4149
		}
		j++;
	}

4150
swap_buffers:
4151 4152 4153
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
	rcu_assign_pointer(thresholds->primary, new);
4154

4155
	/* To be sure that nobody uses thresholds */
4156 4157 4158 4159
	synchronize_rcu();

	mutex_unlock(&memcg->thresholds_lock);
}
4160

K
KAMEZAWA Hiroyuki 已提交
4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185
static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *event;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);
	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

	mutex_lock(&memcg_oom_mutex);

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
	if (atomic_read(&memcg->oom_lock))
		eventfd_signal(eventfd, 1);
	mutex_unlock(&memcg_oom_mutex);

	return 0;
}

4186
static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206
	struct cftype *cft, struct eventfd_ctx *eventfd)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *ev, *tmp;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);

	mutex_lock(&memcg_oom_mutex);

	list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

	mutex_unlock(&memcg_oom_mutex);
}

4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240
static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
	struct cftype *cft,  struct cgroup_map_cb *cb)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);

	cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);

	if (atomic_read(&mem->oom_lock))
		cb->fill(cb, "under_oom", 1);
	else
		cb->fill(cb, "under_oom", 0);
	return 0;
}

static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
	struct cftype *cft, u64 val)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;

	/* cannot set to root cgroup and only 0 and 1 are allowed */
	if (!cgrp->parent || !((val == 0) || (val == 1)))
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);

	cgroup_lock();
	/* oom-kill-disable is a flag for subhierarchy. */
	if ((parent->use_hierarchy) ||
	    (mem->use_hierarchy && !list_empty(&cgrp->children))) {
		cgroup_unlock();
		return -EINVAL;
	}
	mem->oom_kill_disable = val;
4241 4242
	if (!val)
		memcg_oom_recover(mem);
4243 4244 4245 4246
	cgroup_unlock();
	return 0;
}

B
Balbir Singh 已提交
4247 4248
static struct cftype mem_cgroup_files[] = {
	{
4249
		.name = "usage_in_bytes",
4250
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4251
		.read_u64 = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4252 4253
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
B
Balbir Singh 已提交
4254
	},
4255 4256
	{
		.name = "max_usage_in_bytes",
4257
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4258
		.trigger = mem_cgroup_reset,
4259 4260
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
4261
	{
4262
		.name = "limit_in_bytes",
4263
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4264
		.write_string = mem_cgroup_write,
4265
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
4266
	},
4267 4268 4269 4270 4271 4272
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
4273 4274
	{
		.name = "failcnt",
4275
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4276
		.trigger = mem_cgroup_reset,
4277
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
4278
	},
4279 4280
	{
		.name = "stat",
4281
		.read_map = mem_control_stat_show,
4282
	},
4283 4284 4285 4286
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
4287 4288 4289 4290 4291
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
4292 4293 4294 4295 4296
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4297 4298 4299 4300 4301
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4302 4303
	{
		.name = "oom_control",
4304 4305
		.read_map = mem_cgroup_oom_control_read,
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4306 4307 4308 4309
		.register_event = mem_cgroup_oom_register_event,
		.unregister_event = mem_cgroup_oom_unregister_event,
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
B
Balbir Singh 已提交
4310 4311
};

4312 4313 4314 4315 4316 4317
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4318 4319
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
};

static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	if (!do_swap_account)
		return 0;
	return cgroup_add_files(cont, ss, memsw_cgroup_files,
				ARRAY_SIZE(memsw_cgroup_files));
};
#else
static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	return 0;
}
#endif

4355 4356 4357
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
{
	struct mem_cgroup_per_node *pn;
4358
	struct mem_cgroup_per_zone *mz;
4359
	enum lru_list l;
4360
	int zone, tmp = node;
4361 4362 4363 4364 4365 4366 4367 4368
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4369 4370
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4371
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4372 4373
	if (!pn)
		return 1;
4374

4375
	mem->info.nodeinfo[node] = pn;
4376 4377
	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
4378 4379
		for_each_lru(l)
			INIT_LIST_HEAD(&mz->lists[l]);
4380
		mz->usage_in_excess = 0;
4381 4382
		mz->on_tree = false;
		mz->mem = mem;
4383
	}
4384 4385 4386
	return 0;
}

4387 4388 4389 4390 4391
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
{
	kfree(mem->info.nodeinfo[node]);
}

4392 4393 4394
static struct mem_cgroup *mem_cgroup_alloc(void)
{
	struct mem_cgroup *mem;
4395
	int size = sizeof(struct mem_cgroup);
4396

4397
	/* Can be very big if MAX_NUMNODES is very big */
4398
	if (size < PAGE_SIZE)
4399
		mem = kzalloc(size, GFP_KERNEL);
4400
	else
4401
		mem = vzalloc(size);
4402

4403 4404 4405
	if (!mem)
		return NULL;

4406
	mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4407 4408
	if (!mem->stat)
		goto out_free;
4409
	spin_lock_init(&mem->pcp_counter_lock);
4410
	return mem;
4411 4412 4413 4414 4415 4416 4417

out_free:
	if (size < PAGE_SIZE)
		kfree(mem);
	else
		vfree(mem);
	return NULL;
4418 4419
}

4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430
/*
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
 */

4431
static void __mem_cgroup_free(struct mem_cgroup *mem)
4432
{
K
KAMEZAWA Hiroyuki 已提交
4433 4434
	int node;

4435
	mem_cgroup_remove_from_trees(mem);
K
KAMEZAWA Hiroyuki 已提交
4436 4437
	free_css_id(&mem_cgroup_subsys, &mem->css);

K
KAMEZAWA Hiroyuki 已提交
4438 4439 4440
	for_each_node_state(node, N_POSSIBLE)
		free_mem_cgroup_per_zone_info(mem, node);

4441 4442
	free_percpu(mem->stat);
	if (sizeof(struct mem_cgroup) < PAGE_SIZE)
4443 4444 4445 4446 4447
		kfree(mem);
	else
		vfree(mem);
}

4448 4449 4450 4451 4452
static void mem_cgroup_get(struct mem_cgroup *mem)
{
	atomic_inc(&mem->refcnt);
}

4453
static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
4454
{
4455
	if (atomic_sub_and_test(count, &mem->refcnt)) {
4456
		struct mem_cgroup *parent = parent_mem_cgroup(mem);
4457
		__mem_cgroup_free(mem);
4458 4459 4460
		if (parent)
			mem_cgroup_put(parent);
	}
4461 4462
}

4463 4464 4465 4466 4467
static void mem_cgroup_put(struct mem_cgroup *mem)
{
	__mem_cgroup_put(mem, 1);
}

4468 4469 4470 4471 4472 4473 4474 4475 4476
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
{
	if (!mem->res.parent)
		return NULL;
	return mem_cgroup_from_res_counter(mem->res.parent, res);
}
4477

4478 4479 4480
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static void __init enable_swap_cgroup(void)
{
4481
	if (!mem_cgroup_disabled() && really_do_swap_account)
4482 4483 4484 4485 4486 4487 4488 4489
		do_swap_account = 1;
}
#else
static void __init enable_swap_cgroup(void)
{
}
#endif

4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514
static int mem_cgroup_soft_limit_tree_init(void)
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

	for_each_node_state(node, N_POSSIBLE) {
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
		if (!rtpn)
			return 1;

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
	return 0;
}

L
Li Zefan 已提交
4515
static struct cgroup_subsys_state * __ref
B
Balbir Singh 已提交
4516 4517
mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
{
4518
	struct mem_cgroup *mem, *parent;
K
KAMEZAWA Hiroyuki 已提交
4519
	long error = -ENOMEM;
4520
	int node;
B
Balbir Singh 已提交
4521

4522 4523
	mem = mem_cgroup_alloc();
	if (!mem)
K
KAMEZAWA Hiroyuki 已提交
4524
		return ERR_PTR(error);
4525

4526 4527 4528
	for_each_node_state(node, N_POSSIBLE)
		if (alloc_mem_cgroup_per_zone_info(mem, node))
			goto free_out;
4529

4530
	/* root ? */
4531
	if (cont->parent == NULL) {
4532
		int cpu;
4533
		enable_swap_cgroup();
4534
		parent = NULL;
4535
		root_mem_cgroup = mem;
4536 4537
		if (mem_cgroup_soft_limit_tree_init())
			goto free_out;
4538 4539 4540 4541 4542
		for_each_possible_cpu(cpu) {
			struct memcg_stock_pcp *stock =
						&per_cpu(memcg_stock, cpu);
			INIT_WORK(&stock->work, drain_local_stock);
		}
4543
		hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
4544
	} else {
4545
		parent = mem_cgroup_from_cont(cont->parent);
4546
		mem->use_hierarchy = parent->use_hierarchy;
4547
		mem->oom_kill_disable = parent->oom_kill_disable;
4548
	}
4549

4550 4551 4552
	if (parent && parent->use_hierarchy) {
		res_counter_init(&mem->res, &parent->res);
		res_counter_init(&mem->memsw, &parent->memsw);
4553 4554 4555 4556 4557 4558 4559
		/*
		 * We increment refcnt of the parent to ensure that we can
		 * safely access it on res_counter_charge/uncharge.
		 * This refcnt will be decremented when freeing this
		 * mem_cgroup(see mem_cgroup_put).
		 */
		mem_cgroup_get(parent);
4560 4561 4562 4563
	} else {
		res_counter_init(&mem->res, NULL);
		res_counter_init(&mem->memsw, NULL);
	}
K
KAMEZAWA Hiroyuki 已提交
4564
	mem->last_scanned_child = 0;
K
KAMEZAWA Hiroyuki 已提交
4565
	INIT_LIST_HEAD(&mem->oom_notify);
4566

K
KOSAKI Motohiro 已提交
4567 4568
	if (parent)
		mem->swappiness = get_swappiness(parent);
4569
	atomic_set(&mem->refcnt, 1);
4570
	mem->move_charge_at_immigrate = 0;
4571
	mutex_init(&mem->thresholds_lock);
B
Balbir Singh 已提交
4572
	return &mem->css;
4573
free_out:
4574
	__mem_cgroup_free(mem);
4575
	root_mem_cgroup = NULL;
K
KAMEZAWA Hiroyuki 已提交
4576
	return ERR_PTR(error);
B
Balbir Singh 已提交
4577 4578
}

4579
static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
4580 4581 4582
					struct cgroup *cont)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4583 4584

	return mem_cgroup_force_empty(mem, false);
4585 4586
}

B
Balbir Singh 已提交
4587 4588 4589
static void mem_cgroup_destroy(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
4590 4591 4592
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);

	mem_cgroup_put(mem);
B
Balbir Singh 已提交
4593 4594 4595 4596 4597
}

static int mem_cgroup_populate(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
4598 4599 4600 4601 4602 4603 4604 4605
	int ret;

	ret = cgroup_add_files(cont, ss, mem_cgroup_files,
				ARRAY_SIZE(mem_cgroup_files));

	if (!ret)
		ret = register_memsw_files(cont, ss);
	return ret;
B
Balbir Singh 已提交
4606 4607
}

4608
#ifdef CONFIG_MMU
4609
/* Handlers for move charge at task migration. */
4610 4611
#define PRECHARGE_COUNT_AT_ONCE	256
static int mem_cgroup_do_precharge(unsigned long count)
4612
{
4613 4614
	int ret = 0;
	int batch_count = PRECHARGE_COUNT_AT_ONCE;
4615 4616
	struct mem_cgroup *mem = mc.to;

4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651
	if (mem_cgroup_is_root(mem)) {
		mc.precharge += count;
		/* we don't need css_get for root */
		return ret;
	}
	/* try to charge at once */
	if (count > 1) {
		struct res_counter *dummy;
		/*
		 * "mem" cannot be under rmdir() because we've already checked
		 * by cgroup_lock_live_cgroup() that it is not removed and we
		 * are still under the same cgroup_mutex. So we can postpone
		 * css_get().
		 */
		if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
			goto one_by_one;
		if (do_swap_account && res_counter_charge(&mem->memsw,
						PAGE_SIZE * count, &dummy)) {
			res_counter_uncharge(&mem->res, PAGE_SIZE * count);
			goto one_by_one;
		}
		mc.precharge += count;
		return ret;
	}
one_by_one:
	/* fall back to one by one charge */
	while (count--) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		if (!batch_count--) {
			batch_count = PRECHARGE_COUNT_AT_ONCE;
			cond_resched();
		}
4652
		ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, 1, &mem, false);
4653 4654 4655 4656 4657
		if (ret || !mem)
			/* mem_cgroup_clear_mc() will do uncharge later */
			return -ENOMEM;
		mc.precharge++;
	}
4658 4659 4660 4661 4662 4663 4664 4665
	return ret;
}

/**
 * is_target_pte_for_mc - check a pte whether it is valid for move charge
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
4666
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
4667 4668 4669 4670 4671 4672
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
4673 4674 4675
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
4676 4677 4678 4679 4680
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
4681
	swp_entry_t	ent;
4682 4683 4684 4685 4686
};

enum mc_target_type {
	MC_TARGET_NONE,	/* not used */
	MC_TARGET_PAGE,
4687
	MC_TARGET_SWAP,
4688 4689
};

D
Daisuke Nishimura 已提交
4690 4691
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
4692
{
D
Daisuke Nishimura 已提交
4693
	struct page *page = vm_normal_page(vma, addr, ptent);
4694

D
Daisuke Nishimura 已提交
4695 4696 4697 4698 4699 4700
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
		/* we don't move shared anon */
		if (!move_anon() || page_mapcount(page) > 2)
			return NULL;
4701 4702
	} else if (!move_file())
		/* we ignore mapcount for file pages */
D
Daisuke Nishimura 已提交
4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720
		return NULL;
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	int usage_count;
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

	if (!move_anon() || non_swap_entry(ent))
		return NULL;
	usage_count = mem_cgroup_count_swap_user(ent, &page);
	if (usage_count > 1) { /* we don't move shared anon */
4721 4722
		if (page)
			put_page(page);
D
Daisuke Nishimura 已提交
4723
		return NULL;
4724
	}
D
Daisuke Nishimura 已提交
4725 4726 4727 4728 4729 4730
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}

4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct inode *inode;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
	if (!move_file())
		return NULL;

	inode = vma->vm_file->f_path.dentry->d_inode;
	mapping = vma->vm_file->f_mapping;
	if (pte_none(ptent))
		pgoff = linear_page_index(vma, addr);
	else /* pte_file(ptent) is true */
		pgoff = pte_to_pgoff(ptent);

	/* page is moved even if it's not RSS of this task(page-faulted). */
	if (!mapping_cap_swap_backed(mapping)) { /* normal file */
		page = find_get_page(mapping, pgoff);
	} else { /* shmem/tmpfs file. we should take account of swap too. */
		swp_entry_t ent;
		mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
		if (do_swap_account)
			entry->val = ent.val;
	}

	return page;
}

D
Daisuke Nishimura 已提交
4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775
static int is_target_pte_for_mc(struct vm_area_struct *vma,
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
	int ret = 0;
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4776 4777
	else if (pte_none(ptent) || pte_file(ptent))
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
4778 4779 4780

	if (!page && !ent.val)
		return 0;
4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795
	if (page) {
		pc = lookup_page_cgroup(page);
		/*
		 * Do only loose check w/o page_cgroup lock.
		 * mem_cgroup_move_account() checks the pc is valid or not under
		 * the lock.
		 */
		if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
4796 4797
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
4798 4799 4800 4801
			css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813
	}
	return ret;
}

static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

4814 4815
	split_huge_page_pmd(walk->mm, pmd);

4816 4817 4818 4819 4820 4821 4822
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
		if (is_target_pte_for_mc(vma, addr, *pte, NULL))
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

4823 4824 4825
	return 0;
}

4826 4827 4828 4829 4830
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;
	struct vm_area_struct *vma;

4831
	down_read(&mm->mmap_sem);
4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		struct mm_walk mem_cgroup_count_precharge_walk = {
			.pmd_entry = mem_cgroup_count_precharge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		walk_page_range(vma->vm_start, vma->vm_end,
					&mem_cgroup_count_precharge_walk);
	}
4843
	up_read(&mm->mmap_sem);
4844 4845 4846 4847 4848 4849 4850 4851 4852

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
4853 4854 4855 4856 4857
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
4858 4859
}

4860 4861
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
4862
{
4863 4864 4865
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

4866
	/* we must uncharge all the leftover precharges from mc.to */
4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877
	if (mc.precharge) {
		__mem_cgroup_cancel_charge(mc.to, mc.precharge);
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
		mc.moved_charge = 0;
4878
	}
4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
		if (!mem_cgroup_is_root(mc.from))
			res_counter_uncharge(&mc.from->memsw,
						PAGE_SIZE * mc.moved_swap);
		__mem_cgroup_put(mc.from, mc.moved_swap);

		if (!mem_cgroup_is_root(mc.to)) {
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			res_counter_uncharge(&mc.to->res,
						PAGE_SIZE * mc.moved_swap);
		}
		/* we've already done mem_cgroup_get(mc.to) */
		mc.moved_swap = 0;
	}
4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	struct mem_cgroup *from = mc.from;

	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
4913
	spin_lock(&mc.lock);
4914 4915
	mc.from = NULL;
	mc.to = NULL;
4916
	spin_unlock(&mc.lock);
4917
	mem_cgroup_end_move(from);
4918 4919
}

4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937
static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
				struct task_struct *p,
				bool threadgroup)
{
	int ret = 0;
	struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);

	if (mem->move_charge_at_immigrate) {
		struct mm_struct *mm;
		struct mem_cgroup *from = mem_cgroup_from_task(p);

		VM_BUG_ON(from == mem);

		mm = get_task_mm(p);
		if (!mm)
			return 0;
		/* We move charges only when we move a owner of the mm */
4938 4939 4940 4941
		if (mm->owner == p) {
			VM_BUG_ON(mc.from);
			VM_BUG_ON(mc.to);
			VM_BUG_ON(mc.precharge);
4942
			VM_BUG_ON(mc.moved_charge);
4943
			VM_BUG_ON(mc.moved_swap);
4944
			mem_cgroup_start_move(from);
4945
			spin_lock(&mc.lock);
4946 4947
			mc.from = from;
			mc.to = mem;
4948
			spin_unlock(&mc.lock);
4949
			/* We set mc.moving_task later */
4950 4951 4952 4953

			ret = mem_cgroup_precharge_mc(mm);
			if (ret)
				mem_cgroup_clear_mc();
4954 4955
		}
		mmput(mm);
4956 4957 4958 4959 4960 4961 4962 4963 4964
	}
	return ret;
}

static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
				struct task_struct *p,
				bool threadgroup)
{
4965
	mem_cgroup_clear_mc();
4966 4967
}

4968 4969 4970
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
4971
{
4972 4973 4974 4975 4976
	int ret = 0;
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

4977
	split_huge_page_pmd(walk->mm, pmd);
4978 4979 4980 4981 4982 4983 4984 4985
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
		union mc_target target;
		int type;
		struct page *page;
		struct page_cgroup *pc;
4986
		swp_entry_t ent;
4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997

		if (!mc.precharge)
			break;

		type = is_target_pte_for_mc(vma, addr, ptent, &target);
		switch (type) {
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
			pc = lookup_page_cgroup(page);
4998 4999
			if (!mem_cgroup_move_account(page, 1, pc,
						     mc.from, mc.to, false)) {
5000
				mc.precharge--;
5001 5002
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
5003 5004 5005 5006 5007
			}
			putback_lru_page(page);
put:			/* is_target_pte_for_mc() gets the page */
			put_page(page);
			break;
5008 5009
		case MC_TARGET_SWAP:
			ent = target.ent;
5010 5011
			if (!mem_cgroup_move_swap_account(ent,
						mc.from, mc.to, false)) {
5012
				mc.precharge--;
5013 5014 5015
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
5016
			break;
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
5031
		ret = mem_cgroup_do_precharge(1);
5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	lru_add_drain_all();
5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		int ret;
		struct mm_walk mem_cgroup_move_charge_walk = {
			.pmd_entry = mem_cgroup_move_charge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		ret = walk_page_range(vma->vm_start, vma->vm_end,
						&mem_cgroup_move_charge_walk);
		if (ret)
			/*
			 * means we have consumed all precharges and failed in
			 * doing additional charge. Just abandon here.
			 */
			break;
	}
5075
	up_read(&mm->mmap_sem);
5076 5077
}

B
Balbir Singh 已提交
5078 5079 5080
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
5081 5082
				struct task_struct *p,
				bool threadgroup)
B
Balbir Singh 已提交
5083
{
5084 5085 5086
	struct mm_struct *mm;

	if (!mc.to)
5087 5088 5089
		/* no need to move charge */
		return;

5090 5091 5092 5093 5094
	mm = get_task_mm(p);
	if (mm) {
		mem_cgroup_move_charge(mm);
		mmput(mm);
	}
5095
	mem_cgroup_clear_mc();
B
Balbir Singh 已提交
5096
}
5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118
#else	/* !CONFIG_MMU */
static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
				struct task_struct *p,
				bool threadgroup)
{
	return 0;
}
static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
				struct task_struct *p,
				bool threadgroup)
{
}
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
				struct task_struct *p,
				bool threadgroup)
{
}
#endif
B
Balbir Singh 已提交
5119

B
Balbir Singh 已提交
5120 5121 5122 5123
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
	.create = mem_cgroup_create,
5124
	.pre_destroy = mem_cgroup_pre_destroy,
B
Balbir Singh 已提交
5125 5126
	.destroy = mem_cgroup_destroy,
	.populate = mem_cgroup_populate,
5127 5128
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
5129
	.attach = mem_cgroup_move_task,
5130
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
5131
	.use_id = 1,
B
Balbir Singh 已提交
5132
};
5133 5134

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
5135 5136 5137
static int __init enable_swap_account(char *s)
{
	/* consider enabled if no parameter or 1 is given */
5138
	if (!(*s) || !strcmp(s, "=1"))
5139
		really_do_swap_account = 1;
5140
	else if (!strcmp(s, "=0"))
5141 5142 5143 5144
		really_do_swap_account = 0;
	return 1;
}
__setup("swapaccount", enable_swap_account);
5145 5146 5147

static int __init disable_swap_account(char *s)
{
5148
	printk_once("noswapaccount is deprecated and will be removed in 2.6.40. Use swapaccount=0 instead\n");
5149
	enable_swap_account("=0");
5150 5151 5152 5153
	return 1;
}
__setup("noswapaccount", disable_swap_account);
#endif