memcontrol.c 142.3 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
B
Balbir Singh 已提交
13 14 15 16 17 18 19 20 21 22 23 24 25 26
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
27
#include <linux/mm.h>
28
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
29
#include <linux/pagemap.h>
30
#include <linux/smp.h>
31
#include <linux/page-flags.h>
32
#include <linux/backing-dev.h>
33 34
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
35
#include <linux/limits.h>
36
#include <linux/mutex.h>
37
#include <linux/rbtree.h>
38
#include <linux/slab.h>
39
#include <linux/swap.h>
40
#include <linux/swapops.h>
41
#include <linux/spinlock.h>
42 43
#include <linux/eventfd.h>
#include <linux/sort.h>
44
#include <linux/fs.h>
45
#include <linux/seq_file.h>
46
#include <linux/vmalloc.h>
47
#include <linux/mm_inline.h>
48
#include <linux/page_cgroup.h>
49
#include <linux/cpu.h>
50
#include <linux/oom.h>
K
KAMEZAWA Hiroyuki 已提交
51
#include "internal.h"
B
Balbir Singh 已提交
52

53 54
#include <asm/uaccess.h>

55 56
#include <trace/events/vmscan.h>

57 58
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
#define MEM_CGROUP_RECLAIM_RETRIES	5
59
struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
60

61
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
L
Li Zefan 已提交
62
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
63
int do_swap_account __read_mostly;
64 65 66 67 68 69 70 71

/* for remember boot option*/
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata = 0;
#endif

72 73 74 75 76
#else
#define do_swap_account		(0)
#endif


77 78 79 80 81 82 83 84
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
85
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as anon rss */
86
	MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
87
	MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
88
	MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
89
	MEM_CGROUP_ON_MOVE,	/* someone is moving account between groups */
90 91 92
	MEM_CGROUP_STAT_NSTATS,
};

93 94 95 96
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
	MEM_CGROUP_EVENTS_COUNT,	/* # of pages paged in/out */
97 98
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
99 100
	MEM_CGROUP_EVENTS_NSTATS,
};
101 102 103 104 105 106 107 108 109
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
110
	MEM_CGROUP_TARGET_NUMAINFO,
111 112 113 114
	MEM_CGROUP_NTARGETS,
};
#define THRESHOLDS_EVENTS_TARGET (128)
#define SOFTLIMIT_EVENTS_TARGET (1024)
115
#define NUMAINFO_EVENTS_TARGET	(1024)
116

117
struct mem_cgroup_stat_cpu {
118
	long count[MEM_CGROUP_STAT_NSTATS];
119
	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
120
	unsigned long targets[MEM_CGROUP_NTARGETS];
121 122
};

123 124 125 126
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
127 128 129
	/*
	 * spin_lock to protect the per cgroup LRU
	 */
130 131
	struct list_head	lists[NR_LRU_LISTS];
	unsigned long		count[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
132 133

	struct zone_reclaim_stat reclaim_stat;
134 135 136 137
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
138 139
	struct mem_cgroup	*mem;		/* Back pointer, we cannot */
						/* use container_of	   */
140 141 142 143 144 145 146 147 148 149 150 151
};
/* Macro for accessing counter */
#define MEM_CGROUP_ZSTAT(mz, idx)	((mz)->count[(idx)])

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
	struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
};

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

172 173 174 175 176
struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	u64 threshold;
};

K
KAMEZAWA Hiroyuki 已提交
177
/* For threshold */
178 179
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below usage. */
180
	int current_threshold;
181 182 183 184 185
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};
186 187 188 189 190 191 192 193 194 195 196 197

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

K
KAMEZAWA Hiroyuki 已提交
198 199 200 201 202
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
203

204 205
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
206

B
Balbir Singh 已提交
207 208 209 210 211 212 213
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
214 215 216
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
217 218 219 220 221 222 223
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
224 225 226 227
	/*
	 * the counter to account for mem+swap usage.
	 */
	struct res_counter memsw;
228 229 230 231
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 */
232
	struct mem_cgroup_lru_info info;
233
	/*
234
	 * While reclaiming in a hierarchy, we cache the last child we
K
KAMEZAWA Hiroyuki 已提交
235
	 * reclaimed from.
236
	 */
K
KAMEZAWA Hiroyuki 已提交
237
	int last_scanned_child;
238 239 240
	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
241 242
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
243
#endif
244 245 246 247
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
248 249 250 251

	bool		oom_lock;
	atomic_t	under_oom;

252
	atomic_t	refcnt;
253

254
	int	swappiness;
255 256
	/* OOM-Killer disable */
	int		oom_kill_disable;
K
KOSAKI Motohiro 已提交
257

258 259 260
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

261 262 263 264
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
265
	struct mem_cgroup_thresholds thresholds;
266

267
	/* thresholds for mem+swap usage. RCU-protected */
268
	struct mem_cgroup_thresholds memsw_thresholds;
269

K
KAMEZAWA Hiroyuki 已提交
270 271
	/* For oom notifier event fd */
	struct list_head oom_notify;
272

273 274 275 276 277
	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long 	move_charge_at_immigrate;
278
	/*
279
	 * percpu counter.
280
	 */
281
	struct mem_cgroup_stat_cpu *stat;
282 283 284 285 286 287
	/*
	 * used when a cpu is offlined or other synchronizations
	 * See mem_cgroup_read_stat().
	 */
	struct mem_cgroup_stat_cpu nocpu_base;
	spinlock_t pcp_counter_lock;
B
Balbir Singh 已提交
288 289
};

290 291 292 293 294 295
/* Stuffs for move charges at task migration. */
/*
 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
 * left-shifted bitmap of these types.
 */
enum move_type {
296
	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */
297
	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */
298 299 300
	NR_MOVE_TYPE,
};

301 302
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
303
	spinlock_t	  lock; /* for from, to */
304 305 306
	struct mem_cgroup *from;
	struct mem_cgroup *to;
	unsigned long precharge;
307
	unsigned long moved_charge;
308
	unsigned long moved_swap;
309 310 311
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
312
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
313 314
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
315

D
Daisuke Nishimura 已提交
316 317 318 319 320 321
static bool move_anon(void)
{
	return test_bit(MOVE_CHARGE_TYPE_ANON,
					&mc.to->move_charge_at_immigrate);
}

322 323 324 325 326 327
static bool move_file(void)
{
	return test_bit(MOVE_CHARGE_TYPE_FILE,
					&mc.to->move_charge_at_immigrate);
}

328 329 330 331 332 333 334
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		(100)
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	(2)

335 336 337
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
	MEM_CGROUP_CHARGE_TYPE_MAPPED,
338
	MEM_CGROUP_CHARGE_TYPE_SHMEM,	/* used by page migration of shmem */
339
	MEM_CGROUP_CHARGE_TYPE_FORCE,	/* used by force_empty */
K
KAMEZAWA Hiroyuki 已提交
340
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
341
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
342 343 344
	NR_CHARGE_TYPE,
};

345 346 347
/* for encoding cft->private value on file */
#define _MEM			(0)
#define _MEMSWAP		(1)
K
KAMEZAWA Hiroyuki 已提交
348
#define _OOM_TYPE		(2)
349 350 351
#define MEMFILE_PRIVATE(x, val)	(((x) << 16) | (val))
#define MEMFILE_TYPE(val)	(((val) >> 16) & 0xffff)
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
352 353
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
354

355 356 357 358 359 360 361
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
362 363
#define MEM_CGROUP_RECLAIM_SOFT_BIT	0x2
#define MEM_CGROUP_RECLAIM_SOFT		(1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
364

365 366 367 368
static void mem_cgroup_get(struct mem_cgroup *memcg);
static void mem_cgroup_put(struct mem_cgroup *memcg);
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg);
static void drain_all_stock_async(struct mem_cgroup *memcg);
369

370
static struct mem_cgroup_per_zone *
371
mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
372
{
373
	return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
374 375
}

376
struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
377
{
378
	return &memcg->css;
379 380
}

381
static struct mem_cgroup_per_zone *
382
page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
383
{
384 385
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
386

387
	return mem_cgroup_zoneinfo(memcg, nid, zid);
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
406
__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
407
				struct mem_cgroup_per_zone *mz,
408 409
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
410 411 412 413 414 415 416 417
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

418 419 420
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
437 438 439
}

static void
440
__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
441 442 443 444 445 446 447 448 449
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

450
static void
451
mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
452 453 454 455
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
456
	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
457 458 459 460
	spin_unlock(&mctz->lock);
}


461
static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
462
{
463
	unsigned long long excess;
464 465
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
466 467
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
468 469 470
	mctz = soft_limit_tree_from_page(page);

	/*
471 472
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
473
	 */
474 475 476
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		mz = mem_cgroup_zoneinfo(memcg, nid, zid);
		excess = res_counter_soft_limit_excess(&memcg->res);
477 478 479 480
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
481
		if (excess || mz->on_tree) {
482 483 484
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
485
				__mem_cgroup_remove_exceeded(memcg, mz, mctz);
486
			/*
487 488
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
489
			 */
490
			__mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
491 492
			spin_unlock(&mctz->lock);
		}
493 494 495
	}
}

496
static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
497 498 499 500 501 502 503
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

	for_each_node_state(node, N_POSSIBLE) {
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
504
			mz = mem_cgroup_zoneinfo(memcg, node, zone);
505
			mctz = soft_limit_tree_node_zone(node, zone);
506
			mem_cgroup_remove_exceeded(memcg, mz, mctz);
507 508 509 510
		}
	}
}

511 512 513 514
static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
515
	struct mem_cgroup_per_zone *mz;
516 517

retry:
518
	mz = NULL;
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
	__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->mem->res) ||
		!css_tryget(&mz->mem->css))
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
567
static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
568
				 enum mem_cgroup_stat_index idx)
569
{
570
	long val = 0;
571 572
	int cpu;

573 574
	get_online_cpus();
	for_each_online_cpu(cpu)
575
		val += per_cpu(memcg->stat->count[idx], cpu);
576
#ifdef CONFIG_HOTPLUG_CPU
577 578 579
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.count[idx];
	spin_unlock(&memcg->pcp_counter_lock);
580 581
#endif
	put_online_cpus();
582 583 584
	return val;
}

585
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
586 587 588
					 bool charge)
{
	int val = (charge) ? 1 : -1;
589
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
590 591
}

592
void mem_cgroup_pgfault(struct mem_cgroup *memcg, int val)
593
{
594
	this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT], val);
595 596
}

597
void mem_cgroup_pgmajfault(struct mem_cgroup *memcg, int val)
598
{
599
	this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT], val);
600 601
}

602
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
603 604 605 606 607 608
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

	for_each_online_cpu(cpu)
609
		val += per_cpu(memcg->stat->events[idx], cpu);
610
#ifdef CONFIG_HOTPLUG_CPU
611 612 613
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.events[idx];
	spin_unlock(&memcg->pcp_counter_lock);
614 615 616 617
#endif
	return val;
}

618
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
619
					 bool file, int nr_pages)
620
{
621 622
	preempt_disable();

623
	if (file)
624 625
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
				nr_pages);
626
	else
627 628
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
				nr_pages);
629

630 631
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
632
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
633
	else {
634
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
635 636
		nr_pages = -nr_pages; /* for event */
	}
637

638
	__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
639

640
	preempt_enable();
641 642
}

643
unsigned long
644
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
645
			unsigned int lru_mask)
646 647
{
	struct mem_cgroup_per_zone *mz;
648 649 650
	enum lru_list l;
	unsigned long ret = 0;

651
	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
652 653 654 655 656 657 658 659 660

	for_each_lru(l) {
		if (BIT(l) & lru_mask)
			ret += MEM_CGROUP_ZSTAT(mz, l);
	}
	return ret;
}

static unsigned long
661
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
662 663
			int nid, unsigned int lru_mask)
{
664 665 666
	u64 total = 0;
	int zid;

667
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
668 669
		total += mem_cgroup_zone_nr_lru_pages(memcg,
						nid, zid, lru_mask);
670

671 672
	return total;
}
673

674
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
675
			unsigned int lru_mask)
676
{
677
	int nid;
678 679
	u64 total = 0;

680
	for_each_node_state(nid, N_HIGH_MEMORY)
681
		total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
682
	return total;
683 684
}

685
static bool __memcg_event_check(struct mem_cgroup *memcg, int target)
686 687 688
{
	unsigned long val, next;

689 690
	val = this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]);
	next = this_cpu_read(memcg->stat->targets[target]);
691 692 693 694
	/* from time_after() in jiffies.h */
	return ((long)next - (long)val < 0);
}

695
static void __mem_cgroup_target_update(struct mem_cgroup *memcg, int target)
696
{
697
	unsigned long val, next;
698

699
	val = this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]);
700

701 702 703 704 705 706 707
	switch (target) {
	case MEM_CGROUP_TARGET_THRESH:
		next = val + THRESHOLDS_EVENTS_TARGET;
		break;
	case MEM_CGROUP_TARGET_SOFTLIMIT:
		next = val + SOFTLIMIT_EVENTS_TARGET;
		break;
708 709 710
	case MEM_CGROUP_TARGET_NUMAINFO:
		next = val + NUMAINFO_EVENTS_TARGET;
		break;
711 712 713 714
	default:
		return;
	}

715
	this_cpu_write(memcg->stat->targets[target], next);
716 717 718 719 720 721
}

/*
 * Check events in order.
 *
 */
722
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
723 724
{
	/* threshold event is triggered in finer grain than soft limit */
725 726 727 728
	if (unlikely(__memcg_event_check(memcg, MEM_CGROUP_TARGET_THRESH))) {
		mem_cgroup_threshold(memcg);
		__mem_cgroup_target_update(memcg, MEM_CGROUP_TARGET_THRESH);
		if (unlikely(__memcg_event_check(memcg,
729
			     MEM_CGROUP_TARGET_SOFTLIMIT))) {
730 731
			mem_cgroup_update_tree(memcg, page);
			__mem_cgroup_target_update(memcg,
732 733 734
						   MEM_CGROUP_TARGET_SOFTLIMIT);
		}
#if MAX_NUMNODES > 1
735
		if (unlikely(__memcg_event_check(memcg,
736
			MEM_CGROUP_TARGET_NUMAINFO))) {
737 738
			atomic_inc(&memcg->numainfo_events);
			__mem_cgroup_target_update(memcg,
739
				MEM_CGROUP_TARGET_NUMAINFO);
740
		}
741
#endif
742 743 744
	}
}

745
static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
746 747 748 749 750 751
{
	return container_of(cgroup_subsys_state(cont,
				mem_cgroup_subsys_id), struct mem_cgroup,
				css);
}

752
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
753
{
754 755 756 757 758 759 760 761
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

762 763 764 765
	return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
				struct mem_cgroup, css);
}

766
struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
767
{
768
	struct mem_cgroup *memcg = NULL;
769 770 771

	if (!mm)
		return NULL;
772 773 774 775 776 777 778
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
779 780
		memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!memcg))
781
			break;
782
	} while (!css_tryget(&memcg->css));
783
	rcu_read_unlock();
784
	return memcg;
785 786
}

K
KAMEZAWA Hiroyuki 已提交
787
/* The caller has to guarantee "mem" exists before calling this */
788
static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
789
{
790 791 792
	struct cgroup_subsys_state *css;
	int found;

793
	if (!memcg) /* ROOT cgroup has the smallest ID */
794
		return root_mem_cgroup; /*css_put/get against root is ignored*/
795 796 797
	if (!memcg->use_hierarchy) {
		if (css_tryget(&memcg->css))
			return memcg;
798 799 800 801 802 803 804
		return NULL;
	}
	rcu_read_lock();
	/*
	 * searching a memory cgroup which has the smallest ID under given
	 * ROOT cgroup. (ID >= 1)
	 */
805
	css = css_get_next(&mem_cgroup_subsys, 1, &memcg->css, &found);
806
	if (css && css_tryget(css))
807
		memcg = container_of(css, struct mem_cgroup, css);
808
	else
809
		memcg = NULL;
810
	rcu_read_unlock();
811
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
812 813 814 815 816 817 818 819 820
}

static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
					struct mem_cgroup *root,
					bool cond)
{
	int nextid = css_id(&iter->css) + 1;
	int found;
	int hierarchy_used;
K
KAMEZAWA Hiroyuki 已提交
821 822
	struct cgroup_subsys_state *css;

K
KAMEZAWA Hiroyuki 已提交
823
	hierarchy_used = iter->use_hierarchy;
K
KAMEZAWA Hiroyuki 已提交
824

K
KAMEZAWA Hiroyuki 已提交
825
	css_put(&iter->css);
826 827
	/* If no ROOT, walk all, ignore hierarchy */
	if (!cond || (root && !hierarchy_used))
K
KAMEZAWA Hiroyuki 已提交
828
		return NULL;
K
KAMEZAWA Hiroyuki 已提交
829

830 831 832
	if (!root)
		root = root_mem_cgroup;

K
KAMEZAWA Hiroyuki 已提交
833 834
	do {
		iter = NULL;
K
KAMEZAWA Hiroyuki 已提交
835
		rcu_read_lock();
K
KAMEZAWA Hiroyuki 已提交
836 837 838

		css = css_get_next(&mem_cgroup_subsys, nextid,
				&root->css, &found);
K
KAMEZAWA Hiroyuki 已提交
839
		if (css && css_tryget(css))
K
KAMEZAWA Hiroyuki 已提交
840
			iter = container_of(css, struct mem_cgroup, css);
K
KAMEZAWA Hiroyuki 已提交
841
		rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
842
		/* If css is NULL, no more cgroups will be found */
K
KAMEZAWA Hiroyuki 已提交
843
		nextid = found + 1;
K
KAMEZAWA Hiroyuki 已提交
844
	} while (css && !iter);
K
KAMEZAWA Hiroyuki 已提交
845

K
KAMEZAWA Hiroyuki 已提交
846
	return iter;
K
KAMEZAWA Hiroyuki 已提交
847
}
K
KAMEZAWA Hiroyuki 已提交
848 849 850 851 852 853 854 855 856 857 858 859 860
/*
 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
 * be careful that "break" loop is not allowed. We have reference count.
 * Instead of that modify "cond" to be false and "continue" to exit the loop.
 */
#define for_each_mem_cgroup_tree_cond(iter, root, cond)	\
	for (iter = mem_cgroup_start_loop(root);\
	     iter != NULL;\
	     iter = mem_cgroup_get_next(iter, root, cond))

#define for_each_mem_cgroup_tree(iter, root) \
	for_each_mem_cgroup_tree_cond(iter, root, true)

861 862 863
#define for_each_mem_cgroup_all(iter) \
	for_each_mem_cgroup_tree_cond(iter, NULL, true)

K
KAMEZAWA Hiroyuki 已提交
864

865
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
866
{
867
	return (memcg == root_mem_cgroup);
868 869
}

870 871
void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
{
872
	struct mem_cgroup *memcg;
873 874 875 876 877

	if (!mm)
		return;

	rcu_read_lock();
878 879
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
880 881 882 883
		goto out;

	switch (idx) {
	case PGMAJFAULT:
884
		mem_cgroup_pgmajfault(memcg, 1);
885 886
		break;
	case PGFAULT:
887
		mem_cgroup_pgfault(memcg, 1);
888 889 890 891 892 893 894 895 896
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
}
EXPORT_SYMBOL(mem_cgroup_count_vm_event);

K
KAMEZAWA Hiroyuki 已提交
897 898 899 900 901 902 903 904 905 906 907 908 909
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
910

K
KAMEZAWA Hiroyuki 已提交
911 912 913 914
void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
915

916
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
917 918 919
		return;
	pc = lookup_page_cgroup(page);
	/* can happen while we handle swapcache. */
920
	if (!TestClearPageCgroupAcctLRU(pc))
K
KAMEZAWA Hiroyuki 已提交
921
		return;
922
	VM_BUG_ON(!pc->mem_cgroup);
923 924 925 926
	/*
	 * We don't check PCG_USED bit. It's cleared when the "page" is finally
	 * removed from global LRU.
	 */
927
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
928 929
	/* huge page split is done under lru_lock. so, we have no races. */
	MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
930 931 932
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
	VM_BUG_ON(list_empty(&pc->lru));
K
KAMEZAWA Hiroyuki 已提交
933
	list_del_init(&pc->lru);
934 935
}

K
KAMEZAWA Hiroyuki 已提交
936
void mem_cgroup_del_lru(struct page *page)
937
{
K
KAMEZAWA Hiroyuki 已提交
938 939
	mem_cgroup_del_lru_list(page, page_lru(page));
}
940

941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
/*
 * Writeback is about to end against a page which has been marked for immediate
 * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 * inactive list.
 */
void mem_cgroup_rotate_reclaimable_page(struct page *page)
{
	struct mem_cgroup_per_zone *mz;
	struct page_cgroup *pc;
	enum lru_list lru = page_lru(page);

	if (mem_cgroup_disabled())
		return;

	pc = lookup_page_cgroup(page);
	/* unused or root page is not rotated. */
	if (!PageCgroupUsed(pc))
		return;
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
963
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
964 965 966
	list_move_tail(&pc->lru, &mz->lists[lru]);
}

K
KAMEZAWA Hiroyuki 已提交
967 968 969 970
void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;
	struct page_cgroup *pc;
971

972
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
973
		return;
974

K
KAMEZAWA Hiroyuki 已提交
975
	pc = lookup_page_cgroup(page);
976
	/* unused or root page is not rotated. */
977 978 979 980 981
	if (!PageCgroupUsed(pc))
		return;
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
	if (mem_cgroup_is_root(pc->mem_cgroup))
K
KAMEZAWA Hiroyuki 已提交
982
		return;
983
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
K
KAMEZAWA Hiroyuki 已提交
984
	list_move(&pc->lru, &mz->lists[lru]);
985 986
}

K
KAMEZAWA Hiroyuki 已提交
987
void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
988
{
K
KAMEZAWA Hiroyuki 已提交
989 990
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
991

992
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
993 994
		return;
	pc = lookup_page_cgroup(page);
995
	VM_BUG_ON(PageCgroupAcctLRU(pc));
996 997 998 999 1000 1001 1002 1003 1004 1005
	/*
	 * putback:				charge:
	 * SetPageLRU				SetPageCgroupUsed
	 * smp_mb				smp_mb
	 * PageCgroupUsed && add to memcg LRU	PageLRU && add to memcg LRU
	 *
	 * Ensure that one of the two sides adds the page to the memcg
	 * LRU during a race.
	 */
	smp_mb();
K
KAMEZAWA Hiroyuki 已提交
1006
	if (!PageCgroupUsed(pc))
L
Lee Schermerhorn 已提交
1007
		return;
1008 1009
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
1010
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
1011 1012
	/* huge page split is done under lru_lock. so, we have no races. */
	MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
1013 1014 1015
	SetPageCgroupAcctLRU(pc);
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
K
KAMEZAWA Hiroyuki 已提交
1016 1017
	list_add(&pc->lru, &mz->lists[lru]);
}
1018

K
KAMEZAWA Hiroyuki 已提交
1019
/*
1020 1021 1022 1023
 * At handling SwapCache and other FUSE stuff, pc->mem_cgroup may be changed
 * while it's linked to lru because the page may be reused after it's fully
 * uncharged. To handle that, unlink page_cgroup from LRU when charge it again.
 * It's done under lock_page and expected that zone->lru_lock isnever held.
K
KAMEZAWA Hiroyuki 已提交
1024
 */
1025
static void mem_cgroup_lru_del_before_commit(struct page *page)
K
KAMEZAWA Hiroyuki 已提交
1026
{
1027 1028 1029 1030
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);

1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
	/*
	 * Doing this check without taking ->lru_lock seems wrong but this
	 * is safe. Because if page_cgroup's USED bit is unset, the page
	 * will not be added to any memcg's LRU. If page_cgroup's USED bit is
	 * set, the commit after this will fail, anyway.
	 * This all charge/uncharge is done under some mutual execustion.
	 * So, we don't need to taking care of changes in USED bit.
	 */
	if (likely(!PageLRU(page)))
		return;

1042 1043 1044 1045 1046 1047 1048 1049
	spin_lock_irqsave(&zone->lru_lock, flags);
	/*
	 * Forget old LRU when this page_cgroup is *not* used. This Used bit
	 * is guarded by lock_page() because the page is SwapCache.
	 */
	if (!PageCgroupUsed(pc))
		mem_cgroup_del_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
K
KAMEZAWA Hiroyuki 已提交
1050 1051
}

1052
static void mem_cgroup_lru_add_after_commit(struct page *page)
1053 1054 1055 1056
{
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
	/*
	 * putback:				charge:
	 * SetPageLRU				SetPageCgroupUsed
	 * smp_mb				smp_mb
	 * PageCgroupUsed && add to memcg LRU	PageLRU && add to memcg LRU
	 *
	 * Ensure that one of the two sides adds the page to the memcg
	 * LRU during a race.
	 */
	smp_mb();
1067 1068 1069
	/* taking care of that the page is added to LRU while we commit it */
	if (likely(!PageLRU(page)))
		return;
1070 1071
	spin_lock_irqsave(&zone->lru_lock, flags);
	/* link when the page is linked to LRU but page_cgroup isn't */
1072
	if (PageLRU(page) && !PageCgroupAcctLRU(pc))
1073 1074 1075 1076 1077
		mem_cgroup_add_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
}


K
KAMEZAWA Hiroyuki 已提交
1078 1079 1080
void mem_cgroup_move_lists(struct page *page,
			   enum lru_list from, enum lru_list to)
{
1081
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
1082 1083 1084
		return;
	mem_cgroup_del_lru_list(page, from);
	mem_cgroup_add_lru_list(page, to);
1085 1086
}

1087
/*
1088
 * Checks whether given mem is same or in the root_mem_cgroup's
1089 1090
 * hierarchy subtree
 */
1091 1092
static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
		struct mem_cgroup *memcg)
1093
{
1094 1095 1096
	if (root_memcg != memcg) {
		return (root_memcg->use_hierarchy &&
			css_is_ancestor(&memcg->css, &root_memcg->css));
1097 1098 1099 1100 1101
	}

	return true;
}

1102
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
1103 1104
{
	int ret;
1105
	struct mem_cgroup *curr = NULL;
1106
	struct task_struct *p;
1107

1108 1109 1110 1111 1112
	p = find_lock_task_mm(task);
	if (!p)
		return 0;
	curr = try_get_mem_cgroup_from_mm(p->mm);
	task_unlock(p);
1113 1114
	if (!curr)
		return 0;
1115
	/*
1116
	 * We should check use_hierarchy of "memcg" not "curr". Because checking
1117
	 * use_hierarchy of "curr" here make this function true if hierarchy is
1118 1119
	 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
	 * hierarchy(even if use_hierarchy is disabled in "memcg").
1120
	 */
1121
	ret = mem_cgroup_same_or_subtree(memcg, curr);
1122
	css_put(&curr->css);
1123 1124 1125
	return ret;
}

1126
int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
1127
{
1128 1129 1130
	unsigned long inactive_ratio;
	int nid = zone_to_nid(zone);
	int zid = zone_idx(zone);
1131
	unsigned long inactive;
1132
	unsigned long active;
1133
	unsigned long gb;
1134

1135 1136 1137 1138
	inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
						BIT(LRU_INACTIVE_ANON));
	active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
					      BIT(LRU_ACTIVE_ANON));
1139

1140 1141 1142 1143 1144 1145
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

1146
	return inactive * inactive_ratio < active;
1147 1148
}

1149
int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
1150 1151 1152
{
	unsigned long active;
	unsigned long inactive;
1153 1154
	int zid = zone_idx(zone);
	int nid = zone_to_nid(zone);
1155

1156 1157 1158 1159
	inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
						BIT(LRU_INACTIVE_FILE));
	active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
					      BIT(LRU_ACTIVE_FILE));
1160 1161 1162 1163

	return (active > inactive);
}

K
KOSAKI Motohiro 已提交
1164 1165 1166
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
						      struct zone *zone)
{
1167
	int nid = zone_to_nid(zone);
K
KOSAKI Motohiro 已提交
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

	return &mz->reclaim_stat;
}

struct zone_reclaim_stat *
mem_cgroup_get_reclaim_stat_from_page(struct page *page)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;

	if (mem_cgroup_disabled())
		return NULL;

	pc = lookup_page_cgroup(page);
1184 1185
	if (!PageCgroupUsed(pc))
		return NULL;
1186 1187
	/* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
	smp_rmb();
1188
	mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
K
KOSAKI Motohiro 已提交
1189 1190 1191
	return &mz->reclaim_stat;
}

1192 1193 1194
unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
					struct list_head *dst,
					unsigned long *scanned, int order,
1195 1196
					isolate_mode_t mode,
					struct zone *z,
1197
					struct mem_cgroup *mem_cont,
1198
					int active, int file)
1199 1200 1201 1202 1203 1204
{
	unsigned long nr_taken = 0;
	struct page *page;
	unsigned long scan;
	LIST_HEAD(pc_list);
	struct list_head *src;
1205
	struct page_cgroup *pc, *tmp;
1206
	int nid = zone_to_nid(z);
1207 1208
	int zid = zone_idx(z);
	struct mem_cgroup_per_zone *mz;
1209
	int lru = LRU_FILE * file + active;
1210
	int ret;
1211

1212
	BUG_ON(!mem_cont);
1213
	mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
1214
	src = &mz->lists[lru];
1215

1216 1217
	scan = 0;
	list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
H
Hugh Dickins 已提交
1218
		if (scan >= nr_to_scan)
1219
			break;
K
KAMEZAWA Hiroyuki 已提交
1220

1221 1222
		if (unlikely(!PageCgroupUsed(pc)))
			continue;
1223

1224
		page = lookup_cgroup_page(pc);
1225

H
Hugh Dickins 已提交
1226
		if (unlikely(!PageLRU(page)))
1227 1228
			continue;

H
Hugh Dickins 已提交
1229
		scan++;
1230 1231 1232
		ret = __isolate_lru_page(page, mode, file);
		switch (ret) {
		case 0:
1233
			list_move(&page->lru, dst);
1234
			mem_cgroup_del_lru(page);
1235
			nr_taken += hpage_nr_pages(page);
1236 1237 1238 1239 1240 1241 1242
			break;
		case -EBUSY:
			/* we don't affect global LRU but rotate in our LRU */
			mem_cgroup_rotate_lru_list(page, page_lru(page));
			break;
		default:
			break;
1243 1244 1245 1246
		}
	}

	*scanned = scan;
1247 1248 1249 1250

	trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
				      0, 0, 0, mode);

1251 1252 1253
	return nr_taken;
}

1254 1255 1256
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

1257
/**
1258 1259
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
 * @mem: the memory cgroup
1260
 *
1261
 * Returns the maximum amount of memory @mem can be charged with, in
1262
 * pages.
1263
 */
1264
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1265
{
1266 1267
	unsigned long long margin;

1268
	margin = res_counter_margin(&memcg->res);
1269
	if (do_swap_account)
1270
		margin = min(margin, res_counter_margin(&memcg->memsw));
1271
	return margin >> PAGE_SHIFT;
1272 1273
}

1274
int mem_cgroup_swappiness(struct mem_cgroup *memcg)
K
KOSAKI Motohiro 已提交
1275 1276 1277 1278 1279 1280 1281
{
	struct cgroup *cgrp = memcg->css.cgroup;

	/* root ? */
	if (cgrp->parent == NULL)
		return vm_swappiness;

1282
	return memcg->swappiness;
K
KOSAKI Motohiro 已提交
1283 1284
}

1285
static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1286 1287
{
	int cpu;
1288 1289

	get_online_cpus();
1290
	spin_lock(&memcg->pcp_counter_lock);
1291
	for_each_online_cpu(cpu)
1292 1293 1294
		per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
	memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
	spin_unlock(&memcg->pcp_counter_lock);
1295
	put_online_cpus();
1296 1297 1298 1299

	synchronize_rcu();
}

1300
static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1301 1302 1303
{
	int cpu;

1304
	if (!memcg)
1305
		return;
1306
	get_online_cpus();
1307
	spin_lock(&memcg->pcp_counter_lock);
1308
	for_each_online_cpu(cpu)
1309 1310 1311
		per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
	memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
	spin_unlock(&memcg->pcp_counter_lock);
1312
	put_online_cpus();
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
}
/*
 * 2 routines for checking "mem" is under move_account() or not.
 *
 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
 *			  for avoiding race in accounting. If true,
 *			  pc->mem_cgroup may be overwritten.
 *
 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
 *			  under hierarchy of moving cgroups. This is for
 *			  waiting at hith-memory prressure caused by "move".
 */

1326
static bool mem_cgroup_stealed(struct mem_cgroup *memcg)
1327 1328
{
	VM_BUG_ON(!rcu_read_lock_held());
1329
	return this_cpu_read(memcg->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1330
}
1331

1332
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1333
{
1334 1335
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1336
	bool ret = false;
1337 1338 1339 1340 1341 1342 1343 1344 1345
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1346

1347 1348
	ret = mem_cgroup_same_or_subtree(memcg, from)
		|| mem_cgroup_same_or_subtree(memcg, to);
1349 1350
unlock:
	spin_unlock(&mc.lock);
1351 1352 1353
	return ret;
}

1354
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1355 1356
{
	if (mc.moving_task && current != mc.moving_task) {
1357
		if (mem_cgroup_under_move(memcg)) {
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1370
/**
1371
 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;

1390
	if (!memcg || !p)
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
		return;


	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	printk(KERN_INFO "Task in %s killed", memcg_name);

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
	printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
done:

	printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
	printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
		"failcnt %llu\n",
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
}

1437 1438 1439 1440
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1441
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1442 1443
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1444 1445
	struct mem_cgroup *iter;

1446
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1447
		num++;
1448 1449 1450
	return num;
}

D
David Rientjes 已提交
1451 1452 1453 1454 1455 1456 1457 1458
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
{
	u64 limit;
	u64 memsw;

1459 1460 1461
	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	limit += total_swap_pages << PAGE_SHIFT;

D
David Rientjes 已提交
1462 1463 1464 1465 1466 1467 1468 1469
	memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	/*
	 * If memsw is finite and limits the amount of swap space available
	 * to this memcg, return that limit.
	 */
	return min(limit, memsw);
}

1470
/*
K
KAMEZAWA Hiroyuki 已提交
1471 1472 1473 1474 1475
 * Visit the first child (need not be the first child as per the ordering
 * of the cgroup list, since we track last_scanned_child) of @mem and use
 * that to reclaim free pages from.
 */
static struct mem_cgroup *
1476
mem_cgroup_select_victim(struct mem_cgroup *root_memcg)
K
KAMEZAWA Hiroyuki 已提交
1477 1478 1479 1480 1481
{
	struct mem_cgroup *ret = NULL;
	struct cgroup_subsys_state *css;
	int nextid, found;

1482 1483 1484
	if (!root_memcg->use_hierarchy) {
		css_get(&root_memcg->css);
		ret = root_memcg;
K
KAMEZAWA Hiroyuki 已提交
1485 1486 1487 1488
	}

	while (!ret) {
		rcu_read_lock();
1489 1490
		nextid = root_memcg->last_scanned_child + 1;
		css = css_get_next(&mem_cgroup_subsys, nextid, &root_memcg->css,
K
KAMEZAWA Hiroyuki 已提交
1491 1492 1493 1494 1495 1496 1497 1498
				   &found);
		if (css && css_tryget(css))
			ret = container_of(css, struct mem_cgroup, css);

		rcu_read_unlock();
		/* Updates scanning parameter */
		if (!css) {
			/* this means start scan from ID:1 */
1499
			root_memcg->last_scanned_child = 0;
K
KAMEZAWA Hiroyuki 已提交
1500
		} else
1501
			root_memcg->last_scanned_child = found;
K
KAMEZAWA Hiroyuki 已提交
1502 1503 1504 1505 1506
	}

	return ret;
}

1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
/**
 * test_mem_cgroup_node_reclaimable
 * @mem: the target memcg
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1517
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1518 1519
		int nid, bool noswap)
{
1520
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1521 1522 1523
		return true;
	if (noswap || !total_swap_pages)
		return false;
1524
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1525 1526 1527 1528
		return true;
	return false;

}
1529 1530 1531 1532 1533 1534 1535 1536
#if MAX_NUMNODES > 1

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1537
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1538 1539
{
	int nid;
1540 1541 1542 1543
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1544
	if (!atomic_read(&memcg->numainfo_events))
1545
		return;
1546
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1547 1548 1549
		return;

	/* make a nodemask where this memcg uses memory from */
1550
	memcg->scan_nodes = node_states[N_HIGH_MEMORY];
1551 1552 1553

	for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {

1554 1555
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1556
	}
1557

1558 1559
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1574
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1575 1576 1577
{
	int node;

1578 1579
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1580

1581
	node = next_node(node, memcg->scan_nodes);
1582
	if (node == MAX_NUMNODES)
1583
		node = first_node(memcg->scan_nodes);
1584 1585 1586 1587 1588 1589 1590 1591 1592
	/*
	 * We call this when we hit limit, not when pages are added to LRU.
	 * No LRU may hold pages because all pages are UNEVICTABLE or
	 * memcg is too small and all pages are not on LRU. In that case,
	 * we use curret node.
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1593
	memcg->last_scanned_node = node;
1594 1595 1596
	return node;
}

1597 1598 1599 1600 1601 1602
/*
 * Check all nodes whether it contains reclaimable pages or not.
 * For quick scan, we make use of scan_nodes. This will allow us to skip
 * unused nodes. But scan_nodes is lazily updated and may not cotain
 * enough new information. We need to do double check.
 */
1603
bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1604 1605 1606 1607 1608 1609 1610
{
	int nid;

	/*
	 * quick check...making use of scan_node.
	 * We can skip unused nodes.
	 */
1611 1612
	if (!nodes_empty(memcg->scan_nodes)) {
		for (nid = first_node(memcg->scan_nodes);
1613
		     nid < MAX_NUMNODES;
1614
		     nid = next_node(nid, memcg->scan_nodes)) {
1615

1616
			if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1617 1618 1619 1620 1621 1622 1623
				return true;
		}
	}
	/*
	 * Check rest of nodes.
	 */
	for_each_node_state(nid, N_HIGH_MEMORY) {
1624
		if (node_isset(nid, memcg->scan_nodes))
1625
			continue;
1626
		if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1627 1628 1629 1630 1631
			return true;
	}
	return false;
}

1632
#else
1633
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1634 1635 1636
{
	return 0;
}
1637

1638
bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1639
{
1640
	return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1641
}
1642 1643
#endif

K
KAMEZAWA Hiroyuki 已提交
1644 1645 1646 1647
/*
 * Scan the hierarchy if needed to reclaim memory. We remember the last child
 * we reclaimed from, so that we don't end up penalizing one child extensively
 * based on its position in the children list.
1648
 *
1649
 * root_memcg is the original ancestor that we've been reclaim from.
K
KAMEZAWA Hiroyuki 已提交
1650
 *
1651
 * We give up and return to the caller when we visit root_memcg twice.
K
KAMEZAWA Hiroyuki 已提交
1652
 * (other groups can be removed while we're walking....)
1653 1654
 *
 * If shrink==true, for avoiding to free too much, this returns immedieately.
1655
 */
1656
static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg,
1657
						struct zone *zone,
1658
						gfp_t gfp_mask,
1659 1660
						unsigned long reclaim_options,
						unsigned long *total_scanned)
1661
{
K
KAMEZAWA Hiroyuki 已提交
1662 1663 1664
	struct mem_cgroup *victim;
	int ret, total = 0;
	int loop = 0;
1665 1666
	bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
	bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1667
	bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1668
	unsigned long excess;
1669
	unsigned long nr_scanned;
1670

1671
	excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
K
KAMEZAWA Hiroyuki 已提交
1672

1673
	/* If memsw_is_minimum==1, swap-out is of-no-use. */
1674
	if (!check_soft && !shrink && root_memcg->memsw_is_minimum)
1675 1676
		noswap = true;

1677
	while (1) {
1678 1679
		victim = mem_cgroup_select_victim(root_memcg);
		if (victim == root_memcg) {
K
KAMEZAWA Hiroyuki 已提交
1680
			loop++;
1681 1682 1683 1684 1685 1686 1687
			/*
			 * We are not draining per cpu cached charges during
			 * soft limit reclaim  because global reclaim doesn't
			 * care about charges. It tries to free some memory and
			 * charges will not give any.
			 */
			if (!check_soft && loop >= 1)
1688
				drain_all_stock_async(root_memcg);
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!check_soft || !total) {
					css_put(&victim->css);
					break;
				}
				/*
L
Lucas De Marchi 已提交
1700
				 * We want to do more targeted reclaim.
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
					css_put(&victim->css);
					break;
				}
			}
		}
1712
		if (!mem_cgroup_reclaimable(victim, noswap)) {
K
KAMEZAWA Hiroyuki 已提交
1713 1714
			/* this cgroup's local usage == 0 */
			css_put(&victim->css);
1715 1716
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
1717
		/* we use swappiness of local cgroup */
1718
		if (check_soft) {
1719
			ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
1720 1721
				noswap, zone, &nr_scanned);
			*total_scanned += nr_scanned;
1722
		} else
1723
			ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1724
						noswap);
K
KAMEZAWA Hiroyuki 已提交
1725
		css_put(&victim->css);
1726 1727 1728 1729 1730 1731 1732
		/*
		 * At shrinking usage, we can't check we should stop here or
		 * reclaim more. It's depends on callers. last_scanned_child
		 * will work enough for keeping fairness under tree.
		 */
		if (shrink)
			return ret;
K
KAMEZAWA Hiroyuki 已提交
1733
		total += ret;
1734
		if (check_soft) {
1735
			if (!res_counter_soft_limit_excess(&root_memcg->res))
1736
				return total;
1737
		} else if (mem_cgroup_margin(root_memcg))
1738
			return total;
1739
	}
K
KAMEZAWA Hiroyuki 已提交
1740
	return total;
1741 1742
}

K
KAMEZAWA Hiroyuki 已提交
1743 1744 1745
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
1746
 * Has to be called with memcg_oom_lock
K
KAMEZAWA Hiroyuki 已提交
1747
 */
1748
static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1749
{
1750 1751
	struct mem_cgroup *iter, *failed = NULL;
	bool cond = true;
1752

1753
	for_each_mem_cgroup_tree_cond(iter, memcg, cond) {
1754
		if (iter->oom_lock) {
1755 1756 1757 1758 1759 1760
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
			cond = false;
1761 1762
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1763
	}
K
KAMEZAWA Hiroyuki 已提交
1764

1765
	if (!failed)
1766
		return true;
1767 1768 1769 1770 1771 1772

	/*
	 * OK, we failed to lock the whole subtree so we have to clean up
	 * what we set up to the failing subtree
	 */
	cond = true;
1773
	for_each_mem_cgroup_tree_cond(iter, memcg, cond) {
1774 1775 1776 1777 1778 1779
		if (iter == failed) {
			cond = false;
			continue;
		}
		iter->oom_lock = false;
	}
1780
	return false;
1781
}
1782

1783
/*
1784
 * Has to be called with memcg_oom_lock
1785
 */
1786
static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1787
{
K
KAMEZAWA Hiroyuki 已提交
1788 1789
	struct mem_cgroup *iter;

1790
	for_each_mem_cgroup_tree(iter, memcg)
1791 1792 1793 1794
		iter->oom_lock = false;
	return 0;
}

1795
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1796 1797 1798
{
	struct mem_cgroup *iter;

1799
	for_each_mem_cgroup_tree(iter, memcg)
1800 1801 1802
		atomic_inc(&iter->under_oom);
}

1803
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1804 1805 1806
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1807 1808 1809 1810 1811
	/*
	 * When a new child is created while the hierarchy is under oom,
	 * mem_cgroup_oom_lock() may not be called. We have to use
	 * atomic_add_unless() here.
	 */
1812
	for_each_mem_cgroup_tree(iter, memcg)
1813
		atomic_add_unless(&iter->under_oom, -1, 0);
1814 1815
}

1816
static DEFINE_SPINLOCK(memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1817 1818
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1819 1820 1821 1822 1823 1824 1825 1826
struct oom_wait_info {
	struct mem_cgroup *mem;
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
1827 1828
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg,
			  *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1829 1830 1831
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1832
	oom_wait_memcg = oom_wait_info->mem;
K
KAMEZAWA Hiroyuki 已提交
1833 1834 1835 1836 1837

	/*
	 * Both of oom_wait_info->mem and wake_mem are stable under us.
	 * Then we can use css_is_ancestor without taking care of RCU.
	 */
1838 1839
	if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
		&& !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
K
KAMEZAWA Hiroyuki 已提交
1840 1841 1842 1843
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1844
static void memcg_wakeup_oom(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1845
{
1846 1847
	/* for filtering, pass "memcg" as argument. */
	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
K
KAMEZAWA Hiroyuki 已提交
1848 1849
}

1850
static void memcg_oom_recover(struct mem_cgroup *memcg)
1851
{
1852 1853
	if (memcg && atomic_read(&memcg->under_oom))
		memcg_wakeup_oom(memcg);
1854 1855
}

K
KAMEZAWA Hiroyuki 已提交
1856 1857 1858
/*
 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
 */
1859
bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask)
1860
{
K
KAMEZAWA Hiroyuki 已提交
1861
	struct oom_wait_info owait;
1862
	bool locked, need_to_kill;
K
KAMEZAWA Hiroyuki 已提交
1863

1864
	owait.mem = memcg;
K
KAMEZAWA Hiroyuki 已提交
1865 1866 1867 1868
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
1869
	need_to_kill = true;
1870
	mem_cgroup_mark_under_oom(memcg);
1871

1872
	/* At first, try to OOM lock hierarchy under memcg.*/
1873
	spin_lock(&memcg_oom_lock);
1874
	locked = mem_cgroup_oom_lock(memcg);
K
KAMEZAWA Hiroyuki 已提交
1875 1876 1877 1878 1879
	/*
	 * Even if signal_pending(), we can't quit charge() loop without
	 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
	 * under OOM is always welcomed, use TASK_KILLABLE here.
	 */
1880
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1881
	if (!locked || memcg->oom_kill_disable)
1882 1883
		need_to_kill = false;
	if (locked)
1884
		mem_cgroup_oom_notify(memcg);
1885
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1886

1887 1888
	if (need_to_kill) {
		finish_wait(&memcg_oom_waitq, &owait.wait);
1889
		mem_cgroup_out_of_memory(memcg, mask);
1890
	} else {
K
KAMEZAWA Hiroyuki 已提交
1891
		schedule();
K
KAMEZAWA Hiroyuki 已提交
1892
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
1893
	}
1894
	spin_lock(&memcg_oom_lock);
1895
	if (locked)
1896 1897
		mem_cgroup_oom_unlock(memcg);
	memcg_wakeup_oom(memcg);
1898
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1899

1900
	mem_cgroup_unmark_under_oom(memcg);
1901

K
KAMEZAWA Hiroyuki 已提交
1902 1903 1904
	if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
		return false;
	/* Give chance to dying process */
1905
	schedule_timeout_uninterruptible(1);
K
KAMEZAWA Hiroyuki 已提交
1906
	return true;
1907 1908
}

1909 1910 1911
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
 *
 * Notes: Race condition
 *
 * We usually use page_cgroup_lock() for accessing page_cgroup member but
 * it tends to be costly. But considering some conditions, we doesn't need
 * to do so _always_.
 *
 * Considering "charge", lock_page_cgroup() is not required because all
 * file-stat operations happen after a page is attached to radix-tree. There
 * are no race with "charge".
 *
 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
 * if there are race with "uncharge". Statistics itself is properly handled
 * by flags.
 *
 * Considering "move", this is an only case we see a race. To make the race
 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
 * possibility of race condition. If there is, we take a lock.
1931
 */
1932

1933 1934
void mem_cgroup_update_page_stat(struct page *page,
				 enum mem_cgroup_page_stat_item idx, int val)
1935
{
1936
	struct mem_cgroup *memcg;
1937 1938
	struct page_cgroup *pc = lookup_page_cgroup(page);
	bool need_unlock = false;
1939
	unsigned long uninitialized_var(flags);
1940 1941 1942 1943

	if (unlikely(!pc))
		return;

1944
	rcu_read_lock();
1945 1946
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
1947 1948
		goto out;
	/* pc->mem_cgroup is unstable ? */
1949
	if (unlikely(mem_cgroup_stealed(memcg)) || PageTransHuge(page)) {
1950
		/* take a lock against to access pc->mem_cgroup */
1951
		move_lock_page_cgroup(pc, &flags);
1952
		need_unlock = true;
1953 1954
		memcg = pc->mem_cgroup;
		if (!memcg || !PageCgroupUsed(pc))
1955 1956
			goto out;
	}
1957 1958

	switch (idx) {
1959
	case MEMCG_NR_FILE_MAPPED:
1960 1961 1962
		if (val > 0)
			SetPageCgroupFileMapped(pc);
		else if (!page_mapped(page))
1963
			ClearPageCgroupFileMapped(pc);
1964
		idx = MEM_CGROUP_STAT_FILE_MAPPED;
1965 1966 1967
		break;
	default:
		BUG();
1968
	}
1969

1970
	this_cpu_add(memcg->stat->count[idx], val);
1971

1972 1973
out:
	if (unlikely(need_unlock))
1974
		move_unlock_page_cgroup(pc, &flags);
1975 1976
	rcu_read_unlock();
	return;
1977
}
1978
EXPORT_SYMBOL(mem_cgroup_update_page_stat);
1979

1980 1981 1982 1983
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
1984
#define CHARGE_BATCH	32U
1985 1986
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
1987
	unsigned int nr_pages;
1988
	struct work_struct work;
1989 1990
	unsigned long flags;
#define FLUSHING_CACHED_CHARGE	(0)
1991 1992
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1993
static DEFINE_MUTEX(percpu_charge_mutex);
1994 1995

/*
1996
 * Try to consume stocked charge on this cpu. If success, one page is consumed
1997 1998 1999 2000
 * from local stock and true is returned. If the stock is 0 or charges from a
 * cgroup which is not current target, returns false. This stock will be
 * refilled.
 */
2001
static bool consume_stock(struct mem_cgroup *memcg)
2002 2003 2004 2005 2006
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

	stock = &get_cpu_var(memcg_stock);
2007
	if (memcg == stock->cached && stock->nr_pages)
2008
		stock->nr_pages--;
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2022 2023 2024 2025
	if (stock->nr_pages) {
		unsigned long bytes = stock->nr_pages * PAGE_SIZE;

		res_counter_uncharge(&old->res, bytes);
2026
		if (do_swap_account)
2027 2028
			res_counter_uncharge(&old->memsw, bytes);
		stock->nr_pages = 0;
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
2041
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2042 2043 2044 2045
}

/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
2046
 * This will be consumed by consume_stock() function, later.
2047
 */
2048
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2049 2050 2051
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

2052
	if (stock->cached != memcg) { /* reset if necessary */
2053
		drain_stock(stock);
2054
		stock->cached = memcg;
2055
	}
2056
	stock->nr_pages += nr_pages;
2057 2058 2059 2060
	put_cpu_var(memcg_stock);
}

/*
2061
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2062 2063
 * of the hierarchy under it. sync flag says whether we should block
 * until the work is done.
2064
 */
2065
static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2066
{
2067
	int cpu, curcpu;
2068

2069 2070
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
2071
	curcpu = get_cpu();
2072 2073
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2074
		struct mem_cgroup *memcg;
2075

2076 2077
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
2078
			continue;
2079
		if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2080
			continue;
2081 2082 2083 2084 2085 2086
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2087
	}
2088
	put_cpu();
2089 2090 2091 2092 2093 2094

	if (!sync)
		goto out;

	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2095
		if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2096 2097 2098
			flush_work(&stock->work);
	}
out:
2099
 	put_online_cpus();
2100 2101 2102 2103 2104 2105 2106 2107
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
2108
static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2109
{
2110 2111 2112 2113 2114
	/*
	 * If someone calls draining, avoid adding more kworker runs.
	 */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2115
	drain_all_stock(root_memcg, false);
2116
	mutex_unlock(&percpu_charge_mutex);
2117 2118 2119
}

/* This is a synchronous drain interface. */
2120
static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2121 2122
{
	/* called when force_empty is called */
2123
	mutex_lock(&percpu_charge_mutex);
2124
	drain_all_stock(root_memcg, true);
2125
	mutex_unlock(&percpu_charge_mutex);
2126 2127
}

2128 2129 2130 2131
/*
 * This function drains percpu counter value from DEAD cpu and
 * move it to local cpu. Note that this function can be preempted.
 */
2132
static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2133 2134 2135
{
	int i;

2136
	spin_lock(&memcg->pcp_counter_lock);
2137
	for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
2138
		long x = per_cpu(memcg->stat->count[i], cpu);
2139

2140 2141
		per_cpu(memcg->stat->count[i], cpu) = 0;
		memcg->nocpu_base.count[i] += x;
2142
	}
2143
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2144
		unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2145

2146 2147
		per_cpu(memcg->stat->events[i], cpu) = 0;
		memcg->nocpu_base.events[i] += x;
2148
	}
2149
	/* need to clear ON_MOVE value, works as a kind of lock. */
2150 2151
	per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
	spin_unlock(&memcg->pcp_counter_lock);
2152 2153
}

2154
static void synchronize_mem_cgroup_on_move(struct mem_cgroup *memcg, int cpu)
2155 2156 2157
{
	int idx = MEM_CGROUP_ON_MOVE;

2158 2159 2160
	spin_lock(&memcg->pcp_counter_lock);
	per_cpu(memcg->stat->count[idx], cpu) = memcg->nocpu_base.count[idx];
	spin_unlock(&memcg->pcp_counter_lock);
2161 2162 2163
}

static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2164 2165 2166 2167 2168
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;
2169
	struct mem_cgroup *iter;
2170

2171 2172 2173 2174 2175 2176
	if ((action == CPU_ONLINE)) {
		for_each_mem_cgroup_all(iter)
			synchronize_mem_cgroup_on_move(iter, cpu);
		return NOTIFY_OK;
	}

2177
	if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
2178
		return NOTIFY_OK;
2179 2180 2181 2182

	for_each_mem_cgroup_all(iter)
		mem_cgroup_drain_pcp_counter(iter, cpu);

2183 2184 2185 2186 2187
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

2188 2189 2190 2191 2192 2193 2194 2195 2196 2197

/* See __mem_cgroup_try_charge() for details */
enum {
	CHARGE_OK,		/* success */
	CHARGE_RETRY,		/* need to retry but retry is not bad */
	CHARGE_NOMEM,		/* we can't do more. return -ENOMEM */
	CHARGE_WOULDBLOCK,	/* GFP_WAIT wasn't set and no enough res. */
	CHARGE_OOM_DIE,		/* the current is killed because of OOM */
};

2198
static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2199
				unsigned int nr_pages, bool oom_check)
2200
{
2201
	unsigned long csize = nr_pages * PAGE_SIZE;
2202 2203 2204 2205 2206
	struct mem_cgroup *mem_over_limit;
	struct res_counter *fail_res;
	unsigned long flags = 0;
	int ret;

2207
	ret = res_counter_charge(&memcg->res, csize, &fail_res);
2208 2209 2210 2211

	if (likely(!ret)) {
		if (!do_swap_account)
			return CHARGE_OK;
2212
		ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2213 2214 2215
		if (likely(!ret))
			return CHARGE_OK;

2216
		res_counter_uncharge(&memcg->res, csize);
2217 2218 2219 2220
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
		flags |= MEM_CGROUP_RECLAIM_NOSWAP;
	} else
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2221
	/*
2222 2223
	 * nr_pages can be either a huge page (HPAGE_PMD_NR), a batch
	 * of regular pages (CHARGE_BATCH), or a single regular page (1).
2224 2225 2226 2227
	 *
	 * Never reclaim on behalf of optional batching, retry with a
	 * single page instead.
	 */
2228
	if (nr_pages == CHARGE_BATCH)
2229 2230 2231 2232 2233 2234
		return CHARGE_RETRY;

	if (!(gfp_mask & __GFP_WAIT))
		return CHARGE_WOULDBLOCK;

	ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
2235
					      gfp_mask, flags, NULL);
2236
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2237
		return CHARGE_RETRY;
2238
	/*
2239 2240 2241 2242 2243 2244 2245
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
2246
	 */
2247
	if (nr_pages == 1 && ret)
2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266
		return CHARGE_RETRY;

	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		return CHARGE_RETRY;

	/* If we don't need to call oom-killer at el, return immediately */
	if (!oom_check)
		return CHARGE_NOMEM;
	/* check OOM */
	if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
		return CHARGE_OOM_DIE;

	return CHARGE_RETRY;
}

2267 2268 2269
/*
 * Unlike exported interface, "oom" parameter is added. if oom==true,
 * oom-killer can be invoked.
2270
 */
2271
static int __mem_cgroup_try_charge(struct mm_struct *mm,
A
Andrea Arcangeli 已提交
2272
				   gfp_t gfp_mask,
2273
				   unsigned int nr_pages,
2274
				   struct mem_cgroup **ptr,
2275
				   bool oom)
2276
{
2277
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2278
	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2279
	struct mem_cgroup *memcg = NULL;
2280
	int ret;
2281

K
KAMEZAWA Hiroyuki 已提交
2282 2283 2284 2285 2286 2287 2288 2289
	/*
	 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
	 * in system level. So, allow to go ahead dying process in addition to
	 * MEMDIE process.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)
		     || fatal_signal_pending(current)))
		goto bypass;
2290

2291
	/*
2292 2293
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
2294 2295 2296
	 * thread group leader migrates. It's possible that mm is not
	 * set, if so charge the init_mm (happens for pagecache usage).
	 */
2297
	if (!*ptr && !mm)
K
KAMEZAWA Hiroyuki 已提交
2298 2299
		goto bypass;
again:
2300 2301 2302 2303
	if (*ptr) { /* css should be a valid one */
		memcg = *ptr;
		VM_BUG_ON(css_is_removed(&memcg->css));
		if (mem_cgroup_is_root(memcg))
K
KAMEZAWA Hiroyuki 已提交
2304
			goto done;
2305
		if (nr_pages == 1 && consume_stock(memcg))
K
KAMEZAWA Hiroyuki 已提交
2306
			goto done;
2307
		css_get(&memcg->css);
2308
	} else {
K
KAMEZAWA Hiroyuki 已提交
2309
		struct task_struct *p;
2310

K
KAMEZAWA Hiroyuki 已提交
2311 2312 2313
		rcu_read_lock();
		p = rcu_dereference(mm->owner);
		/*
2314
		 * Because we don't have task_lock(), "p" can exit.
2315
		 * In that case, "memcg" can point to root or p can be NULL with
2316 2317 2318 2319 2320 2321
		 * race with swapoff. Then, we have small risk of mis-accouning.
		 * But such kind of mis-account by race always happens because
		 * we don't have cgroup_mutex(). It's overkill and we allo that
		 * small race, here.
		 * (*) swapoff at el will charge against mm-struct not against
		 * task-struct. So, mm->owner can be NULL.
K
KAMEZAWA Hiroyuki 已提交
2322
		 */
2323 2324
		memcg = mem_cgroup_from_task(p);
		if (!memcg || mem_cgroup_is_root(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2325 2326 2327
			rcu_read_unlock();
			goto done;
		}
2328
		if (nr_pages == 1 && consume_stock(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
			/*
			 * It seems dagerous to access memcg without css_get().
			 * But considering how consume_stok works, it's not
			 * necessary. If consume_stock success, some charges
			 * from this memcg are cached on this cpu. So, we
			 * don't need to call css_get()/css_tryget() before
			 * calling consume_stock().
			 */
			rcu_read_unlock();
			goto done;
		}
		/* after here, we may be blocked. we need to get refcnt */
2341
		if (!css_tryget(&memcg->css)) {
K
KAMEZAWA Hiroyuki 已提交
2342 2343 2344 2345 2346
			rcu_read_unlock();
			goto again;
		}
		rcu_read_unlock();
	}
2347

2348 2349
	do {
		bool oom_check;
2350

2351
		/* If killed, bypass charge */
K
KAMEZAWA Hiroyuki 已提交
2352
		if (fatal_signal_pending(current)) {
2353
			css_put(&memcg->css);
2354
			goto bypass;
K
KAMEZAWA Hiroyuki 已提交
2355
		}
2356

2357 2358 2359 2360
		oom_check = false;
		if (oom && !nr_oom_retries) {
			oom_check = true;
			nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2361
		}
2362

2363
		ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, oom_check);
2364 2365 2366 2367
		switch (ret) {
		case CHARGE_OK:
			break;
		case CHARGE_RETRY: /* not in OOM situation but retry */
2368
			batch = nr_pages;
2369 2370
			css_put(&memcg->css);
			memcg = NULL;
K
KAMEZAWA Hiroyuki 已提交
2371
			goto again;
2372
		case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2373
			css_put(&memcg->css);
2374 2375
			goto nomem;
		case CHARGE_NOMEM: /* OOM routine works */
K
KAMEZAWA Hiroyuki 已提交
2376
			if (!oom) {
2377
				css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2378
				goto nomem;
K
KAMEZAWA Hiroyuki 已提交
2379
			}
2380 2381 2382 2383
			/* If oom, we never return -ENOMEM */
			nr_oom_retries--;
			break;
		case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2384
			css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2385
			goto bypass;
2386
		}
2387 2388
	} while (ret != CHARGE_OK);

2389
	if (batch > nr_pages)
2390 2391
		refill_stock(memcg, batch - nr_pages);
	css_put(&memcg->css);
2392
done:
2393
	*ptr = memcg;
2394 2395
	return 0;
nomem:
2396
	*ptr = NULL;
2397
	return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2398
bypass:
2399
	*ptr = NULL;
K
KAMEZAWA Hiroyuki 已提交
2400
	return 0;
2401
}
2402

2403 2404 2405 2406 2407
/*
 * Somemtimes we have to undo a charge we got by try_charge().
 * This function is for that and do uncharge, put css's refcnt.
 * gotten by try_charge().
 */
2408
static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2409
				       unsigned int nr_pages)
2410
{
2411
	if (!mem_cgroup_is_root(memcg)) {
2412 2413
		unsigned long bytes = nr_pages * PAGE_SIZE;

2414
		res_counter_uncharge(&memcg->res, bytes);
2415
		if (do_swap_account)
2416
			res_counter_uncharge(&memcg->memsw, bytes);
2417
	}
2418 2419
}

2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
/*
 * A helper function to get mem_cgroup from ID. must be called under
 * rcu_read_lock(). The caller must check css_is_removed() or some if
 * it's concern. (dropping refcnt from swap can be called against removed
 * memcg.)
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
	return container_of(css, struct mem_cgroup, css);
}

2439
struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2440
{
2441
	struct mem_cgroup *memcg = NULL;
2442
	struct page_cgroup *pc;
2443
	unsigned short id;
2444 2445
	swp_entry_t ent;

2446 2447 2448
	VM_BUG_ON(!PageLocked(page));

	pc = lookup_page_cgroup(page);
2449
	lock_page_cgroup(pc);
2450
	if (PageCgroupUsed(pc)) {
2451 2452 2453
		memcg = pc->mem_cgroup;
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2454
	} else if (PageSwapCache(page)) {
2455
		ent.val = page_private(page);
2456 2457
		id = lookup_swap_cgroup(ent);
		rcu_read_lock();
2458 2459 2460
		memcg = mem_cgroup_lookup(id);
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2461
		rcu_read_unlock();
2462
	}
2463
	unlock_page_cgroup(pc);
2464
	return memcg;
2465 2466
}

2467
static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2468
				       struct page *page,
2469
				       unsigned int nr_pages,
2470
				       struct page_cgroup *pc,
2471
				       enum charge_type ctype)
2472
{
2473 2474 2475
	lock_page_cgroup(pc);
	if (unlikely(PageCgroupUsed(pc))) {
		unlock_page_cgroup(pc);
2476
		__mem_cgroup_cancel_charge(memcg, nr_pages);
2477 2478 2479 2480 2481 2482
		return;
	}
	/*
	 * we don't need page_cgroup_lock about tail pages, becase they are not
	 * accessed by any other context at this point.
	 */
2483
	pc->mem_cgroup = memcg;
2484 2485 2486 2487 2488 2489 2490
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
2491
	smp_wmb();
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_CACHE:
	case MEM_CGROUP_CHARGE_TYPE_SHMEM:
		SetPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
		ClearPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	default:
		break;
	}
2505

2506
	mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), nr_pages);
2507
	unlock_page_cgroup(pc);
2508 2509 2510 2511 2512
	/*
	 * "charge_statistics" updated event counter. Then, check it.
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
	 */
2513
	memcg_check_events(memcg, page);
2514
}
2515

2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
			(1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
/*
 * Because tail pages are not marked as "used", set it. We're under
 * zone->lru_lock, 'splitting on pmd' and compund_lock.
 */
void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
{
	struct page_cgroup *head_pc = lookup_page_cgroup(head);
	struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
	unsigned long flags;

2530 2531
	if (mem_cgroup_disabled())
		return;
2532
	/*
2533
	 * We have no races with charge/uncharge but will have races with
2534 2535 2536 2537 2538 2539
	 * page state accounting.
	 */
	move_lock_page_cgroup(head_pc, &flags);

	tail_pc->mem_cgroup = head_pc->mem_cgroup;
	smp_wmb(); /* see __commit_charge() */
2540 2541 2542 2543 2544 2545 2546 2547 2548 2549
	if (PageCgroupAcctLRU(head_pc)) {
		enum lru_list lru;
		struct mem_cgroup_per_zone *mz;

		/*
		 * LRU flags cannot be copied because we need to add tail
		 *.page to LRU by generic call and our hook will be called.
		 * We hold lru_lock, then, reduce counter directly.
		 */
		lru = page_lru(head);
2550
		mz = page_cgroup_zoneinfo(head_pc->mem_cgroup, head);
2551 2552
		MEM_CGROUP_ZSTAT(mz, lru) -= 1;
	}
2553 2554 2555 2556 2557
	tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
	move_unlock_page_cgroup(head_pc, &flags);
}
#endif

2558
/**
2559
 * mem_cgroup_move_account - move account of the page
2560
 * @page: the page
2561
 * @nr_pages: number of regular pages (>1 for huge pages)
2562 2563 2564
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
2565
 * @uncharge: whether we should call uncharge and css_put against @from.
2566 2567
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
2568
 * - page is not on LRU (isolate_page() is useful.)
2569
 * - compound_lock is held when nr_pages > 1
2570
 *
2571
 * This function doesn't do "charge" nor css_get to new cgroup. It should be
L
Lucas De Marchi 已提交
2572
 * done by a caller(__mem_cgroup_try_charge would be useful). If @uncharge is
2573 2574
 * true, this function does "uncharge" from old cgroup, but it doesn't if
 * @uncharge is false, so a caller should do "uncharge".
2575
 */
2576 2577 2578 2579 2580 2581
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct page_cgroup *pc,
				   struct mem_cgroup *from,
				   struct mem_cgroup *to,
				   bool uncharge)
2582
{
2583 2584
	unsigned long flags;
	int ret;
2585

2586
	VM_BUG_ON(from == to);
2587
	VM_BUG_ON(PageLRU(page));
2588 2589 2590 2591 2592 2593 2594
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
2595
	if (nr_pages > 1 && !PageTransHuge(page))
2596 2597 2598 2599 2600 2601 2602 2603 2604
		goto out;

	lock_page_cgroup(pc);

	ret = -EINVAL;
	if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
		goto unlock;

	move_lock_page_cgroup(pc, &flags);
2605

2606
	if (PageCgroupFileMapped(pc)) {
2607 2608 2609 2610 2611
		/* Update mapped_file data for mem_cgroup */
		preempt_disable();
		__this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		__this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		preempt_enable();
2612
	}
2613
	mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
2614 2615
	if (uncharge)
		/* This is not "cancel", but cancel_charge does all we need. */
2616
		__mem_cgroup_cancel_charge(from, nr_pages);
2617

2618
	/* caller should have done css_get */
K
KAMEZAWA Hiroyuki 已提交
2619
	pc->mem_cgroup = to;
2620
	mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages);
2621 2622 2623
	/*
	 * We charges against "to" which may not have any tasks. Then, "to"
	 * can be under rmdir(). But in current implementation, caller of
2624
	 * this function is just force_empty() and move charge, so it's
L
Lucas De Marchi 已提交
2625
	 * guaranteed that "to" is never removed. So, we don't check rmdir
2626
	 * status here.
2627
	 */
2628 2629 2630
	move_unlock_page_cgroup(pc, &flags);
	ret = 0;
unlock:
2631
	unlock_page_cgroup(pc);
2632 2633 2634
	/*
	 * check events
	 */
2635 2636
	memcg_check_events(to, page);
	memcg_check_events(from, page);
2637
out:
2638 2639 2640 2641 2642 2643 2644
	return ret;
}

/*
 * move charges to its parent.
 */

2645 2646
static int mem_cgroup_move_parent(struct page *page,
				  struct page_cgroup *pc,
2647 2648 2649 2650 2651 2652
				  struct mem_cgroup *child,
				  gfp_t gfp_mask)
{
	struct cgroup *cg = child->css.cgroup;
	struct cgroup *pcg = cg->parent;
	struct mem_cgroup *parent;
2653
	unsigned int nr_pages;
2654
	unsigned long uninitialized_var(flags);
2655 2656 2657 2658 2659 2660
	int ret;

	/* Is ROOT ? */
	if (!pcg)
		return -EINVAL;

2661 2662 2663 2664 2665
	ret = -EBUSY;
	if (!get_page_unless_zero(page))
		goto out;
	if (isolate_lru_page(page))
		goto put;
2666

2667
	nr_pages = hpage_nr_pages(page);
K
KAMEZAWA Hiroyuki 已提交
2668

2669
	parent = mem_cgroup_from_cont(pcg);
2670
	ret = __mem_cgroup_try_charge(NULL, gfp_mask, nr_pages, &parent, false);
2671
	if (ret || !parent)
2672
		goto put_back;
2673

2674
	if (nr_pages > 1)
2675 2676
		flags = compound_lock_irqsave(page);

2677
	ret = mem_cgroup_move_account(page, nr_pages, pc, child, parent, true);
2678
	if (ret)
2679
		__mem_cgroup_cancel_charge(parent, nr_pages);
2680

2681
	if (nr_pages > 1)
2682
		compound_unlock_irqrestore(page, flags);
2683
put_back:
K
KAMEZAWA Hiroyuki 已提交
2684
	putback_lru_page(page);
2685
put:
2686
	put_page(page);
2687
out:
2688 2689 2690
	return ret;
}

2691 2692 2693 2694 2695 2696 2697
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2698
				gfp_t gfp_mask, enum charge_type ctype)
2699
{
2700
	struct mem_cgroup *memcg = NULL;
2701
	unsigned int nr_pages = 1;
2702
	struct page_cgroup *pc;
2703
	bool oom = true;
2704
	int ret;
A
Andrea Arcangeli 已提交
2705

A
Andrea Arcangeli 已提交
2706
	if (PageTransHuge(page)) {
2707
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
2708
		VM_BUG_ON(!PageTransHuge(page));
2709 2710 2711 2712 2713
		/*
		 * Never OOM-kill a process for a huge page.  The
		 * fault handler will fall back to regular pages.
		 */
		oom = false;
A
Andrea Arcangeli 已提交
2714
	}
2715 2716

	pc = lookup_page_cgroup(page);
2717
	BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */
2718

2719 2720
	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
	if (ret || !memcg)
2721 2722
		return ret;

2723
	__mem_cgroup_commit_charge(memcg, page, nr_pages, pc, ctype);
2724 2725 2726
	return 0;
}

2727 2728
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
2729
{
2730
	if (mem_cgroup_disabled())
2731
		return 0;
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
	/*
	 * If already mapped, we don't have to account.
	 * If page cache, page->mapping has address_space.
	 * But page->mapping may have out-of-use anon_vma pointer,
	 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
	 * is NULL.
  	 */
	if (page_mapped(page) || (page->mapping && !PageAnon(page)))
		return 0;
	if (unlikely(!mm))
		mm = &init_mm;
2743
	return mem_cgroup_charge_common(page, mm, gfp_mask,
2744
				MEM_CGROUP_CHARGE_TYPE_MAPPED);
2745 2746
}

D
Daisuke Nishimura 已提交
2747 2748 2749 2750
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype);

2751
static void
2752
__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
2753 2754 2755 2756 2757 2758 2759 2760 2761
					enum charge_type ctype)
{
	struct page_cgroup *pc = lookup_page_cgroup(page);
	/*
	 * In some case, SwapCache, FUSE(splice_buf->radixtree), the page
	 * is already on LRU. It means the page may on some other page_cgroup's
	 * LRU. Take care of it.
	 */
	mem_cgroup_lru_del_before_commit(page);
2762
	__mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
2763 2764 2765 2766
	mem_cgroup_lru_add_after_commit(page);
	return;
}

2767 2768
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
2769
{
2770
	struct mem_cgroup *memcg = NULL;
2771 2772
	int ret;

2773
	if (mem_cgroup_disabled())
2774
		return 0;
2775 2776
	if (PageCompound(page))
		return 0;
2777

2778
	if (unlikely(!mm))
2779
		mm = &init_mm;
2780

2781
	if (page_is_file_cache(page)) {
2782 2783
		ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, &memcg, true);
		if (ret || !memcg)
2784
			return ret;
2785

2786 2787 2788 2789 2790
		/*
		 * FUSE reuses pages without going through the final
		 * put that would remove them from the LRU list, make
		 * sure that they get relinked properly.
		 */
2791
		__mem_cgroup_commit_charge_lrucare(page, memcg,
2792 2793 2794
					MEM_CGROUP_CHARGE_TYPE_CACHE);
		return ret;
	}
D
Daisuke Nishimura 已提交
2795 2796
	/* shmem */
	if (PageSwapCache(page)) {
2797
		ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &memcg);
D
Daisuke Nishimura 已提交
2798
		if (!ret)
2799
			__mem_cgroup_commit_charge_swapin(page, memcg,
D
Daisuke Nishimura 已提交
2800 2801 2802
					MEM_CGROUP_CHARGE_TYPE_SHMEM);
	} else
		ret = mem_cgroup_charge_common(page, mm, gfp_mask,
2803
					MEM_CGROUP_CHARGE_TYPE_SHMEM);
2804 2805

	return ret;
2806 2807
}

2808 2809 2810
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
2811
 * struct page_cgroup is acquired. This refcnt will be consumed by
2812 2813
 * "commit()" or removed by "cancel()"
 */
2814 2815 2816 2817
int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
				 struct page *page,
				 gfp_t mask, struct mem_cgroup **ptr)
{
2818
	struct mem_cgroup *memcg;
2819
	int ret;
2820

2821 2822
	*ptr = NULL;

2823
	if (mem_cgroup_disabled())
2824 2825 2826 2827 2828 2829
		return 0;

	if (!do_swap_account)
		goto charge_cur_mm;
	/*
	 * A racing thread's fault, or swapoff, may have already updated
H
Hugh Dickins 已提交
2830 2831 2832
	 * the pte, and even removed page from swap cache: in those cases
	 * do_swap_page()'s pte_same() test will fail; but there's also a
	 * KSM case which does need to charge the page.
2833 2834
	 */
	if (!PageSwapCache(page))
H
Hugh Dickins 已提交
2835
		goto charge_cur_mm;
2836 2837
	memcg = try_get_mem_cgroup_from_page(page);
	if (!memcg)
2838
		goto charge_cur_mm;
2839
	*ptr = memcg;
2840
	ret = __mem_cgroup_try_charge(NULL, mask, 1, ptr, true);
2841
	css_put(&memcg->css);
2842
	return ret;
2843 2844 2845
charge_cur_mm:
	if (unlikely(!mm))
		mm = &init_mm;
2846
	return __mem_cgroup_try_charge(mm, mask, 1, ptr, true);
2847 2848
}

D
Daisuke Nishimura 已提交
2849 2850 2851
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype)
2852
{
2853
	if (mem_cgroup_disabled())
2854 2855 2856
		return;
	if (!ptr)
		return;
2857
	cgroup_exclude_rmdir(&ptr->css);
2858 2859

	__mem_cgroup_commit_charge_lrucare(page, ptr, ctype);
2860 2861 2862
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
2863 2864 2865
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
2866
	 */
2867
	if (do_swap_account && PageSwapCache(page)) {
2868
		swp_entry_t ent = {.val = page_private(page)};
2869
		unsigned short id;
2870
		struct mem_cgroup *memcg;
2871 2872 2873 2874

		id = swap_cgroup_record(ent, 0);
		rcu_read_lock();
		memcg = mem_cgroup_lookup(id);
2875
		if (memcg) {
2876 2877 2878 2879
			/*
			 * This recorded memcg can be obsolete one. So, avoid
			 * calling css_tryget
			 */
2880
			if (!mem_cgroup_is_root(memcg))
2881
				res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2882
			mem_cgroup_swap_statistics(memcg, false);
2883 2884
			mem_cgroup_put(memcg);
		}
2885
		rcu_read_unlock();
2886
	}
2887 2888 2889 2890 2891 2892
	/*
	 * At swapin, we may charge account against cgroup which has no tasks.
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
	cgroup_release_and_wakeup_rmdir(&ptr->css);
2893 2894
}

D
Daisuke Nishimura 已提交
2895 2896 2897 2898 2899 2900
void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
{
	__mem_cgroup_commit_charge_swapin(page, ptr,
					MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

2901
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
2902
{
2903
	if (mem_cgroup_disabled())
2904
		return;
2905
	if (!memcg)
2906
		return;
2907
	__mem_cgroup_cancel_charge(memcg, 1);
2908 2909
}

2910
static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
2911 2912
				   unsigned int nr_pages,
				   const enum charge_type ctype)
2913 2914 2915
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
2916

2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
2928
		batch->memcg = memcg;
2929 2930
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
L
Lucas De Marchi 已提交
2931
	 * In those cases, all pages freed continuously can be expected to be in
2932 2933 2934 2935 2936 2937 2938 2939
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */

	if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

2940
	if (nr_pages > 1)
A
Andrea Arcangeli 已提交
2941 2942
		goto direct_uncharge;

2943 2944 2945 2946 2947
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
2948
	if (batch->memcg != memcg)
2949 2950
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
2951
	batch->nr_pages++;
2952
	if (uncharge_memsw)
2953
		batch->memsw_nr_pages++;
2954 2955
	return;
direct_uncharge:
2956
	res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
2957
	if (uncharge_memsw)
2958 2959 2960
		res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
	if (unlikely(batch->memcg != memcg))
		memcg_oom_recover(memcg);
2961 2962
	return;
}
2963

2964
/*
2965
 * uncharge if !page_mapped(page)
2966
 */
2967
static struct mem_cgroup *
2968
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2969
{
2970
	struct mem_cgroup *memcg = NULL;
2971 2972
	unsigned int nr_pages = 1;
	struct page_cgroup *pc;
2973

2974
	if (mem_cgroup_disabled())
2975
		return NULL;
2976

K
KAMEZAWA Hiroyuki 已提交
2977
	if (PageSwapCache(page))
2978
		return NULL;
K
KAMEZAWA Hiroyuki 已提交
2979

A
Andrea Arcangeli 已提交
2980
	if (PageTransHuge(page)) {
2981
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
2982 2983
		VM_BUG_ON(!PageTransHuge(page));
	}
2984
	/*
2985
	 * Check if our page_cgroup is valid
2986
	 */
2987 2988
	pc = lookup_page_cgroup(page);
	if (unlikely(!pc || !PageCgroupUsed(pc)))
2989
		return NULL;
2990

2991
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
2992

2993
	memcg = pc->mem_cgroup;
2994

K
KAMEZAWA Hiroyuki 已提交
2995 2996 2997 2998 2999
	if (!PageCgroupUsed(pc))
		goto unlock_out;

	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
K
KAMEZAWA Hiroyuki 已提交
3000
	case MEM_CGROUP_CHARGE_TYPE_DROP:
3001 3002
		/* See mem_cgroup_prepare_migration() */
		if (page_mapped(page) || PageCgroupMigration(pc))
K
KAMEZAWA Hiroyuki 已提交
3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
3014
	}
K
KAMEZAWA Hiroyuki 已提交
3015

3016
	mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -nr_pages);
K
KAMEZAWA Hiroyuki 已提交
3017

3018
	ClearPageCgroupUsed(pc);
3019 3020 3021 3022 3023 3024
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
3025

3026
	unlock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
3027
	/*
3028
	 * even after unlock, we have memcg->res.usage here and this memcg
K
KAMEZAWA Hiroyuki 已提交
3029 3030
	 * will never be freed.
	 */
3031
	memcg_check_events(memcg, page);
K
KAMEZAWA Hiroyuki 已提交
3032
	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
3033 3034
		mem_cgroup_swap_statistics(memcg, true);
		mem_cgroup_get(memcg);
K
KAMEZAWA Hiroyuki 已提交
3035
	}
3036 3037
	if (!mem_cgroup_is_root(memcg))
		mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
3038

3039
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
3040 3041 3042

unlock_out:
	unlock_page_cgroup(pc);
3043
	return NULL;
3044 3045
}

3046 3047
void mem_cgroup_uncharge_page(struct page *page)
{
3048 3049 3050 3051 3052
	/* early check. */
	if (page_mapped(page))
		return;
	if (page->mapping && !PageAnon(page))
		return;
3053 3054 3055 3056 3057 3058
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
3059
	VM_BUG_ON(page->mapping);
3060 3061 3062
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
}

3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
3077 3078
		current->memcg_batch.nr_pages = 0;
		current->memcg_batch.memsw_nr_pages = 0;
3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
3099 3100 3101 3102 3103 3104
	if (batch->nr_pages)
		res_counter_uncharge(&batch->memcg->res,
				     batch->nr_pages * PAGE_SIZE);
	if (batch->memsw_nr_pages)
		res_counter_uncharge(&batch->memcg->memsw,
				     batch->memsw_nr_pages * PAGE_SIZE);
3105
	memcg_oom_recover(batch->memcg);
3106 3107 3108 3109
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

3110
#ifdef CONFIG_SWAP
3111
/*
3112
 * called after __delete_from_swap_cache() and drop "page" account.
3113 3114
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
3115 3116
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
3117 3118
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
3119 3120 3121 3122 3123 3124
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

	memcg = __mem_cgroup_uncharge_common(page, ctype);
3125

K
KAMEZAWA Hiroyuki 已提交
3126 3127 3128 3129 3130
	/*
	 * record memcg information,  if swapout && memcg != NULL,
	 * mem_cgroup_get() was called in uncharge().
	 */
	if (do_swap_account && swapout && memcg)
3131
		swap_cgroup_record(ent, css_id(&memcg->css));
3132
}
3133
#endif
3134 3135 3136 3137 3138 3139 3140

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
3141
{
3142
	struct mem_cgroup *memcg;
3143
	unsigned short id;
3144 3145 3146 3147

	if (!do_swap_account)
		return;

3148 3149 3150
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
3151
	if (memcg) {
3152 3153 3154 3155
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
3156
		if (!mem_cgroup_is_root(memcg))
3157
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
3158
		mem_cgroup_swap_statistics(memcg, false);
3159 3160
		mem_cgroup_put(memcg);
	}
3161
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
3162
}
3163 3164 3165 3166 3167 3168

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
3169
 * @need_fixup: whether we should fixup res_counters and refcounts.
3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
 * The caller must have charged to @to, IOW, called res_counter_charge() about
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
3180
		struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
3181 3182 3183 3184 3185 3186 3187 3188
{
	unsigned short old_id, new_id;

	old_id = css_id(&from->css);
	new_id = css_id(&to->css);

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
3189
		mem_cgroup_swap_statistics(to, true);
3190
		/*
3191 3192 3193 3194 3195 3196
		 * This function is only called from task migration context now.
		 * It postpones res_counter and refcount handling till the end
		 * of task migration(mem_cgroup_clear_mc()) for performance
		 * improvement. But we cannot postpone mem_cgroup_get(to)
		 * because if the process that has been moved to @to does
		 * swap-in, the refcount of @to might be decreased to 0.
3197 3198
		 */
		mem_cgroup_get(to);
3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209
		if (need_fixup) {
			if (!mem_cgroup_is_root(from))
				res_counter_uncharge(&from->memsw, PAGE_SIZE);
			mem_cgroup_put(from);
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			if (!mem_cgroup_is_root(to))
				res_counter_uncharge(&to->res, PAGE_SIZE);
		}
3210 3211 3212 3213 3214 3215
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3216
		struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
3217 3218 3219
{
	return -EINVAL;
}
3220
#endif
K
KAMEZAWA Hiroyuki 已提交
3221

3222
/*
3223 3224
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
3225
 */
3226
int mem_cgroup_prepare_migration(struct page *page,
3227
	struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
3228
{
3229
	struct mem_cgroup *memcg = NULL;
3230
	struct page_cgroup *pc;
3231
	enum charge_type ctype;
3232
	int ret = 0;
3233

3234 3235
	*ptr = NULL;

A
Andrea Arcangeli 已提交
3236
	VM_BUG_ON(PageTransHuge(page));
3237
	if (mem_cgroup_disabled())
3238 3239
		return 0;

3240 3241 3242
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
3243 3244
		memcg = pc->mem_cgroup;
		css_get(&memcg->css);
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275
		/*
		 * At migrating an anonymous page, its mapcount goes down
		 * to 0 and uncharge() will be called. But, even if it's fully
		 * unmapped, migration may fail and this page has to be
		 * charged again. We set MIGRATION flag here and delay uncharge
		 * until end_migration() is called
		 *
		 * Corner Case Thinking
		 * A)
		 * When the old page was mapped as Anon and it's unmap-and-freed
		 * while migration was ongoing.
		 * If unmap finds the old page, uncharge() of it will be delayed
		 * until end_migration(). If unmap finds a new page, it's
		 * uncharged when it make mapcount to be 1->0. If unmap code
		 * finds swap_migration_entry, the new page will not be mapped
		 * and end_migration() will find it(mapcount==0).
		 *
		 * B)
		 * When the old page was mapped but migraion fails, the kernel
		 * remaps it. A charge for it is kept by MIGRATION flag even
		 * if mapcount goes down to 0. We can do remap successfully
		 * without charging it again.
		 *
		 * C)
		 * The "old" page is under lock_page() until the end of
		 * migration, so, the old page itself will not be swapped-out.
		 * If the new page is swapped out before end_migraton, our
		 * hook to usual swap-out path will catch the event.
		 */
		if (PageAnon(page))
			SetPageCgroupMigration(pc);
3276
	}
3277
	unlock_page_cgroup(pc);
3278 3279 3280 3281
	/*
	 * If the page is not charged at this point,
	 * we return here.
	 */
3282
	if (!memcg)
3283
		return 0;
3284

3285
	*ptr = memcg;
3286
	ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, ptr, false);
3287
	css_put(&memcg->css);/* drop extra refcnt */
3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298
	if (ret || *ptr == NULL) {
		if (PageAnon(page)) {
			lock_page_cgroup(pc);
			ClearPageCgroupMigration(pc);
			unlock_page_cgroup(pc);
			/*
			 * The old page may be fully unmapped while we kept it.
			 */
			mem_cgroup_uncharge_page(page);
		}
		return -ENOMEM;
3299
	}
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312
	/*
	 * We charge new page before it's used/mapped. So, even if unlock_page()
	 * is called before end_migration, we can catch all events on this new
	 * page. In the case new page is migrated but not remapped, new page's
	 * mapcount will be finally 0 and we call uncharge in end_migration().
	 */
	pc = lookup_page_cgroup(newpage);
	if (PageAnon(page))
		ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
	else if (page_is_file_cache(page))
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
	else
		ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
3313
	__mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
3314
	return ret;
3315
}
3316

3317
/* remove redundant charge if migration failed*/
3318
void mem_cgroup_end_migration(struct mem_cgroup *memcg,
3319
	struct page *oldpage, struct page *newpage, bool migration_ok)
3320
{
3321
	struct page *used, *unused;
3322 3323
	struct page_cgroup *pc;

3324
	if (!memcg)
3325
		return;
3326
	/* blocks rmdir() */
3327
	cgroup_exclude_rmdir(&memcg->css);
3328
	if (!migration_ok) {
3329 3330
		used = oldpage;
		unused = newpage;
3331
	} else {
3332
		used = newpage;
3333 3334
		unused = oldpage;
	}
3335
	/*
3336 3337 3338
	 * We disallowed uncharge of pages under migration because mapcount
	 * of the page goes down to zero, temporarly.
	 * Clear the flag and check the page should be charged.
3339
	 */
3340 3341 3342 3343
	pc = lookup_page_cgroup(oldpage);
	lock_page_cgroup(pc);
	ClearPageCgroupMigration(pc);
	unlock_page_cgroup(pc);
3344

3345 3346
	__mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);

3347
	/*
3348 3349 3350 3351 3352 3353
	 * If a page is a file cache, radix-tree replacement is very atomic
	 * and we can skip this check. When it was an Anon page, its mapcount
	 * goes down to 0. But because we added MIGRATION flage, it's not
	 * uncharged yet. There are several case but page->mapcount check
	 * and USED bit check in mem_cgroup_uncharge_page() will do enough
	 * check. (see prepare_charge() also)
3354
	 */
3355 3356
	if (PageAnon(used))
		mem_cgroup_uncharge_page(used);
3357
	/*
3358 3359
	 * At migration, we may charge account against cgroup which has no
	 * tasks.
3360 3361 3362
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
3363
	cgroup_release_and_wakeup_rmdir(&memcg->css);
3364
}
3365

3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411
#ifdef CONFIG_DEBUG_VM
static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
	if (likely(pc) && PageCgroupUsed(pc))
		return pc;
	return NULL;
}

bool mem_cgroup_bad_page_check(struct page *page)
{
	if (mem_cgroup_disabled())
		return false;

	return lookup_page_cgroup_used(page) != NULL;
}

void mem_cgroup_print_bad_page(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup_used(page);
	if (pc) {
		int ret = -1;
		char *path;

		printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p",
		       pc, pc->flags, pc->mem_cgroup);

		path = kmalloc(PATH_MAX, GFP_KERNEL);
		if (path) {
			rcu_read_lock();
			ret = cgroup_path(pc->mem_cgroup->css.cgroup,
							path, PATH_MAX);
			rcu_read_unlock();
		}

		printk(KERN_CONT "(%s)\n",
				(ret < 0) ? "cannot get the path" : path);
		kfree(path);
	}
}
#endif

3412 3413
static DEFINE_MUTEX(set_limit_mutex);

3414
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
3415
				unsigned long long val)
3416
{
3417
	int retry_count;
3418
	u64 memswlimit, memlimit;
3419
	int ret = 0;
3420 3421
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;
3422
	int enlarge;
3423 3424 3425 3426 3427 3428 3429 3430 3431

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3432

3433
	enlarge = 0;
3434
	while (retry_count) {
3435 3436 3437 3438
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
3439 3440 3441
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
3442
		 * We have to guarantee memcg->res.limit < memcg->memsw.limit.
3443 3444 3445 3446 3447 3448
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
3449 3450
			break;
		}
3451 3452 3453 3454 3455

		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit < val)
			enlarge = 1;

3456
		ret = res_counter_set_limit(&memcg->res, val);
3457 3458 3459 3460 3461 3462
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3463 3464 3465 3466 3467
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3468
		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3469 3470
						MEM_CGROUP_RECLAIM_SHRINK,
						NULL);
3471 3472 3473 3474 3475 3476
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
3477
	}
3478 3479
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3480

3481 3482 3483
	return ret;
}

L
Li Zefan 已提交
3484 3485
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
3486
{
3487
	int retry_count;
3488
	u64 memlimit, memswlimit, oldusage, curusage;
3489 3490
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
3491
	int enlarge = 0;
3492

3493 3494 3495
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3496 3497 3498 3499 3500 3501 3502 3503
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
3504
		 * We have to guarantee memcg->res.limit < memcg->memsw.limit.
3505 3506 3507 3508 3509 3510 3511 3512
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
3513 3514 3515
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val)
			enlarge = 1;
3516
		ret = res_counter_set_limit(&memcg->memsw, val);
3517 3518 3519 3520 3521 3522
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3523 3524 3525 3526 3527
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3528
		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3529
						MEM_CGROUP_RECLAIM_NOSWAP |
3530 3531
						MEM_CGROUP_RECLAIM_SHRINK,
						NULL);
3532
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3533
		/* Usage is reduced ? */
3534
		if (curusage >= oldusage)
3535
			retry_count--;
3536 3537
		else
			oldusage = curusage;
3538
	}
3539 3540
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3541 3542 3543
	return ret;
}

3544
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3545 3546
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
3547 3548 3549 3550 3551 3552
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
3553
	unsigned long long excess;
3554
	unsigned long nr_scanned;
3555 3556 3557 3558

	if (order > 0)
		return 0;

3559
	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

3573
		nr_scanned = 0;
3574 3575
		reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
						gfp_mask,
3576 3577
						MEM_CGROUP_RECLAIM_SOFT,
						&nr_scanned);
3578
		nr_reclaimed += reclaimed;
3579
		*total_scanned += nr_scanned;
3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
3602
				if (next_mz == mz)
3603
					css_put(&next_mz->mem->css);
3604
				else /* next_mz == NULL or other memcg */
3605 3606 3607 3608
					break;
			} while (1);
		}
		__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
3609
		excess = res_counter_soft_limit_excess(&mz->mem->res);
3610 3611 3612 3613 3614 3615 3616 3617
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
3618 3619
		/* If excess == 0, no tree ops */
		__mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637
		spin_unlock(&mctz->lock);
		css_put(&mz->mem->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->mem->css);
	return nr_reclaimed;
}

3638 3639 3640 3641
/*
 * This routine traverse page_cgroup in given list and drop them all.
 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
 */
3642
static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
K
KAMEZAWA Hiroyuki 已提交
3643
				int node, int zid, enum lru_list lru)
3644
{
K
KAMEZAWA Hiroyuki 已提交
3645 3646
	struct zone *zone;
	struct mem_cgroup_per_zone *mz;
3647
	struct page_cgroup *pc, *busy;
K
KAMEZAWA Hiroyuki 已提交
3648
	unsigned long flags, loop;
3649
	struct list_head *list;
3650
	int ret = 0;
3651

K
KAMEZAWA Hiroyuki 已提交
3652
	zone = &NODE_DATA(node)->node_zones[zid];
3653
	mz = mem_cgroup_zoneinfo(memcg, node, zid);
3654
	list = &mz->lists[lru];
3655

3656 3657 3658 3659 3660
	loop = MEM_CGROUP_ZSTAT(mz, lru);
	/* give some margin against EBUSY etc...*/
	loop += 256;
	busy = NULL;
	while (loop--) {
3661 3662
		struct page *page;

3663
		ret = 0;
K
KAMEZAWA Hiroyuki 已提交
3664
		spin_lock_irqsave(&zone->lru_lock, flags);
3665
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
3666
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3667
			break;
3668 3669 3670 3671
		}
		pc = list_entry(list->prev, struct page_cgroup, lru);
		if (busy == pc) {
			list_move(&pc->lru, list);
3672
			busy = NULL;
K
KAMEZAWA Hiroyuki 已提交
3673
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3674 3675
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
3676
		spin_unlock_irqrestore(&zone->lru_lock, flags);
3677

3678
		page = lookup_cgroup_page(pc);
3679

3680
		ret = mem_cgroup_move_parent(page, pc, memcg, GFP_KERNEL);
3681
		if (ret == -ENOMEM)
3682
			break;
3683 3684 3685 3686 3687 3688 3689

		if (ret == -EBUSY || ret == -EINVAL) {
			/* found lock contention or "pc" is obsolete. */
			busy = pc;
			cond_resched();
		} else
			busy = NULL;
3690
	}
K
KAMEZAWA Hiroyuki 已提交
3691

3692 3693 3694
	if (!ret && !list_empty(list))
		return -EBUSY;
	return ret;
3695 3696 3697 3698 3699 3700
}

/*
 * make mem_cgroup's charge to be 0 if there is no task.
 * This enables deleting this mem_cgroup.
 */
3701
static int mem_cgroup_force_empty(struct mem_cgroup *memcg, bool free_all)
3702
{
3703 3704 3705
	int ret;
	int node, zid, shrink;
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
3706
	struct cgroup *cgrp = memcg->css.cgroup;
3707

3708
	css_get(&memcg->css);
3709 3710

	shrink = 0;
3711 3712 3713
	/* should free all ? */
	if (free_all)
		goto try_to_free;
3714
move_account:
3715
	do {
3716
		ret = -EBUSY;
3717 3718 3719 3720
		if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
			goto out;
		ret = -EINTR;
		if (signal_pending(current))
3721
			goto out;
3722 3723
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
3724
		drain_all_stock_sync(memcg);
3725
		ret = 0;
3726
		mem_cgroup_start_move(memcg);
3727
		for_each_node_state(node, N_HIGH_MEMORY) {
3728
			for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
3729
				enum lru_list l;
3730
				for_each_lru(l) {
3731
					ret = mem_cgroup_force_empty_list(memcg,
K
KAMEZAWA Hiroyuki 已提交
3732
							node, zid, l);
3733 3734 3735
					if (ret)
						break;
				}
3736
			}
3737 3738 3739
			if (ret)
				break;
		}
3740 3741
		mem_cgroup_end_move(memcg);
		memcg_oom_recover(memcg);
3742 3743 3744
		/* it seems parent cgroup doesn't have enough mem */
		if (ret == -ENOMEM)
			goto try_to_free;
3745
		cond_resched();
3746
	/* "ret" should also be checked to ensure all lists are empty. */
3747
	} while (memcg->res.usage > 0 || ret);
3748
out:
3749
	css_put(&memcg->css);
3750
	return ret;
3751 3752

try_to_free:
3753 3754
	/* returns EBUSY if there is a task or if we come here twice. */
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
3755 3756 3757
		ret = -EBUSY;
		goto out;
	}
3758 3759
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
3760 3761
	/* try to free all pages in this cgroup */
	shrink = 1;
3762
	while (nr_retries && memcg->res.usage > 0) {
3763
		int progress;
3764 3765 3766 3767 3768

		if (signal_pending(current)) {
			ret = -EINTR;
			goto out;
		}
3769
		progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
3770
						false);
3771
		if (!progress) {
3772
			nr_retries--;
3773
			/* maybe some writeback is necessary */
3774
			congestion_wait(BLK_RW_ASYNC, HZ/10);
3775
		}
3776 3777

	}
K
KAMEZAWA Hiroyuki 已提交
3778
	lru_add_drain();
3779
	/* try move_account...there may be some *locked* pages. */
3780
	goto move_account;
3781 3782
}

3783 3784 3785 3786 3787 3788
int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
{
	return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
}


3789 3790 3791 3792 3793 3794 3795 3796 3797
static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
{
	return mem_cgroup_from_cont(cont)->use_hierarchy;
}

static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
					u64 val)
{
	int retval = 0;
3798
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3799
	struct cgroup *parent = cont->parent;
3800
	struct mem_cgroup *parent_memcg = NULL;
3801 3802

	if (parent)
3803
		parent_memcg = mem_cgroup_from_cont(parent);
3804 3805 3806

	cgroup_lock();
	/*
3807
	 * If parent's use_hierarchy is set, we can't make any modifications
3808 3809 3810 3811 3812 3813
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
3814
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3815 3816
				(val == 1 || val == 0)) {
		if (list_empty(&cont->children))
3817
			memcg->use_hierarchy = val;
3818 3819 3820 3821 3822 3823 3824 3825 3826
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
	cgroup_unlock();

	return retval;
}

3827

3828
static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
3829
					       enum mem_cgroup_stat_index idx)
3830
{
K
KAMEZAWA Hiroyuki 已提交
3831
	struct mem_cgroup *iter;
3832
	long val = 0;
3833

3834
	/* Per-cpu values can be negative, use a signed accumulator */
3835
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
3836 3837 3838 3839 3840
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
3841 3842
}

3843
static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3844
{
K
KAMEZAWA Hiroyuki 已提交
3845
	u64 val;
3846

3847
	if (!mem_cgroup_is_root(memcg)) {
3848
		if (!swap)
3849
			return res_counter_read_u64(&memcg->res, RES_USAGE);
3850
		else
3851
			return res_counter_read_u64(&memcg->memsw, RES_USAGE);
3852 3853
	}

3854 3855
	val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
3856

K
KAMEZAWA Hiroyuki 已提交
3857
	if (swap)
3858
		val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
3859 3860 3861 3862

	return val << PAGE_SHIFT;
}

3863
static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
B
Balbir Singh 已提交
3864
{
3865
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3866
	u64 val;
3867 3868 3869 3870 3871 3872
	int type, name;

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (type) {
	case _MEM:
3873
		if (name == RES_USAGE)
3874
			val = mem_cgroup_usage(memcg, false);
3875
		else
3876
			val = res_counter_read_u64(&memcg->res, name);
3877 3878
		break;
	case _MEMSWAP:
3879
		if (name == RES_USAGE)
3880
			val = mem_cgroup_usage(memcg, true);
3881
		else
3882
			val = res_counter_read_u64(&memcg->memsw, name);
3883 3884 3885 3886 3887 3888
		break;
	default:
		BUG();
		break;
	}
	return val;
B
Balbir Singh 已提交
3889
}
3890 3891 3892 3893
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3894 3895
static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
			    const char *buffer)
B
Balbir Singh 已提交
3896
{
3897
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3898
	int type, name;
3899 3900 3901
	unsigned long long val;
	int ret;

3902 3903 3904
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (name) {
3905
	case RES_LIMIT:
3906 3907 3908 3909
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3910 3911
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
3912 3913 3914
		if (ret)
			break;
		if (type == _MEM)
3915
			ret = mem_cgroup_resize_limit(memcg, val);
3916 3917
		else
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
3918
		break;
3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
3933 3934 3935 3936 3937
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
3938 3939
}

3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	struct cgroup *cgroup;
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	cgroup = memcg->css.cgroup;
	if (!memcg->use_hierarchy)
		goto out;

	while (cgroup->parent) {
		cgroup = cgroup->parent;
		memcg = mem_cgroup_from_cont(cgroup);
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
	return;
}

3968
static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
3969
{
3970
	struct mem_cgroup *memcg;
3971
	int type, name;
3972

3973
	memcg = mem_cgroup_from_cont(cont);
3974 3975 3976
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
	switch (name) {
3977
	case RES_MAX_USAGE:
3978
		if (type == _MEM)
3979
			res_counter_reset_max(&memcg->res);
3980
		else
3981
			res_counter_reset_max(&memcg->memsw);
3982 3983
		break;
	case RES_FAILCNT:
3984
		if (type == _MEM)
3985
			res_counter_reset_failcnt(&memcg->res);
3986
		else
3987
			res_counter_reset_failcnt(&memcg->memsw);
3988 3989
		break;
	}
3990

3991
	return 0;
3992 3993
}

3994 3995 3996 3997 3998 3999
static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
					struct cftype *cft)
{
	return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
}

4000
#ifdef CONFIG_MMU
4001 4002 4003
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
4004
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4005 4006 4007 4008 4009 4010 4011 4012 4013

	if (val >= (1 << NR_MOVE_TYPE))
		return -EINVAL;
	/*
	 * We check this value several times in both in can_attach() and
	 * attach(), so we need cgroup lock to prevent this value from being
	 * inconsistent.
	 */
	cgroup_lock();
4014
	memcg->move_charge_at_immigrate = val;
4015 4016 4017 4018
	cgroup_unlock();

	return 0;
}
4019 4020 4021 4022 4023 4024 4025
#else
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
4026

K
KAMEZAWA Hiroyuki 已提交
4027 4028 4029 4030 4031

/* For read statistics */
enum {
	MCS_CACHE,
	MCS_RSS,
4032
	MCS_FILE_MAPPED,
K
KAMEZAWA Hiroyuki 已提交
4033 4034
	MCS_PGPGIN,
	MCS_PGPGOUT,
4035
	MCS_SWAP,
4036 4037
	MCS_PGFAULT,
	MCS_PGMAJFAULT,
K
KAMEZAWA Hiroyuki 已提交
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047
	MCS_INACTIVE_ANON,
	MCS_ACTIVE_ANON,
	MCS_INACTIVE_FILE,
	MCS_ACTIVE_FILE,
	MCS_UNEVICTABLE,
	NR_MCS_STAT,
};

struct mcs_total_stat {
	s64 stat[NR_MCS_STAT];
4048 4049
};

K
KAMEZAWA Hiroyuki 已提交
4050 4051 4052 4053 4054 4055
struct {
	char *local_name;
	char *total_name;
} memcg_stat_strings[NR_MCS_STAT] = {
	{"cache", "total_cache"},
	{"rss", "total_rss"},
4056
	{"mapped_file", "total_mapped_file"},
K
KAMEZAWA Hiroyuki 已提交
4057 4058
	{"pgpgin", "total_pgpgin"},
	{"pgpgout", "total_pgpgout"},
4059
	{"swap", "total_swap"},
4060 4061
	{"pgfault", "total_pgfault"},
	{"pgmajfault", "total_pgmajfault"},
K
KAMEZAWA Hiroyuki 已提交
4062 4063 4064 4065 4066 4067 4068 4069
	{"inactive_anon", "total_inactive_anon"},
	{"active_anon", "total_active_anon"},
	{"inactive_file", "total_inactive_file"},
	{"active_file", "total_active_file"},
	{"unevictable", "total_unevictable"}
};


K
KAMEZAWA Hiroyuki 已提交
4070
static void
4071
mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
K
KAMEZAWA Hiroyuki 已提交
4072 4073 4074 4075
{
	s64 val;

	/* per cpu stat */
4076
	val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_CACHE);
K
KAMEZAWA Hiroyuki 已提交
4077
	s->stat[MCS_CACHE] += val * PAGE_SIZE;
4078
	val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_RSS);
K
KAMEZAWA Hiroyuki 已提交
4079
	s->stat[MCS_RSS] += val * PAGE_SIZE;
4080
	val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
4081
	s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
4082
	val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGIN);
K
KAMEZAWA Hiroyuki 已提交
4083
	s->stat[MCS_PGPGIN] += val;
4084
	val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGOUT);
K
KAMEZAWA Hiroyuki 已提交
4085
	s->stat[MCS_PGPGOUT] += val;
4086
	if (do_swap_account) {
4087
		val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
4088 4089
		s->stat[MCS_SWAP] += val * PAGE_SIZE;
	}
4090
	val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGFAULT);
4091
	s->stat[MCS_PGFAULT] += val;
4092
	val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT);
4093
	s->stat[MCS_PGMAJFAULT] += val;
K
KAMEZAWA Hiroyuki 已提交
4094 4095

	/* per zone stat */
4096
	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
K
KAMEZAWA Hiroyuki 已提交
4097
	s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
4098
	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));
K
KAMEZAWA Hiroyuki 已提交
4099
	s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
4100
	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
K
KAMEZAWA Hiroyuki 已提交
4101
	s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
4102
	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));
K
KAMEZAWA Hiroyuki 已提交
4103
	s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
4104
	val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
K
KAMEZAWA Hiroyuki 已提交
4105 4106 4107 4108
	s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
}

static void
4109
mem_cgroup_get_total_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
K
KAMEZAWA Hiroyuki 已提交
4110
{
K
KAMEZAWA Hiroyuki 已提交
4111 4112
	struct mem_cgroup *iter;

4113
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
4114
		mem_cgroup_get_local_stat(iter, s);
K
KAMEZAWA Hiroyuki 已提交
4115 4116
}

4117 4118 4119 4120 4121 4122 4123 4124 4125
#ifdef CONFIG_NUMA
static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
{
	int nid;
	unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
	unsigned long node_nr;
	struct cgroup *cont = m->private;
	struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);

4126
	total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
4127 4128
	seq_printf(m, "total=%lu", total_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4129
		node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
4130 4131 4132 4133
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4134
	file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
4135 4136
	seq_printf(m, "file=%lu", file_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4137 4138
		node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
				LRU_ALL_FILE);
4139 4140 4141 4142
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4143
	anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
4144 4145
	seq_printf(m, "anon=%lu", anon_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4146 4147
		node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
				LRU_ALL_ANON);
4148 4149 4150 4151
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4152
	unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
4153 4154
	seq_printf(m, "unevictable=%lu", unevictable_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4155 4156
		node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
				BIT(LRU_UNEVICTABLE));
4157 4158 4159 4160 4161 4162 4163
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');
	return 0;
}
#endif /* CONFIG_NUMA */

4164 4165
static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
				 struct cgroup_map_cb *cb)
4166 4167
{
	struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
K
KAMEZAWA Hiroyuki 已提交
4168
	struct mcs_total_stat mystat;
4169 4170
	int i;

K
KAMEZAWA Hiroyuki 已提交
4171 4172
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_local_stat(mem_cont, &mystat);
4173

4174

4175 4176 4177
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
4178
		cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
4179
	}
L
Lee Schermerhorn 已提交
4180

K
KAMEZAWA Hiroyuki 已提交
4181
	/* Hierarchical information */
4182 4183 4184 4185 4186 4187 4188
	{
		unsigned long long limit, memsw_limit;
		memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
		cb->fill(cb, "hierarchical_memory_limit", limit);
		if (do_swap_account)
			cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
	}
K
KOSAKI Motohiro 已提交
4189

K
KAMEZAWA Hiroyuki 已提交
4190 4191
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_total_stat(mem_cont, &mystat);
4192 4193 4194
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
4195
		cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
4196
	}
K
KAMEZAWA Hiroyuki 已提交
4197

K
KOSAKI Motohiro 已提交
4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
				mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);

				recent_rotated[0] +=
					mz->reclaim_stat.recent_rotated[0];
				recent_rotated[1] +=
					mz->reclaim_stat.recent_rotated[1];
				recent_scanned[0] +=
					mz->reclaim_stat.recent_scanned[0];
				recent_scanned[1] +=
					mz->reclaim_stat.recent_scanned[1];
			}
		cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
		cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
		cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
		cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
	}
#endif

4225 4226 4227
	return 0;
}

K
KOSAKI Motohiro 已提交
4228 4229 4230 4231
static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);

4232
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
4233 4234 4235 4236 4237 4238 4239
}

static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
				       u64 val)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;
4240

K
KOSAKI Motohiro 已提交
4241 4242 4243 4244 4245 4246 4247
	if (val > 100)
		return -EINVAL;

	if (cgrp->parent == NULL)
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);
4248 4249 4250

	cgroup_lock();

K
KOSAKI Motohiro 已提交
4251 4252
	/* If under hierarchy, only empty-root can set this value */
	if ((parent->use_hierarchy) ||
4253 4254
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
		cgroup_unlock();
K
KOSAKI Motohiro 已提交
4255
		return -EINVAL;
4256
	}
K
KOSAKI Motohiro 已提交
4257 4258 4259

	memcg->swappiness = val;

4260 4261
	cgroup_unlock();

K
KOSAKI Motohiro 已提交
4262 4263 4264
	return 0;
}

4265 4266 4267 4268 4269 4270 4271 4272
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
	u64 usage;
	int i;

	rcu_read_lock();
	if (!swap)
4273
		t = rcu_dereference(memcg->thresholds.primary);
4274
	else
4275
		t = rcu_dereference(memcg->memsw_thresholds.primary);
4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286

	if (!t)
		goto unlock;

	usage = mem_cgroup_usage(memcg, swap);

	/*
	 * current_threshold points to threshold just below usage.
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
4287
	i = t->current_threshold;
4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
4311
	t->current_threshold = i - 1;
4312 4313 4314 4315 4316 4317
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
4318 4319 4320 4321 4322 4323 4324
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
4325 4326 4327 4328 4329 4330 4331 4332 4333 4334
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

	return _a->threshold - _b->threshold;
}

4335
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4336 4337 4338
{
	struct mem_cgroup_eventfd_list *ev;

4339
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
4340 4341 4342 4343
		eventfd_signal(ev->eventfd, 1);
	return 0;
}

4344
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4345
{
K
KAMEZAWA Hiroyuki 已提交
4346 4347
	struct mem_cgroup *iter;

4348
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
4349
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
4350 4351 4352 4353
}

static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4354 4355
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4356 4357
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4358 4359
	int type = MEMFILE_TYPE(cft->private);
	u64 threshold, usage;
4360
	int i, size, ret;
4361 4362 4363 4364 4365 4366

	ret = res_counter_memparse_write_strategy(args, &threshold);
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
4367

4368
	if (type == _MEM)
4369
		thresholds = &memcg->thresholds;
4370
	else if (type == _MEMSWAP)
4371
		thresholds = &memcg->memsw_thresholds;
4372 4373 4374 4375 4376 4377
	else
		BUG();

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before adding a new one */
4378
	if (thresholds->primary)
4379 4380
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

4381
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4382 4383

	/* Allocate memory for new array of thresholds */
4384
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
4385
			GFP_KERNEL);
4386
	if (!new) {
4387 4388 4389
		ret = -ENOMEM;
		goto unlock;
	}
4390
	new->size = size;
4391 4392

	/* Copy thresholds (if any) to new array */
4393 4394
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
4395
				sizeof(struct mem_cgroup_threshold));
4396 4397
	}

4398
	/* Add new threshold */
4399 4400
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
4401 4402

	/* Sort thresholds. Registering of new threshold isn't time-critical */
4403
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
4404 4405 4406
			compare_thresholds, NULL);

	/* Find current threshold */
4407
	new->current_threshold = -1;
4408
	for (i = 0; i < size; i++) {
4409
		if (new->entries[i].threshold < usage) {
4410
			/*
4411 4412
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
4413 4414
			 * it here.
			 */
4415
			++new->current_threshold;
4416 4417 4418
		}
	}

4419 4420 4421 4422 4423
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
4424

4425
	/* To be sure that nobody uses thresholds */
4426 4427 4428 4429 4430 4431 4432 4433
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

4434
static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4435
	struct cftype *cft, struct eventfd_ctx *eventfd)
4436 4437
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4438 4439
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4440 4441
	int type = MEMFILE_TYPE(cft->private);
	u64 usage;
4442
	int i, j, size;
4443 4444 4445

	mutex_lock(&memcg->thresholds_lock);
	if (type == _MEM)
4446
		thresholds = &memcg->thresholds;
4447
	else if (type == _MEMSWAP)
4448
		thresholds = &memcg->memsw_thresholds;
4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463
	else
		BUG();

	/*
	 * Something went wrong if we trying to unregister a threshold
	 * if we don't have thresholds
	 */
	BUG_ON(!thresholds);

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
4464 4465 4466
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
4467 4468 4469
			size++;
	}

4470
	new = thresholds->spare;
4471

4472 4473
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
4474 4475
		kfree(new);
		new = NULL;
4476
		goto swap_buffers;
4477 4478
	}

4479
	new->size = size;
4480 4481

	/* Copy thresholds and find current threshold */
4482 4483 4484
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
4485 4486
			continue;

4487 4488
		new->entries[j] = thresholds->primary->entries[i];
		if (new->entries[j].threshold < usage) {
4489
			/*
4490
			 * new->current_threshold will not be used
4491 4492 4493
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
4494
			++new->current_threshold;
4495 4496 4497 4498
		}
		j++;
	}

4499
swap_buffers:
4500 4501 4502
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
	rcu_assign_pointer(thresholds->primary, new);
4503

4504
	/* To be sure that nobody uses thresholds */
4505 4506 4507 4508
	synchronize_rcu();

	mutex_unlock(&memcg->thresholds_lock);
}
4509

K
KAMEZAWA Hiroyuki 已提交
4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521
static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *event;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);
	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

4522
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4523 4524 4525 4526 4527

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
4528
	if (atomic_read(&memcg->under_oom))
K
KAMEZAWA Hiroyuki 已提交
4529
		eventfd_signal(eventfd, 1);
4530
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4531 4532 4533 4534

	return 0;
}

4535
static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4536 4537
	struct cftype *cft, struct eventfd_ctx *eventfd)
{
4538
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
K
KAMEZAWA Hiroyuki 已提交
4539 4540 4541 4542 4543
	struct mem_cgroup_eventfd_list *ev, *tmp;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);

4544
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4545

4546
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
4547 4548 4549 4550 4551 4552
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

4553
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4554 4555
}

4556 4557 4558
static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
	struct cftype *cft,  struct cgroup_map_cb *cb)
{
4559
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4560

4561
	cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
4562

4563
	if (atomic_read(&memcg->under_oom))
4564 4565 4566 4567 4568 4569 4570 4571 4572
		cb->fill(cb, "under_oom", 1);
	else
		cb->fill(cb, "under_oom", 0);
	return 0;
}

static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
	struct cftype *cft, u64 val)
{
4573
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584
	struct mem_cgroup *parent;

	/* cannot set to root cgroup and only 0 and 1 are allowed */
	if (!cgrp->parent || !((val == 0) || (val == 1)))
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);

	cgroup_lock();
	/* oom-kill-disable is a flag for subhierarchy. */
	if ((parent->use_hierarchy) ||
4585
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
4586 4587 4588
		cgroup_unlock();
		return -EINVAL;
	}
4589
	memcg->oom_kill_disable = val;
4590
	if (!val)
4591
		memcg_oom_recover(memcg);
4592 4593 4594 4595
	cgroup_unlock();
	return 0;
}

4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611
#ifdef CONFIG_NUMA
static const struct file_operations mem_control_numa_stat_file_operations = {
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
{
	struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;

	file->f_op = &mem_control_numa_stat_file_operations;
	return single_open(file, mem_control_numa_stat_show, cont);
}
#endif /* CONFIG_NUMA */

B
Balbir Singh 已提交
4612 4613
static struct cftype mem_cgroup_files[] = {
	{
4614
		.name = "usage_in_bytes",
4615
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4616
		.read_u64 = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4617 4618
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
B
Balbir Singh 已提交
4619
	},
4620 4621
	{
		.name = "max_usage_in_bytes",
4622
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4623
		.trigger = mem_cgroup_reset,
4624 4625
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
4626
	{
4627
		.name = "limit_in_bytes",
4628
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4629
		.write_string = mem_cgroup_write,
4630
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
4631
	},
4632 4633 4634 4635 4636 4637
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
4638 4639
	{
		.name = "failcnt",
4640
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4641
		.trigger = mem_cgroup_reset,
4642
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
4643
	},
4644 4645
	{
		.name = "stat",
4646
		.read_map = mem_control_stat_show,
4647
	},
4648 4649 4650 4651
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
4652 4653 4654 4655 4656
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
4657 4658 4659 4660 4661
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4662 4663 4664 4665 4666
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4667 4668
	{
		.name = "oom_control",
4669 4670
		.read_map = mem_cgroup_oom_control_read,
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4671 4672 4673 4674
		.register_event = mem_cgroup_oom_register_event,
		.unregister_event = mem_cgroup_oom_unregister_event,
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
4675 4676 4677 4678
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
		.open = mem_control_numa_stat_open,
4679
		.mode = S_IRUGO,
4680 4681
	},
#endif
B
Balbir Singh 已提交
4682 4683
};

4684 4685 4686 4687 4688 4689
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4690 4691
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
};

static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	if (!do_swap_account)
		return 0;
	return cgroup_add_files(cont, ss, memsw_cgroup_files,
				ARRAY_SIZE(memsw_cgroup_files));
};
#else
static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	return 0;
}
#endif

4727
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4728 4729
{
	struct mem_cgroup_per_node *pn;
4730
	struct mem_cgroup_per_zone *mz;
4731
	enum lru_list l;
4732
	int zone, tmp = node;
4733 4734 4735 4736 4737 4738 4739 4740
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4741 4742
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4743
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4744 4745
	if (!pn)
		return 1;
4746 4747 4748

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
4749 4750
		for_each_lru(l)
			INIT_LIST_HEAD(&mz->lists[l]);
4751
		mz->usage_in_excess = 0;
4752
		mz->on_tree = false;
4753
		mz->mem = memcg;
4754
	}
4755
	memcg->info.nodeinfo[node] = pn;
4756 4757 4758
	return 0;
}

4759
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4760
{
4761
	kfree(memcg->info.nodeinfo[node]);
4762 4763
}

4764 4765 4766
static struct mem_cgroup *mem_cgroup_alloc(void)
{
	struct mem_cgroup *mem;
4767
	int size = sizeof(struct mem_cgroup);
4768

4769
	/* Can be very big if MAX_NUMNODES is very big */
4770
	if (size < PAGE_SIZE)
4771
		mem = kzalloc(size, GFP_KERNEL);
4772
	else
4773
		mem = vzalloc(size);
4774

4775 4776 4777
	if (!mem)
		return NULL;

4778
	mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4779 4780
	if (!mem->stat)
		goto out_free;
4781
	spin_lock_init(&mem->pcp_counter_lock);
4782
	return mem;
4783 4784 4785 4786 4787 4788 4789

out_free:
	if (size < PAGE_SIZE)
		kfree(mem);
	else
		vfree(mem);
	return NULL;
4790 4791
}

4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802
/*
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
 */

4803
static void __mem_cgroup_free(struct mem_cgroup *memcg)
4804
{
K
KAMEZAWA Hiroyuki 已提交
4805 4806
	int node;

4807 4808
	mem_cgroup_remove_from_trees(memcg);
	free_css_id(&mem_cgroup_subsys, &memcg->css);
K
KAMEZAWA Hiroyuki 已提交
4809

K
KAMEZAWA Hiroyuki 已提交
4810
	for_each_node_state(node, N_POSSIBLE)
4811
		free_mem_cgroup_per_zone_info(memcg, node);
K
KAMEZAWA Hiroyuki 已提交
4812

4813
	free_percpu(memcg->stat);
4814
	if (sizeof(struct mem_cgroup) < PAGE_SIZE)
4815
		kfree(memcg);
4816
	else
4817
		vfree(memcg);
4818 4819
}

4820
static void mem_cgroup_get(struct mem_cgroup *memcg)
4821
{
4822
	atomic_inc(&memcg->refcnt);
4823 4824
}

4825
static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
4826
{
4827 4828 4829
	if (atomic_sub_and_test(count, &memcg->refcnt)) {
		struct mem_cgroup *parent = parent_mem_cgroup(memcg);
		__mem_cgroup_free(memcg);
4830 4831 4832
		if (parent)
			mem_cgroup_put(parent);
	}
4833 4834
}

4835
static void mem_cgroup_put(struct mem_cgroup *memcg)
4836
{
4837
	__mem_cgroup_put(memcg, 1);
4838 4839
}

4840 4841 4842
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
4843
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
4844
{
4845
	if (!memcg->res.parent)
4846
		return NULL;
4847
	return mem_cgroup_from_res_counter(memcg->res.parent, res);
4848
}
4849

4850 4851 4852
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static void __init enable_swap_cgroup(void)
{
4853
	if (!mem_cgroup_disabled() && really_do_swap_account)
4854 4855 4856 4857 4858 4859 4860 4861
		do_swap_account = 1;
}
#else
static void __init enable_swap_cgroup(void)
{
}
#endif

4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886
static int mem_cgroup_soft_limit_tree_init(void)
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

	for_each_node_state(node, N_POSSIBLE) {
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
		if (!rtpn)
			return 1;

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
	return 0;
}

L
Li Zefan 已提交
4887
static struct cgroup_subsys_state * __ref
B
Balbir Singh 已提交
4888 4889
mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
{
4890
	struct mem_cgroup *memcg, *parent;
K
KAMEZAWA Hiroyuki 已提交
4891
	long error = -ENOMEM;
4892
	int node;
B
Balbir Singh 已提交
4893

4894 4895
	memcg = mem_cgroup_alloc();
	if (!memcg)
K
KAMEZAWA Hiroyuki 已提交
4896
		return ERR_PTR(error);
4897

4898
	for_each_node_state(node, N_POSSIBLE)
4899
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
4900
			goto free_out;
4901

4902
	/* root ? */
4903
	if (cont->parent == NULL) {
4904
		int cpu;
4905
		enable_swap_cgroup();
4906
		parent = NULL;
4907
		root_mem_cgroup = memcg;
4908 4909
		if (mem_cgroup_soft_limit_tree_init())
			goto free_out;
4910 4911 4912 4913 4914
		for_each_possible_cpu(cpu) {
			struct memcg_stock_pcp *stock =
						&per_cpu(memcg_stock, cpu);
			INIT_WORK(&stock->work, drain_local_stock);
		}
4915
		hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
4916
	} else {
4917
		parent = mem_cgroup_from_cont(cont->parent);
4918 4919
		memcg->use_hierarchy = parent->use_hierarchy;
		memcg->oom_kill_disable = parent->oom_kill_disable;
4920
	}
4921

4922
	if (parent && parent->use_hierarchy) {
4923 4924
		res_counter_init(&memcg->res, &parent->res);
		res_counter_init(&memcg->memsw, &parent->memsw);
4925 4926 4927 4928 4929 4930 4931
		/*
		 * We increment refcnt of the parent to ensure that we can
		 * safely access it on res_counter_charge/uncharge.
		 * This refcnt will be decremented when freeing this
		 * mem_cgroup(see mem_cgroup_put).
		 */
		mem_cgroup_get(parent);
4932
	} else {
4933 4934
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
4935
	}
4936 4937 4938
	memcg->last_scanned_child = 0;
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
4939

K
KOSAKI Motohiro 已提交
4940
	if (parent)
4941 4942 4943 4944 4945
		memcg->swappiness = mem_cgroup_swappiness(parent);
	atomic_set(&memcg->refcnt, 1);
	memcg->move_charge_at_immigrate = 0;
	mutex_init(&memcg->thresholds_lock);
	return &memcg->css;
4946
free_out:
4947
	__mem_cgroup_free(memcg);
4948
	root_mem_cgroup = NULL;
K
KAMEZAWA Hiroyuki 已提交
4949
	return ERR_PTR(error);
B
Balbir Singh 已提交
4950 4951
}

4952
static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
4953 4954
					struct cgroup *cont)
{
4955
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4956

4957
	return mem_cgroup_force_empty(memcg, false);
4958 4959
}

B
Balbir Singh 已提交
4960 4961 4962
static void mem_cgroup_destroy(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
4963
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4964

4965
	mem_cgroup_put(memcg);
B
Balbir Singh 已提交
4966 4967 4968 4969 4970
}

static int mem_cgroup_populate(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
4971 4972 4973 4974 4975 4976 4977 4978
	int ret;

	ret = cgroup_add_files(cont, ss, mem_cgroup_files,
				ARRAY_SIZE(mem_cgroup_files));

	if (!ret)
		ret = register_memsw_files(cont, ss);
	return ret;
B
Balbir Singh 已提交
4979 4980
}

4981
#ifdef CONFIG_MMU
4982
/* Handlers for move charge at task migration. */
4983 4984
#define PRECHARGE_COUNT_AT_ONCE	256
static int mem_cgroup_do_precharge(unsigned long count)
4985
{
4986 4987
	int ret = 0;
	int batch_count = PRECHARGE_COUNT_AT_ONCE;
4988
	struct mem_cgroup *memcg = mc.to;
4989

4990
	if (mem_cgroup_is_root(memcg)) {
4991 4992 4993 4994 4995 4996 4997 4998
		mc.precharge += count;
		/* we don't need css_get for root */
		return ret;
	}
	/* try to charge at once */
	if (count > 1) {
		struct res_counter *dummy;
		/*
4999
		 * "memcg" cannot be under rmdir() because we've already checked
5000 5001 5002 5003
		 * by cgroup_lock_live_cgroup() that it is not removed and we
		 * are still under the same cgroup_mutex. So we can postpone
		 * css_get().
		 */
5004
		if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
5005
			goto one_by_one;
5006
		if (do_swap_account && res_counter_charge(&memcg->memsw,
5007
						PAGE_SIZE * count, &dummy)) {
5008
			res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024
			goto one_by_one;
		}
		mc.precharge += count;
		return ret;
	}
one_by_one:
	/* fall back to one by one charge */
	while (count--) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		if (!batch_count--) {
			batch_count = PRECHARGE_COUNT_AT_ONCE;
			cond_resched();
		}
5025 5026 5027
		ret = __mem_cgroup_try_charge(NULL,
					GFP_KERNEL, 1, &memcg, false);
		if (ret || !memcg)
5028 5029 5030 5031
			/* mem_cgroup_clear_mc() will do uncharge later */
			return -ENOMEM;
		mc.precharge++;
	}
5032 5033 5034 5035 5036 5037 5038 5039
	return ret;
}

/**
 * is_target_pte_for_mc - check a pte whether it is valid for move charge
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
5040
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
5041 5042 5043 5044 5045 5046
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
5047 5048 5049
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
5050 5051 5052 5053 5054
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
5055
	swp_entry_t	ent;
5056 5057 5058 5059 5060
};

enum mc_target_type {
	MC_TARGET_NONE,	/* not used */
	MC_TARGET_PAGE,
5061
	MC_TARGET_SWAP,
5062 5063
};

D
Daisuke Nishimura 已提交
5064 5065
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
5066
{
D
Daisuke Nishimura 已提交
5067
	struct page *page = vm_normal_page(vma, addr, ptent);
5068

D
Daisuke Nishimura 已提交
5069 5070 5071 5072 5073 5074
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
		/* we don't move shared anon */
		if (!move_anon() || page_mapcount(page) > 2)
			return NULL;
5075 5076
	} else if (!move_file())
		/* we ignore mapcount for file pages */
D
Daisuke Nishimura 已提交
5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094
		return NULL;
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	int usage_count;
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

	if (!move_anon() || non_swap_entry(ent))
		return NULL;
	usage_count = mem_cgroup_count_swap_user(ent, &page);
	if (usage_count > 1) { /* we don't move shared anon */
5095 5096
		if (page)
			put_page(page);
D
Daisuke Nishimura 已提交
5097
		return NULL;
5098
	}
D
Daisuke Nishimura 已提交
5099 5100 5101 5102 5103 5104
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}

5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct inode *inode;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
	if (!move_file())
		return NULL;

	inode = vma->vm_file->f_path.dentry->d_inode;
	mapping = vma->vm_file->f_mapping;
	if (pte_none(ptent))
		pgoff = linear_page_index(vma, addr);
	else /* pte_file(ptent) is true */
		pgoff = pte_to_pgoff(ptent);

	/* page is moved even if it's not RSS of this task(page-faulted). */
5126 5127 5128 5129 5130 5131
	page = find_get_page(mapping, pgoff);

#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
	if (radix_tree_exceptional_entry(page)) {
		swp_entry_t swap = radix_to_swp_entry(page);
5132
		if (do_swap_account)
5133 5134
			*entry = swap;
		page = find_get_page(&swapper_space, swap.val);
5135
	}
5136
#endif
5137 5138 5139
	return page;
}

D
Daisuke Nishimura 已提交
5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151
static int is_target_pte_for_mc(struct vm_area_struct *vma,
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
	int ret = 0;
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
5152 5153
	else if (pte_none(ptent) || pte_file(ptent))
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
5154 5155 5156

	if (!page && !ent.val)
		return 0;
5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
	if (page) {
		pc = lookup_page_cgroup(page);
		/*
		 * Do only loose check w/o page_cgroup lock.
		 * mem_cgroup_move_account() checks the pc is valid or not under
		 * the lock.
		 */
		if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
5172 5173
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
5174 5175 5176 5177
			css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189
	}
	return ret;
}

static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

5190 5191
	split_huge_page_pmd(walk->mm, pmd);

5192 5193 5194 5195 5196 5197 5198
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
		if (is_target_pte_for_mc(vma, addr, *pte, NULL))
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

5199 5200 5201
	return 0;
}

5202 5203 5204 5205 5206
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;
	struct vm_area_struct *vma;

5207
	down_read(&mm->mmap_sem);
5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		struct mm_walk mem_cgroup_count_precharge_walk = {
			.pmd_entry = mem_cgroup_count_precharge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		walk_page_range(vma->vm_start, vma->vm_end,
					&mem_cgroup_count_precharge_walk);
	}
5219
	up_read(&mm->mmap_sem);
5220 5221 5222 5223 5224 5225 5226 5227 5228

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
5229 5230 5231 5232 5233
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
5234 5235
}

5236 5237
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
5238
{
5239 5240 5241
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

5242
	/* we must uncharge all the leftover precharges from mc.to */
5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253
	if (mc.precharge) {
		__mem_cgroup_cancel_charge(mc.to, mc.precharge);
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
		mc.moved_charge = 0;
5254
	}
5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
		if (!mem_cgroup_is_root(mc.from))
			res_counter_uncharge(&mc.from->memsw,
						PAGE_SIZE * mc.moved_swap);
		__mem_cgroup_put(mc.from, mc.moved_swap);

		if (!mem_cgroup_is_root(mc.to)) {
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			res_counter_uncharge(&mc.to->res,
						PAGE_SIZE * mc.moved_swap);
		}
		/* we've already done mem_cgroup_get(mc.to) */
		mc.moved_swap = 0;
	}
5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	struct mem_cgroup *from = mc.from;

	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
5289
	spin_lock(&mc.lock);
5290 5291
	mc.from = NULL;
	mc.to = NULL;
5292
	spin_unlock(&mc.lock);
5293
	mem_cgroup_end_move(from);
5294 5295
}

5296 5297
static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
5298
				struct task_struct *p)
5299 5300
{
	int ret = 0;
5301
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
5302

5303
	if (memcg->move_charge_at_immigrate) {
5304 5305 5306
		struct mm_struct *mm;
		struct mem_cgroup *from = mem_cgroup_from_task(p);

5307
		VM_BUG_ON(from == memcg);
5308 5309 5310 5311 5312

		mm = get_task_mm(p);
		if (!mm)
			return 0;
		/* We move charges only when we move a owner of the mm */
5313 5314 5315 5316
		if (mm->owner == p) {
			VM_BUG_ON(mc.from);
			VM_BUG_ON(mc.to);
			VM_BUG_ON(mc.precharge);
5317
			VM_BUG_ON(mc.moved_charge);
5318
			VM_BUG_ON(mc.moved_swap);
5319
			mem_cgroup_start_move(from);
5320
			spin_lock(&mc.lock);
5321
			mc.from = from;
5322
			mc.to = memcg;
5323
			spin_unlock(&mc.lock);
5324
			/* We set mc.moving_task later */
5325 5326 5327 5328

			ret = mem_cgroup_precharge_mc(mm);
			if (ret)
				mem_cgroup_clear_mc();
5329 5330
		}
		mmput(mm);
5331 5332 5333 5334 5335 5336
	}
	return ret;
}

static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
5337
				struct task_struct *p)
5338
{
5339
	mem_cgroup_clear_mc();
5340 5341
}

5342 5343 5344
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
5345
{
5346 5347 5348 5349 5350
	int ret = 0;
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

5351
	split_huge_page_pmd(walk->mm, pmd);
5352 5353 5354 5355 5356 5357 5358 5359
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
		union mc_target target;
		int type;
		struct page *page;
		struct page_cgroup *pc;
5360
		swp_entry_t ent;
5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371

		if (!mc.precharge)
			break;

		type = is_target_pte_for_mc(vma, addr, ptent, &target);
		switch (type) {
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
			pc = lookup_page_cgroup(page);
5372 5373
			if (!mem_cgroup_move_account(page, 1, pc,
						     mc.from, mc.to, false)) {
5374
				mc.precharge--;
5375 5376
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
5377 5378 5379 5380 5381
			}
			putback_lru_page(page);
put:			/* is_target_pte_for_mc() gets the page */
			put_page(page);
			break;
5382 5383
		case MC_TARGET_SWAP:
			ent = target.ent;
5384 5385
			if (!mem_cgroup_move_swap_account(ent,
						mc.from, mc.to, false)) {
5386
				mc.precharge--;
5387 5388 5389
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
5390
			break;
5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
5405
		ret = mem_cgroup_do_precharge(1);
5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	lru_add_drain_all();
5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		int ret;
		struct mm_walk mem_cgroup_move_charge_walk = {
			.pmd_entry = mem_cgroup_move_charge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		ret = walk_page_range(vma->vm_start, vma->vm_end,
						&mem_cgroup_move_charge_walk);
		if (ret)
			/*
			 * means we have consumed all precharges and failed in
			 * doing additional charge. Just abandon here.
			 */
			break;
	}
5449
	up_read(&mm->mmap_sem);
5450 5451
}

B
Balbir Singh 已提交
5452 5453 5454
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
5455
				struct task_struct *p)
B
Balbir Singh 已提交
5456
{
5457
	struct mm_struct *mm = get_task_mm(p);
5458 5459

	if (mm) {
5460 5461 5462
		if (mc.to)
			mem_cgroup_move_charge(mm);
		put_swap_token(mm);
5463 5464
		mmput(mm);
	}
5465 5466
	if (mc.to)
		mem_cgroup_clear_mc();
B
Balbir Singh 已提交
5467
}
5468 5469 5470
#else	/* !CONFIG_MMU */
static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
5471
				struct task_struct *p)
5472 5473 5474 5475 5476
{
	return 0;
}
static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
				struct cgroup *cgroup,
5477
				struct task_struct *p)
5478 5479 5480 5481 5482
{
}
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
5483
				struct task_struct *p)
5484 5485 5486
{
}
#endif
B
Balbir Singh 已提交
5487

B
Balbir Singh 已提交
5488 5489 5490 5491
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
	.create = mem_cgroup_create,
5492
	.pre_destroy = mem_cgroup_pre_destroy,
B
Balbir Singh 已提交
5493 5494
	.destroy = mem_cgroup_destroy,
	.populate = mem_cgroup_populate,
5495 5496
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
5497
	.attach = mem_cgroup_move_task,
5498
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
5499
	.use_id = 1,
B
Balbir Singh 已提交
5500
};
5501 5502

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
5503 5504 5505
static int __init enable_swap_account(char *s)
{
	/* consider enabled if no parameter or 1 is given */
5506
	if (!strcmp(s, "1"))
5507
		really_do_swap_account = 1;
5508
	else if (!strcmp(s, "0"))
5509 5510 5511
		really_do_swap_account = 0;
	return 1;
}
5512
__setup("swapaccount=", enable_swap_account);
5513 5514

#endif