memcontrol.c 146.3 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
B
Balbir Singh 已提交
13 14 15 16 17 18 19 20 21 22 23 24 25 26
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
27
#include <linux/mm.h>
28
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
29
#include <linux/pagemap.h>
30
#include <linux/smp.h>
31
#include <linux/page-flags.h>
32
#include <linux/backing-dev.h>
33 34
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
35
#include <linux/limits.h>
36
#include <linux/export.h>
37
#include <linux/mutex.h>
38
#include <linux/rbtree.h>
39
#include <linux/slab.h>
40
#include <linux/swap.h>
41
#include <linux/swapops.h>
42
#include <linux/spinlock.h>
43 44
#include <linux/eventfd.h>
#include <linux/sort.h>
45
#include <linux/fs.h>
46
#include <linux/seq_file.h>
47
#include <linux/vmalloc.h>
48
#include <linux/mm_inline.h>
49
#include <linux/page_cgroup.h>
50
#include <linux/cpu.h>
51
#include <linux/oom.h>
K
KAMEZAWA Hiroyuki 已提交
52
#include "internal.h"
G
Glauber Costa 已提交
53
#include <net/sock.h>
M
Michal Hocko 已提交
54
#include <net/ip.h>
G
Glauber Costa 已提交
55
#include <net/tcp_memcontrol.h>
B
Balbir Singh 已提交
56

57 58
#include <asm/uaccess.h>

59 60
#include <trace/events/vmscan.h>

61 62
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
#define MEM_CGROUP_RECLAIM_RETRIES	5
63
static struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
64

A
Andrew Morton 已提交
65
#ifdef CONFIG_MEMCG_SWAP
L
Li Zefan 已提交
66
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
67
int do_swap_account __read_mostly;
68 69

/* for remember boot option*/
A
Andrew Morton 已提交
70
#ifdef CONFIG_MEMCG_SWAP_ENABLED
71 72 73 74 75
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata = 0;
#endif

76
#else
77
#define do_swap_account		0
78 79 80
#endif


81 82 83 84 85 86 87 88
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
89
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as anon rss */
90
	MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
91
	MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
92 93 94
	MEM_CGROUP_STAT_NSTATS,
};

95 96 97 98 99 100 101
static const char * const mem_cgroup_stat_names[] = {
	"cache",
	"rss",
	"mapped_file",
	"swap",
};

102 103 104
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
105 106
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
107 108
	MEM_CGROUP_EVENTS_NSTATS,
};
109 110 111 112 113 114 115 116

static const char * const mem_cgroup_events_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

117 118 119 120 121 122 123 124 125
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
126
	MEM_CGROUP_TARGET_NUMAINFO,
127 128
	MEM_CGROUP_NTARGETS,
};
129 130 131
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
132

133
struct mem_cgroup_stat_cpu {
134
	long count[MEM_CGROUP_STAT_NSTATS];
135
	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
136
	unsigned long nr_page_events;
137
	unsigned long targets[MEM_CGROUP_NTARGETS];
138 139
};

140 141 142 143 144 145 146
struct mem_cgroup_reclaim_iter {
	/* css_id of the last scanned hierarchy member */
	int position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

147 148 149 150
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
151
	struct lruvec		lruvec;
152
	unsigned long		lru_size[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
153

154 155
	struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];

156 157 158 159
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
160
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
161
						/* use container_of	   */
162 163 164 165 166 167 168 169 170 171
};

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
	struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
};

172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

192 193 194 195 196
struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	u64 threshold;
};

K
KAMEZAWA Hiroyuki 已提交
197
/* For threshold */
198
struct mem_cgroup_threshold_ary {
199
	/* An array index points to threshold just below or equal to usage. */
200
	int current_threshold;
201 202 203 204 205
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};
206 207 208 209 210 211 212 213 214 215 216 217

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

K
KAMEZAWA Hiroyuki 已提交
218 219 220 221 222
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
223

224 225
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
226

B
Balbir Singh 已提交
227 228 229 230 231 232 233
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
234 235 236
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
237 238 239 240 241 242 243
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261

	union {
		/*
		 * the counter to account for mem+swap usage.
		 */
		struct res_counter memsw;

		/*
		 * rcu_freeing is used only when freeing struct mem_cgroup,
		 * so put it into a union to avoid wasting more memory.
		 * It must be disjoint from the css field.  It could be
		 * in a union with the res field, but res plays a much
		 * larger part in mem_cgroup life than memsw, and might
		 * be of interest, even at time of free, when debugging.
		 * So share rcu_head with the less interesting memsw.
		 */
		struct rcu_head rcu_freeing;
		/*
262 263
		 * We also need some space for a worker in deferred freeing.
		 * By the time we call it, rcu_freeing is no longer in use.
264 265 266 267
		 */
		struct work_struct work_freeing;
	};

268 269 270 271
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 */
272
	struct mem_cgroup_lru_info info;
273 274 275
	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
276 277
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
278
#endif
279 280 281 282
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
283 284 285 286

	bool		oom_lock;
	atomic_t	under_oom;

287
	atomic_t	refcnt;
288

289
	int	swappiness;
290 291
	/* OOM-Killer disable */
	int		oom_kill_disable;
K
KOSAKI Motohiro 已提交
292

293 294 295
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

296 297 298 299
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
300
	struct mem_cgroup_thresholds thresholds;
301

302
	/* thresholds for mem+swap usage. RCU-protected */
303
	struct mem_cgroup_thresholds memsw_thresholds;
304

K
KAMEZAWA Hiroyuki 已提交
305 306
	/* For oom notifier event fd */
	struct list_head oom_notify;
307

308 309 310 311 312
	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long 	move_charge_at_immigrate;
313 314 315 316
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t	moving_account;
317 318
	/* taken only while moving_account > 0 */
	spinlock_t	move_lock;
319
	/*
320
	 * percpu counter.
321
	 */
322
	struct mem_cgroup_stat_cpu __percpu *stat;
323 324 325 326 327 328
	/*
	 * used when a cpu is offlined or other synchronizations
	 * See mem_cgroup_read_stat().
	 */
	struct mem_cgroup_stat_cpu nocpu_base;
	spinlock_t pcp_counter_lock;
G
Glauber Costa 已提交
329

M
Michal Hocko 已提交
330
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
G
Glauber Costa 已提交
331 332
	struct tcp_memcontrol tcp_mem;
#endif
B
Balbir Singh 已提交
333 334
};

335 336 337 338 339 340
/* Stuffs for move charges at task migration. */
/*
 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
 * left-shifted bitmap of these types.
 */
enum move_type {
341
	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */
342
	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */
343 344 345
	NR_MOVE_TYPE,
};

346 347
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
348
	spinlock_t	  lock; /* for from, to */
349 350 351
	struct mem_cgroup *from;
	struct mem_cgroup *to;
	unsigned long precharge;
352
	unsigned long moved_charge;
353
	unsigned long moved_swap;
354 355 356
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
357
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
358 359
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
360

D
Daisuke Nishimura 已提交
361 362 363 364 365 366
static bool move_anon(void)
{
	return test_bit(MOVE_CHARGE_TYPE_ANON,
					&mc.to->move_charge_at_immigrate);
}

367 368 369 370 371 372
static bool move_file(void)
{
	return test_bit(MOVE_CHARGE_TYPE_FILE,
					&mc.to->move_charge_at_immigrate);
}

373 374 375 376
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
377 378
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
379

380 381
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
382
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
383
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
384
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
385 386 387
	NR_CHARGE_TYPE,
};

388
/* for encoding cft->private value on file */
389 390 391
#define _MEM			(0)
#define _MEMSWAP		(1)
#define _OOM_TYPE		(2)
392 393
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
394
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
395 396
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
397

398 399 400 401 402 403 404 405
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)

406 407
static void mem_cgroup_get(struct mem_cgroup *memcg);
static void mem_cgroup_put(struct mem_cgroup *memcg);
G
Glauber Costa 已提交
408

409 410 411 412 413 414
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
{
	return container_of(s, struct mem_cgroup, css);
}

415 416 417 418 419
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

G
Glauber Costa 已提交
420
/* Writing them here to avoid exposing memcg's inner layout */
M
Michal Hocko 已提交
421
#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
G
Glauber Costa 已提交
422 423 424

void sock_update_memcg(struct sock *sk)
{
425
	if (mem_cgroup_sockets_enabled) {
G
Glauber Costa 已提交
426
		struct mem_cgroup *memcg;
427
		struct cg_proto *cg_proto;
G
Glauber Costa 已提交
428 429 430

		BUG_ON(!sk->sk_prot->proto_cgroup);

431 432 433 434 435 436 437 438 439 440 441 442 443 444
		/* Socket cloning can throw us here with sk_cgrp already
		 * filled. It won't however, necessarily happen from
		 * process context. So the test for root memcg given
		 * the current task's memcg won't help us in this case.
		 *
		 * Respecting the original socket's memcg is a better
		 * decision in this case.
		 */
		if (sk->sk_cgrp) {
			BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
			mem_cgroup_get(sk->sk_cgrp->memcg);
			return;
		}

G
Glauber Costa 已提交
445 446
		rcu_read_lock();
		memcg = mem_cgroup_from_task(current);
447 448
		cg_proto = sk->sk_prot->proto_cgroup(memcg);
		if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
G
Glauber Costa 已提交
449
			mem_cgroup_get(memcg);
450
			sk->sk_cgrp = cg_proto;
G
Glauber Costa 已提交
451 452 453 454 455 456 457 458
		}
		rcu_read_unlock();
	}
}
EXPORT_SYMBOL(sock_update_memcg);

void sock_release_memcg(struct sock *sk)
{
459
	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
G
Glauber Costa 已提交
460 461 462 463 464 465
		struct mem_cgroup *memcg;
		WARN_ON(!sk->sk_cgrp->memcg);
		memcg = sk->sk_cgrp->memcg;
		mem_cgroup_put(memcg);
	}
}
G
Glauber Costa 已提交
466 467 468 469 470 471 472 473 474

struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg || mem_cgroup_is_root(memcg))
		return NULL;

	return &memcg->tcp_mem.cg_proto;
}
EXPORT_SYMBOL(tcp_proto_cgroup);
G
Glauber Costa 已提交
475

476 477 478 479 480 481 482 483 484 485 486 487
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
	if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
		return;
	static_key_slow_dec(&memcg_socket_limit_enabled);
}
#else
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
}
#endif

488
static void drain_all_stock_async(struct mem_cgroup *memcg);
489

490
static struct mem_cgroup_per_zone *
491
mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
492
{
493
	return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
494 495
}

496
struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
497
{
498
	return &memcg->css;
499 500
}

501
static struct mem_cgroup_per_zone *
502
page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
503
{
504 505
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
506

507
	return mem_cgroup_zoneinfo(memcg, nid, zid);
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
526
__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
527
				struct mem_cgroup_per_zone *mz,
528 529
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
530 531 532 533 534 535 536 537
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

538 539 540
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
557 558 559
}

static void
560
__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
561 562 563 564 565 566 567 568 569
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

570
static void
571
mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
572 573 574 575
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
576
	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
577 578 579 580
	spin_unlock(&mctz->lock);
}


581
static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
582
{
583
	unsigned long long excess;
584 585
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
586 587
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
588 589 590
	mctz = soft_limit_tree_from_page(page);

	/*
591 592
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
593
	 */
594 595 596
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		mz = mem_cgroup_zoneinfo(memcg, nid, zid);
		excess = res_counter_soft_limit_excess(&memcg->res);
597 598 599 600
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
601
		if (excess || mz->on_tree) {
602 603 604
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
605
				__mem_cgroup_remove_exceeded(memcg, mz, mctz);
606
			/*
607 608
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
609
			 */
610
			__mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
611 612
			spin_unlock(&mctz->lock);
		}
613 614 615
	}
}

616
static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
617 618 619 620 621
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

B
Bob Liu 已提交
622
	for_each_node(node) {
623
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
624
			mz = mem_cgroup_zoneinfo(memcg, node, zone);
625
			mctz = soft_limit_tree_node_zone(node, zone);
626
			mem_cgroup_remove_exceeded(memcg, mz, mctz);
627 628 629 630
		}
	}
}

631 632 633 634
static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
635
	struct mem_cgroup_per_zone *mz;
636 637

retry:
638
	mz = NULL;
639 640 641 642 643 644 645 646 647 648
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
649 650 651
	__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
		!css_tryget(&mz->memcg->css))
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
687
static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
688
				 enum mem_cgroup_stat_index idx)
689
{
690
	long val = 0;
691 692
	int cpu;

693 694
	get_online_cpus();
	for_each_online_cpu(cpu)
695
		val += per_cpu(memcg->stat->count[idx], cpu);
696
#ifdef CONFIG_HOTPLUG_CPU
697 698 699
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.count[idx];
	spin_unlock(&memcg->pcp_counter_lock);
700 701
#endif
	put_online_cpus();
702 703 704
	return val;
}

705
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
706 707 708
					 bool charge)
{
	int val = (charge) ? 1 : -1;
709
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
710 711
}

712
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
713 714 715 716 717 718
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

	for_each_online_cpu(cpu)
719
		val += per_cpu(memcg->stat->events[idx], cpu);
720
#ifdef CONFIG_HOTPLUG_CPU
721 722 723
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.events[idx];
	spin_unlock(&memcg->pcp_counter_lock);
724 725 726 727
#endif
	return val;
}

728
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
729
					 bool anon, int nr_pages)
730
{
731 732
	preempt_disable();

733 734 735 736 737 738
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
	if (anon)
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
739
				nr_pages);
740
	else
741
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
742
				nr_pages);
743

744 745
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
746
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
747
	else {
748
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
749 750
		nr_pages = -nr_pages; /* for event */
	}
751

752
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
753

754
	preempt_enable();
755 756
}

757
unsigned long
758
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
759 760 761 762 763 764 765 766
{
	struct mem_cgroup_per_zone *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	return mz->lru_size[lru];
}

static unsigned long
767
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
768
			unsigned int lru_mask)
769 770
{
	struct mem_cgroup_per_zone *mz;
H
Hugh Dickins 已提交
771
	enum lru_list lru;
772 773
	unsigned long ret = 0;

774
	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
775

H
Hugh Dickins 已提交
776 777 778
	for_each_lru(lru) {
		if (BIT(lru) & lru_mask)
			ret += mz->lru_size[lru];
779 780 781 782 783
	}
	return ret;
}

static unsigned long
784
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
785 786
			int nid, unsigned int lru_mask)
{
787 788 789
	u64 total = 0;
	int zid;

790
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
791 792
		total += mem_cgroup_zone_nr_lru_pages(memcg,
						nid, zid, lru_mask);
793

794 795
	return total;
}
796

797
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
798
			unsigned int lru_mask)
799
{
800
	int nid;
801 802
	u64 total = 0;

803
	for_each_node_state(nid, N_HIGH_MEMORY)
804
		total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
805
	return total;
806 807
}

808 809
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
810 811 812
{
	unsigned long val, next;

813
	val = __this_cpu_read(memcg->stat->nr_page_events);
814
	next = __this_cpu_read(memcg->stat->targets[target]);
815
	/* from time_after() in jiffies.h */
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
	if ((long)next - (long)val < 0) {
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
		__this_cpu_write(memcg->stat->targets[target], next);
		return true;
832
	}
833
	return false;
834 835 836 837 838 839
}

/*
 * Check events in order.
 *
 */
840
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
841
{
842
	preempt_disable();
843
	/* threshold event is triggered in finer grain than soft limit */
844 845
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
846 847
		bool do_softlimit;
		bool do_numainfo __maybe_unused;
848 849 850 851 852 853 854 855 856

		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
		preempt_enable();

857
		mem_cgroup_threshold(memcg);
858
		if (unlikely(do_softlimit))
859
			mem_cgroup_update_tree(memcg, page);
860
#if MAX_NUMNODES > 1
861
		if (unlikely(do_numainfo))
862
			atomic_inc(&memcg->numainfo_events);
863
#endif
864 865
	} else
		preempt_enable();
866 867
}

G
Glauber Costa 已提交
868
struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
869
{
870 871
	return mem_cgroup_from_css(
		cgroup_subsys_state(cont, mem_cgroup_subsys_id));
B
Balbir Singh 已提交
872 873
}

874
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
875
{
876 877 878 879 880 881 882 883
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

884
	return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
885 886
}

887
struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
888
{
889
	struct mem_cgroup *memcg = NULL;
890 891 892

	if (!mm)
		return NULL;
893 894 895 896 897 898 899
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
900 901
		memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!memcg))
902
			break;
903
	} while (!css_tryget(&memcg->css));
904
	rcu_read_unlock();
905
	return memcg;
906 907
}

908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
 * Reclaimers can specify a zone and a priority level in @reclaim to
 * divide up the memcgs in the hierarchy among all concurrent
 * reclaimers operating on the same zone and priority.
 */
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
				   struct mem_cgroup *prev,
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
928
{
929 930
	struct mem_cgroup *memcg = NULL;
	int id = 0;
931

932 933 934
	if (mem_cgroup_disabled())
		return NULL;

935 936
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
937

938 939
	if (prev && !reclaim)
		id = css_id(&prev->css);
K
KAMEZAWA Hiroyuki 已提交
940

941 942
	if (prev && prev != root)
		css_put(&prev->css);
K
KAMEZAWA Hiroyuki 已提交
943

944 945 946 947 948
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
			return NULL;
		return root;
	}
K
KAMEZAWA Hiroyuki 已提交
949

950
	while (!memcg) {
951
		struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
952
		struct cgroup_subsys_state *css;
953

954 955 956 957 958 959 960 961 962 963 964
		if (reclaim) {
			int nid = zone_to_nid(reclaim->zone);
			int zid = zone_idx(reclaim->zone);
			struct mem_cgroup_per_zone *mz;

			mz = mem_cgroup_zoneinfo(root, nid, zid);
			iter = &mz->reclaim_iter[reclaim->priority];
			if (prev && reclaim->generation != iter->generation)
				return NULL;
			id = iter->position;
		}
K
KAMEZAWA Hiroyuki 已提交
965

966 967 968 969
		rcu_read_lock();
		css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
		if (css) {
			if (css == &root->css || css_tryget(css))
970
				memcg = mem_cgroup_from_css(css);
971 972
		} else
			id = 0;
K
KAMEZAWA Hiroyuki 已提交
973 974
		rcu_read_unlock();

975 976 977 978 979 980 981
		if (reclaim) {
			iter->position = id;
			if (!css)
				iter->generation++;
			else if (!prev && memcg)
				reclaim->generation = iter->generation;
		}
982 983 984 985 986

		if (prev && !css)
			return NULL;
	}
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
987
}
K
KAMEZAWA Hiroyuki 已提交
988

989 990 991 992 993 994 995
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
996 997 998 999 1000 1001
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
1002

1003 1004 1005 1006 1007 1008
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
1009
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
1010
	     iter != NULL;				\
1011
	     iter = mem_cgroup_iter(root, iter, NULL))
1012

1013
#define for_each_mem_cgroup(iter)			\
1014
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
1015
	     iter != NULL;				\
1016
	     iter = mem_cgroup_iter(NULL, iter, NULL))
K
KAMEZAWA Hiroyuki 已提交
1017

1018 1019
void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
{
1020
	struct mem_cgroup *memcg;
1021 1022 1023 1024 1025

	if (!mm)
		return;

	rcu_read_lock();
1026 1027
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
1028 1029 1030 1031
		goto out;

	switch (idx) {
	case PGFAULT:
1032 1033 1034 1035
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
		break;
	case PGMAJFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1036 1037 1038 1039 1040 1041 1042 1043 1044
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
}
EXPORT_SYMBOL(mem_cgroup_count_vm_event);

1045 1046 1047
/**
 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
 * @zone: zone of the wanted lruvec
1048
 * @memcg: memcg of the wanted lruvec
1049 1050 1051 1052 1053 1054 1055 1056 1057
 *
 * Returns the lru list vector holding pages for the given @zone and
 * @mem.  This can be the global zone lruvec, if the memory controller
 * is disabled.
 */
struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
				      struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_zone *mz;
1058
	struct lruvec *lruvec;
1059

1060 1061 1062 1063
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1064 1065

	mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
1076 1077
}

K
KAMEZAWA Hiroyuki 已提交
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
1091

1092
/**
1093
 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1094
 * @page: the page
1095
 * @zone: zone of the page
1096
 */
1097
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
K
KAMEZAWA Hiroyuki 已提交
1098 1099
{
	struct mem_cgroup_per_zone *mz;
1100 1101
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;
1102
	struct lruvec *lruvec;
1103

1104 1105 1106 1107
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1108

K
KAMEZAWA Hiroyuki 已提交
1109
	pc = lookup_page_cgroup(page);
1110
	memcg = pc->mem_cgroup;
1111 1112

	/*
1113
	 * Surreptitiously switch any uncharged offlist page to root:
1114 1115 1116 1117 1118 1119 1120
	 * an uncharged page off lru does nothing to secure
	 * its former mem_cgroup from sudden removal.
	 *
	 * Our caller holds lru_lock, and PageCgroupUsed is updated
	 * under page_cgroup lock: between them, they make all uses
	 * of pc->mem_cgroup safe.
	 */
1121
	if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1122 1123
		pc->mem_cgroup = memcg = root_mem_cgroup;

1124
	mz = page_cgroup_zoneinfo(memcg, page);
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1135
}
1136

1137
/**
1138 1139 1140 1141
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
 * @nr_pages: positive when adding or negative when removing
1142
 *
1143 1144
 * This function must be called when a page is added to or removed from an
 * lru list.
1145
 */
1146 1147
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
				int nr_pages)
1148 1149
{
	struct mem_cgroup_per_zone *mz;
1150
	unsigned long *lru_size;
1151 1152 1153 1154

	if (mem_cgroup_disabled())
		return;

1155 1156 1157 1158
	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	lru_size = mz->lru_size + lru;
	*lru_size += nr_pages;
	VM_BUG_ON((long)(*lru_size) < 0);
K
KAMEZAWA Hiroyuki 已提交
1159
}
1160

1161
/*
1162
 * Checks whether given mem is same or in the root_mem_cgroup's
1163 1164
 * hierarchy subtree
 */
1165 1166
bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				  struct mem_cgroup *memcg)
1167
{
1168 1169
	if (root_memcg == memcg)
		return true;
1170
	if (!root_memcg->use_hierarchy || !memcg)
1171
		return false;
1172 1173 1174 1175 1176 1177 1178 1179
	return css_is_ancestor(&memcg->css, &root_memcg->css);
}

static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				       struct mem_cgroup *memcg)
{
	bool ret;

1180
	rcu_read_lock();
1181
	ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1182 1183
	rcu_read_unlock();
	return ret;
1184 1185
}

1186
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
1187 1188
{
	int ret;
1189
	struct mem_cgroup *curr = NULL;
1190
	struct task_struct *p;
1191

1192
	p = find_lock_task_mm(task);
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
	if (p) {
		curr = try_get_mem_cgroup_from_mm(p->mm);
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
		task_lock(task);
		curr = mem_cgroup_from_task(task);
		if (curr)
			css_get(&curr->css);
		task_unlock(task);
	}
1208 1209
	if (!curr)
		return 0;
1210
	/*
1211
	 * We should check use_hierarchy of "memcg" not "curr". Because checking
1212
	 * use_hierarchy of "curr" here make this function true if hierarchy is
1213 1214
	 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
	 * hierarchy(even if use_hierarchy is disabled in "memcg").
1215
	 */
1216
	ret = mem_cgroup_same_or_subtree(memcg, curr);
1217
	css_put(&curr->css);
1218 1219 1220
	return ret;
}

1221
int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1222
{
1223
	unsigned long inactive_ratio;
1224
	unsigned long inactive;
1225
	unsigned long active;
1226
	unsigned long gb;
1227

1228 1229
	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1230

1231 1232 1233 1234 1235 1236
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

1237
	return inactive * inactive_ratio < active;
1238 1239
}

1240
int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec)
1241 1242 1243 1244
{
	unsigned long active;
	unsigned long inactive;

1245 1246
	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_FILE);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_FILE);
1247 1248 1249 1250

	return (active > inactive);
}

1251 1252 1253
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

1254
/**
1255
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1256
 * @memcg: the memory cgroup
1257
 *
1258
 * Returns the maximum amount of memory @mem can be charged with, in
1259
 * pages.
1260
 */
1261
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1262
{
1263 1264
	unsigned long long margin;

1265
	margin = res_counter_margin(&memcg->res);
1266
	if (do_swap_account)
1267
		margin = min(margin, res_counter_margin(&memcg->memsw));
1268
	return margin >> PAGE_SHIFT;
1269 1270
}

1271
int mem_cgroup_swappiness(struct mem_cgroup *memcg)
K
KOSAKI Motohiro 已提交
1272 1273 1274 1275 1276 1277 1278
{
	struct cgroup *cgrp = memcg->css.cgroup;

	/* root ? */
	if (cgrp->parent == NULL)
		return vm_swappiness;

1279
	return memcg->swappiness;
K
KOSAKI Motohiro 已提交
1280 1281
}

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
/*
 * memcg->moving_account is used for checking possibility that some thread is
 * calling move_account(). When a thread on CPU-A starts moving pages under
 * a memcg, other threads should check memcg->moving_account under
 * rcu_read_lock(), like this:
 *
 *         CPU-A                                    CPU-B
 *                                              rcu_read_lock()
 *         memcg->moving_account+1              if (memcg->mocing_account)
 *                                                   take heavy locks.
 *         synchronize_rcu()                    update something.
 *                                              rcu_read_unlock()
 *         start move here.
 */
1296 1297 1298 1299

/* for quick checking without looking up memcg */
atomic_t memcg_moving __read_mostly;

1300
static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1301
{
1302
	atomic_inc(&memcg_moving);
1303
	atomic_inc(&memcg->moving_account);
1304 1305 1306
	synchronize_rcu();
}

1307
static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1308
{
1309 1310 1311 1312
	/*
	 * Now, mem_cgroup_clear_mc() may call this function with NULL.
	 * We check NULL in callee rather than caller.
	 */
1313 1314
	if (memcg) {
		atomic_dec(&memcg_moving);
1315
		atomic_dec(&memcg->moving_account);
1316
	}
1317
}
1318

1319 1320 1321
/*
 * 2 routines for checking "mem" is under move_account() or not.
 *
1322 1323
 * mem_cgroup_stolen() -  checking whether a cgroup is mc.from or not. This
 *			  is used for avoiding races in accounting.  If true,
1324 1325 1326 1327 1328 1329 1330
 *			  pc->mem_cgroup may be overwritten.
 *
 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
 *			  under hierarchy of moving cgroups. This is for
 *			  waiting at hith-memory prressure caused by "move".
 */

1331
static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1332 1333
{
	VM_BUG_ON(!rcu_read_lock_held());
1334
	return atomic_read(&memcg->moving_account) > 0;
1335
}
1336

1337
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1338
{
1339 1340
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1341
	bool ret = false;
1342 1343 1344 1345 1346 1347 1348 1349 1350
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1351

1352 1353
	ret = mem_cgroup_same_or_subtree(memcg, from)
		|| mem_cgroup_same_or_subtree(memcg, to);
1354 1355
unlock:
	spin_unlock(&mc.lock);
1356 1357 1358
	return ret;
}

1359
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1360 1361
{
	if (mc.moving_task && current != mc.moving_task) {
1362
		if (mem_cgroup_under_move(memcg)) {
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1375 1376 1377 1378
/*
 * Take this lock when
 * - a code tries to modify page's memcg while it's USED.
 * - a code tries to modify page state accounting in a memcg.
1379
 * see mem_cgroup_stolen(), too.
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
 */
static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
				  unsigned long *flags)
{
	spin_lock_irqsave(&memcg->move_lock, *flags);
}

static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
				unsigned long *flags)
{
	spin_unlock_irqrestore(&memcg->move_lock, *flags);
}

1393
/**
1394
 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;

1413
	if (!memcg || !p)
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
		return;

	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	printk(KERN_INFO "Task in %s killed", memcg_name);

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
	printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
done:

	printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
	printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
		"failcnt %llu\n",
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
}

1459 1460 1461 1462
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1463
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1464 1465
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1466 1467
	struct mem_cgroup *iter;

1468
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1469
		num++;
1470 1471 1472
	return num;
}

D
David Rientjes 已提交
1473 1474 1475
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1476
static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1477 1478 1479
{
	u64 limit;

1480 1481
	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);

D
David Rientjes 已提交
1482
	/*
1483
	 * Do not consider swap space if we cannot swap due to swappiness
D
David Rientjes 已提交
1484
	 */
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
	if (mem_cgroup_swappiness(memcg)) {
		u64 memsw;

		limit += total_swap_pages << PAGE_SHIFT;
		memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);

		/*
		 * If memsw is finite and limits the amount of swap space
		 * available to this memcg, return that limit.
		 */
		limit = min(limit, memsw);
	}

	return limit;
D
David Rientjes 已提交
1499 1500
}

1501 1502
static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
				     int order)
1503 1504 1505 1506 1507 1508 1509
{
	struct mem_cgroup *iter;
	unsigned long chosen_points = 0;
	unsigned long totalpages;
	unsigned int points = 0;
	struct task_struct *chosen = NULL;

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
	/*
	 * If current has a pending SIGKILL, then automatically select it.  The
	 * goal is to allow it to allocate so that it may quickly exit and free
	 * its memory.
	 */
	if (fatal_signal_pending(current)) {
		set_thread_flag(TIF_MEMDIE);
		return;
	}

	check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
	totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
	for_each_mem_cgroup_tree(iter, memcg) {
		struct cgroup *cgroup = iter->css.cgroup;
		struct cgroup_iter it;
		struct task_struct *task;

		cgroup_iter_start(cgroup, &it);
		while ((task = cgroup_iter_next(cgroup, &it))) {
			switch (oom_scan_process_thread(task, totalpages, NULL,
							false)) {
			case OOM_SCAN_SELECT:
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = ULONG_MAX;
				get_task_struct(chosen);
				/* fall through */
			case OOM_SCAN_CONTINUE:
				continue;
			case OOM_SCAN_ABORT:
				cgroup_iter_end(cgroup, &it);
				mem_cgroup_iter_break(memcg, iter);
				if (chosen)
					put_task_struct(chosen);
				return;
			case OOM_SCAN_OK:
				break;
			};
			points = oom_badness(task, memcg, NULL, totalpages);
			if (points > chosen_points) {
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = points;
				get_task_struct(chosen);
			}
		}
		cgroup_iter_end(cgroup, &it);
	}

	if (!chosen)
		return;
	points = chosen_points * 1000 / totalpages;
	oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
			 NULL, "Memory cgroup out of memory");
}

1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
					gfp_t gfp_mask,
					unsigned long flags)
{
	unsigned long total = 0;
	bool noswap = false;
	int loop;

	if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
		noswap = true;
	if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
		noswap = true;

	for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
		if (loop)
			drain_all_stock_async(memcg);
		total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
		/*
		 * Allow limit shrinkers, which are triggered directly
		 * by userspace, to catch signals and stop reclaim
		 * after minimal progress, regardless of the margin.
		 */
		if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
			break;
		if (mem_cgroup_margin(memcg))
			break;
		/*
		 * If nothing was reclaimed after two attempts, there
		 * may be no reclaimable pages in this hierarchy.
		 */
		if (loop && !total)
			break;
	}
	return total;
}

1604 1605
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1606
 * @memcg: the target memcg
1607 1608 1609 1610 1611 1612 1613
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1614
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1615 1616
		int nid, bool noswap)
{
1617
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1618 1619 1620
		return true;
	if (noswap || !total_swap_pages)
		return false;
1621
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1622 1623 1624 1625
		return true;
	return false;

}
1626 1627 1628 1629 1630 1631 1632 1633
#if MAX_NUMNODES > 1

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1634
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1635 1636
{
	int nid;
1637 1638 1639 1640
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1641
	if (!atomic_read(&memcg->numainfo_events))
1642
		return;
1643
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1644 1645 1646
		return;

	/* make a nodemask where this memcg uses memory from */
1647
	memcg->scan_nodes = node_states[N_HIGH_MEMORY];
1648 1649 1650

	for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {

1651 1652
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1653
	}
1654

1655 1656
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1671
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1672 1673 1674
{
	int node;

1675 1676
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1677

1678
	node = next_node(node, memcg->scan_nodes);
1679
	if (node == MAX_NUMNODES)
1680
		node = first_node(memcg->scan_nodes);
1681 1682 1683 1684 1685 1686 1687 1688 1689
	/*
	 * We call this when we hit limit, not when pages are added to LRU.
	 * No LRU may hold pages because all pages are UNEVICTABLE or
	 * memcg is too small and all pages are not on LRU. In that case,
	 * we use curret node.
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1690
	memcg->last_scanned_node = node;
1691 1692 1693
	return node;
}

1694 1695 1696 1697 1698 1699
/*
 * Check all nodes whether it contains reclaimable pages or not.
 * For quick scan, we make use of scan_nodes. This will allow us to skip
 * unused nodes. But scan_nodes is lazily updated and may not cotain
 * enough new information. We need to do double check.
 */
1700
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1701 1702 1703 1704 1705 1706 1707
{
	int nid;

	/*
	 * quick check...making use of scan_node.
	 * We can skip unused nodes.
	 */
1708 1709
	if (!nodes_empty(memcg->scan_nodes)) {
		for (nid = first_node(memcg->scan_nodes);
1710
		     nid < MAX_NUMNODES;
1711
		     nid = next_node(nid, memcg->scan_nodes)) {
1712

1713
			if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1714 1715 1716 1717 1718 1719 1720
				return true;
		}
	}
	/*
	 * Check rest of nodes.
	 */
	for_each_node_state(nid, N_HIGH_MEMORY) {
1721
		if (node_isset(nid, memcg->scan_nodes))
1722
			continue;
1723
		if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1724 1725 1726 1727 1728
			return true;
	}
	return false;
}

1729
#else
1730
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1731 1732 1733
{
	return 0;
}
1734

1735
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1736
{
1737
	return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1738
}
1739 1740
#endif

1741 1742 1743 1744
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
				   struct zone *zone,
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
1745
{
1746
	struct mem_cgroup *victim = NULL;
1747
	int total = 0;
K
KAMEZAWA Hiroyuki 已提交
1748
	int loop = 0;
1749
	unsigned long excess;
1750
	unsigned long nr_scanned;
1751 1752 1753 1754
	struct mem_cgroup_reclaim_cookie reclaim = {
		.zone = zone,
		.priority = 0,
	};
1755

1756
	excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
K
KAMEZAWA Hiroyuki 已提交
1757

1758
	while (1) {
1759
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1760
		if (!victim) {
K
KAMEZAWA Hiroyuki 已提交
1761
			loop++;
1762 1763 1764 1765 1766 1767
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
1768
				if (!total)
1769 1770
					break;
				/*
L
Lucas De Marchi 已提交
1771
				 * We want to do more targeted reclaim.
1772 1773 1774 1775 1776
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
1777
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1778 1779
					break;
			}
1780
			continue;
1781
		}
1782
		if (!mem_cgroup_reclaimable(victim, false))
1783
			continue;
1784 1785 1786 1787
		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
						     zone, &nr_scanned);
		*total_scanned += nr_scanned;
		if (!res_counter_soft_limit_excess(&root_memcg->res))
1788
			break;
1789
	}
1790
	mem_cgroup_iter_break(root_memcg, victim);
K
KAMEZAWA Hiroyuki 已提交
1791
	return total;
1792 1793
}

K
KAMEZAWA Hiroyuki 已提交
1794 1795 1796
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
1797
 * Has to be called with memcg_oom_lock
K
KAMEZAWA Hiroyuki 已提交
1798
 */
1799
static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1800
{
1801
	struct mem_cgroup *iter, *failed = NULL;
1802

1803
	for_each_mem_cgroup_tree(iter, memcg) {
1804
		if (iter->oom_lock) {
1805 1806 1807 1808 1809
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1810 1811
			mem_cgroup_iter_break(memcg, iter);
			break;
1812 1813
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1814
	}
K
KAMEZAWA Hiroyuki 已提交
1815

1816
	if (!failed)
1817
		return true;
1818 1819 1820 1821 1822

	/*
	 * OK, we failed to lock the whole subtree so we have to clean up
	 * what we set up to the failing subtree
	 */
1823
	for_each_mem_cgroup_tree(iter, memcg) {
1824
		if (iter == failed) {
1825 1826
			mem_cgroup_iter_break(memcg, iter);
			break;
1827 1828 1829
		}
		iter->oom_lock = false;
	}
1830
	return false;
1831
}
1832

1833
/*
1834
 * Has to be called with memcg_oom_lock
1835
 */
1836
static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1837
{
K
KAMEZAWA Hiroyuki 已提交
1838 1839
	struct mem_cgroup *iter;

1840
	for_each_mem_cgroup_tree(iter, memcg)
1841 1842 1843 1844
		iter->oom_lock = false;
	return 0;
}

1845
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1846 1847 1848
{
	struct mem_cgroup *iter;

1849
	for_each_mem_cgroup_tree(iter, memcg)
1850 1851 1852
		atomic_inc(&iter->under_oom);
}

1853
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1854 1855 1856
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1857 1858 1859 1860 1861
	/*
	 * When a new child is created while the hierarchy is under oom,
	 * mem_cgroup_oom_lock() may not be called. We have to use
	 * atomic_add_unless() here.
	 */
1862
	for_each_mem_cgroup_tree(iter, memcg)
1863
		atomic_add_unless(&iter->under_oom, -1, 0);
1864 1865
}

1866
static DEFINE_SPINLOCK(memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1867 1868
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1869
struct oom_wait_info {
1870
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
1871 1872 1873 1874 1875 1876
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
1877 1878
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1879 1880 1881
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1882
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
1883 1884

	/*
1885
	 * Both of oom_wait_info->memcg and wake_memcg are stable under us.
K
KAMEZAWA Hiroyuki 已提交
1886 1887
	 * Then we can use css_is_ancestor without taking care of RCU.
	 */
1888 1889
	if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
		&& !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
K
KAMEZAWA Hiroyuki 已提交
1890 1891 1892 1893
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1894
static void memcg_wakeup_oom(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1895
{
1896 1897
	/* for filtering, pass "memcg" as argument. */
	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
K
KAMEZAWA Hiroyuki 已提交
1898 1899
}

1900
static void memcg_oom_recover(struct mem_cgroup *memcg)
1901
{
1902 1903
	if (memcg && atomic_read(&memcg->under_oom))
		memcg_wakeup_oom(memcg);
1904 1905
}

K
KAMEZAWA Hiroyuki 已提交
1906 1907 1908
/*
 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
 */
1909 1910
static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
				  int order)
1911
{
K
KAMEZAWA Hiroyuki 已提交
1912
	struct oom_wait_info owait;
1913
	bool locked, need_to_kill;
K
KAMEZAWA Hiroyuki 已提交
1914

1915
	owait.memcg = memcg;
K
KAMEZAWA Hiroyuki 已提交
1916 1917 1918 1919
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
1920
	need_to_kill = true;
1921
	mem_cgroup_mark_under_oom(memcg);
1922

1923
	/* At first, try to OOM lock hierarchy under memcg.*/
1924
	spin_lock(&memcg_oom_lock);
1925
	locked = mem_cgroup_oom_lock(memcg);
K
KAMEZAWA Hiroyuki 已提交
1926 1927 1928 1929 1930
	/*
	 * Even if signal_pending(), we can't quit charge() loop without
	 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
	 * under OOM is always welcomed, use TASK_KILLABLE here.
	 */
1931
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1932
	if (!locked || memcg->oom_kill_disable)
1933 1934
		need_to_kill = false;
	if (locked)
1935
		mem_cgroup_oom_notify(memcg);
1936
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1937

1938 1939
	if (need_to_kill) {
		finish_wait(&memcg_oom_waitq, &owait.wait);
1940
		mem_cgroup_out_of_memory(memcg, mask, order);
1941
	} else {
K
KAMEZAWA Hiroyuki 已提交
1942
		schedule();
K
KAMEZAWA Hiroyuki 已提交
1943
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
1944
	}
1945
	spin_lock(&memcg_oom_lock);
1946
	if (locked)
1947 1948
		mem_cgroup_oom_unlock(memcg);
	memcg_wakeup_oom(memcg);
1949
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
1950

1951
	mem_cgroup_unmark_under_oom(memcg);
1952

K
KAMEZAWA Hiroyuki 已提交
1953 1954 1955
	if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
		return false;
	/* Give chance to dying process */
1956
	schedule_timeout_uninterruptible(1);
K
KAMEZAWA Hiroyuki 已提交
1957
	return true;
1958 1959
}

1960 1961 1962
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979
 *
 * Notes: Race condition
 *
 * We usually use page_cgroup_lock() for accessing page_cgroup member but
 * it tends to be costly. But considering some conditions, we doesn't need
 * to do so _always_.
 *
 * Considering "charge", lock_page_cgroup() is not required because all
 * file-stat operations happen after a page is attached to radix-tree. There
 * are no race with "charge".
 *
 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
 * if there are race with "uncharge". Statistics itself is properly handled
 * by flags.
 *
 * Considering "move", this is an only case we see a race. To make the race
1980 1981
 * small, we check mm->moving_account and detect there are possibility of race
 * If there is, we take a lock.
1982
 */
1983

1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996
void __mem_cgroup_begin_update_page_stat(struct page *page,
				bool *locked, unsigned long *flags)
{
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
again:
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
		return;
	/*
	 * If this memory cgroup is not under account moving, we don't
1997
	 * need to take move_lock_mem_cgroup(). Because we already hold
1998
	 * rcu_read_lock(), any calls to move_account will be delayed until
1999
	 * rcu_read_unlock() if mem_cgroup_stolen() == true.
2000
	 */
2001
	if (!mem_cgroup_stolen(memcg))
2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018
		return;

	move_lock_mem_cgroup(memcg, flags);
	if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
		move_unlock_mem_cgroup(memcg, flags);
		goto again;
	}
	*locked = true;
}

void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
{
	struct page_cgroup *pc = lookup_page_cgroup(page);

	/*
	 * It's guaranteed that pc->mem_cgroup never changes while
	 * lock is held because a routine modifies pc->mem_cgroup
2019
	 * should take move_lock_mem_cgroup().
2020 2021 2022 2023
	 */
	move_unlock_mem_cgroup(pc->mem_cgroup, flags);
}

2024 2025
void mem_cgroup_update_page_stat(struct page *page,
				 enum mem_cgroup_page_stat_item idx, int val)
2026
{
2027
	struct mem_cgroup *memcg;
2028
	struct page_cgroup *pc = lookup_page_cgroup(page);
2029
	unsigned long uninitialized_var(flags);
2030

2031
	if (mem_cgroup_disabled())
2032
		return;
2033

2034 2035
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
2036
		return;
2037 2038

	switch (idx) {
2039 2040
	case MEMCG_NR_FILE_MAPPED:
		idx = MEM_CGROUP_STAT_FILE_MAPPED;
2041 2042 2043
		break;
	default:
		BUG();
2044
	}
2045

2046
	this_cpu_add(memcg->stat->count[idx], val);
2047
}
2048

2049 2050 2051 2052
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
2053
#define CHARGE_BATCH	32U
2054 2055
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
2056
	unsigned int nr_pages;
2057
	struct work_struct work;
2058
	unsigned long flags;
2059
#define FLUSHING_CACHED_CHARGE	0
2060 2061
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2062
static DEFINE_MUTEX(percpu_charge_mutex);
2063 2064

/*
2065
 * Try to consume stocked charge on this cpu. If success, one page is consumed
2066 2067 2068 2069
 * from local stock and true is returned. If the stock is 0 or charges from a
 * cgroup which is not current target, returns false. This stock will be
 * refilled.
 */
2070
static bool consume_stock(struct mem_cgroup *memcg)
2071 2072 2073 2074 2075
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

	stock = &get_cpu_var(memcg_stock);
2076
	if (memcg == stock->cached && stock->nr_pages)
2077
		stock->nr_pages--;
2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2091 2092 2093 2094
	if (stock->nr_pages) {
		unsigned long bytes = stock->nr_pages * PAGE_SIZE;

		res_counter_uncharge(&old->res, bytes);
2095
		if (do_swap_account)
2096 2097
			res_counter_uncharge(&old->memsw, bytes);
		stock->nr_pages = 0;
2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
2110
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2111 2112 2113 2114
}

/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
2115
 * This will be consumed by consume_stock() function, later.
2116
 */
2117
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2118 2119 2120
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

2121
	if (stock->cached != memcg) { /* reset if necessary */
2122
		drain_stock(stock);
2123
		stock->cached = memcg;
2124
	}
2125
	stock->nr_pages += nr_pages;
2126 2127 2128 2129
	put_cpu_var(memcg_stock);
}

/*
2130
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2131 2132
 * of the hierarchy under it. sync flag says whether we should block
 * until the work is done.
2133
 */
2134
static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2135
{
2136
	int cpu, curcpu;
2137

2138 2139
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
2140
	curcpu = get_cpu();
2141 2142
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2143
		struct mem_cgroup *memcg;
2144

2145 2146
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
2147
			continue;
2148
		if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2149
			continue;
2150 2151 2152 2153 2154 2155
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2156
	}
2157
	put_cpu();
2158 2159 2160 2161 2162 2163

	if (!sync)
		goto out;

	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2164
		if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2165 2166 2167
			flush_work(&stock->work);
	}
out:
2168
 	put_online_cpus();
2169 2170 2171 2172 2173 2174 2175 2176
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
2177
static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2178
{
2179 2180 2181 2182 2183
	/*
	 * If someone calls draining, avoid adding more kworker runs.
	 */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2184
	drain_all_stock(root_memcg, false);
2185
	mutex_unlock(&percpu_charge_mutex);
2186 2187 2188
}

/* This is a synchronous drain interface. */
2189
static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2190 2191
{
	/* called when force_empty is called */
2192
	mutex_lock(&percpu_charge_mutex);
2193
	drain_all_stock(root_memcg, true);
2194
	mutex_unlock(&percpu_charge_mutex);
2195 2196
}

2197 2198 2199 2200
/*
 * This function drains percpu counter value from DEAD cpu and
 * move it to local cpu. Note that this function can be preempted.
 */
2201
static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2202 2203 2204
{
	int i;

2205
	spin_lock(&memcg->pcp_counter_lock);
2206
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2207
		long x = per_cpu(memcg->stat->count[i], cpu);
2208

2209 2210
		per_cpu(memcg->stat->count[i], cpu) = 0;
		memcg->nocpu_base.count[i] += x;
2211
	}
2212
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2213
		unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2214

2215 2216
		per_cpu(memcg->stat->events[i], cpu) = 0;
		memcg->nocpu_base.events[i] += x;
2217
	}
2218
	spin_unlock(&memcg->pcp_counter_lock);
2219 2220 2221
}

static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2222 2223 2224 2225 2226
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;
2227
	struct mem_cgroup *iter;
2228

2229
	if (action == CPU_ONLINE)
2230 2231
		return NOTIFY_OK;

2232
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2233
		return NOTIFY_OK;
2234

2235
	for_each_mem_cgroup(iter)
2236 2237
		mem_cgroup_drain_pcp_counter(iter, cpu);

2238 2239 2240 2241 2242
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

2243 2244 2245 2246 2247 2248 2249 2250 2251 2252

/* See __mem_cgroup_try_charge() for details */
enum {
	CHARGE_OK,		/* success */
	CHARGE_RETRY,		/* need to retry but retry is not bad */
	CHARGE_NOMEM,		/* we can't do more. return -ENOMEM */
	CHARGE_WOULDBLOCK,	/* GFP_WAIT wasn't set and no enough res. */
	CHARGE_OOM_DIE,		/* the current is killed because of OOM */
};

2253
static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2254
				unsigned int nr_pages, bool oom_check)
2255
{
2256
	unsigned long csize = nr_pages * PAGE_SIZE;
2257 2258 2259 2260 2261
	struct mem_cgroup *mem_over_limit;
	struct res_counter *fail_res;
	unsigned long flags = 0;
	int ret;

2262
	ret = res_counter_charge(&memcg->res, csize, &fail_res);
2263 2264 2265 2266

	if (likely(!ret)) {
		if (!do_swap_account)
			return CHARGE_OK;
2267
		ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2268 2269 2270
		if (likely(!ret))
			return CHARGE_OK;

2271
		res_counter_uncharge(&memcg->res, csize);
2272 2273 2274 2275
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
		flags |= MEM_CGROUP_RECLAIM_NOSWAP;
	} else
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2276
	/*
2277 2278
	 * nr_pages can be either a huge page (HPAGE_PMD_NR), a batch
	 * of regular pages (CHARGE_BATCH), or a single regular page (1).
2279 2280 2281 2282
	 *
	 * Never reclaim on behalf of optional batching, retry with a
	 * single page instead.
	 */
2283
	if (nr_pages == CHARGE_BATCH)
2284 2285 2286 2287 2288
		return CHARGE_RETRY;

	if (!(gfp_mask & __GFP_WAIT))
		return CHARGE_WOULDBLOCK;

2289
	ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2290
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2291
		return CHARGE_RETRY;
2292
	/*
2293 2294 2295 2296 2297 2298 2299
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
2300
	 */
2301
	if (nr_pages == 1 && ret)
2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
		return CHARGE_RETRY;

	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		return CHARGE_RETRY;

	/* If we don't need to call oom-killer at el, return immediately */
	if (!oom_check)
		return CHARGE_NOMEM;
	/* check OOM */
2315
	if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
2316 2317 2318 2319 2320
		return CHARGE_OOM_DIE;

	return CHARGE_RETRY;
}

2321
/*
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
 * __mem_cgroup_try_charge() does
 * 1. detect memcg to be charged against from passed *mm and *ptr,
 * 2. update res_counter
 * 3. call memory reclaim if necessary.
 *
 * In some special case, if the task is fatal, fatal_signal_pending() or
 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
 * as possible without any hazards. 2: all pages should have a valid
 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
 * pointer, that is treated as a charge to root_mem_cgroup.
 *
 * So __mem_cgroup_try_charge() will return
 *  0       ...  on success, filling *ptr with a valid memcg pointer.
 *  -ENOMEM ...  charge failure because of resource limits.
 *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
 *
 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
 * the oom-killer can be invoked.
2341
 */
2342
static int __mem_cgroup_try_charge(struct mm_struct *mm,
A
Andrea Arcangeli 已提交
2343
				   gfp_t gfp_mask,
2344
				   unsigned int nr_pages,
2345
				   struct mem_cgroup **ptr,
2346
				   bool oom)
2347
{
2348
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2349
	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2350
	struct mem_cgroup *memcg = NULL;
2351
	int ret;
2352

K
KAMEZAWA Hiroyuki 已提交
2353 2354 2355 2356 2357 2358 2359 2360
	/*
	 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
	 * in system level. So, allow to go ahead dying process in addition to
	 * MEMDIE process.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)
		     || fatal_signal_pending(current)))
		goto bypass;
2361

2362
	/*
2363 2364
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
2365
	 * thread group leader migrates. It's possible that mm is not
2366
	 * set, if so charge the root memcg (happens for pagecache usage).
2367
	 */
2368
	if (!*ptr && !mm)
2369
		*ptr = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
2370
again:
2371 2372 2373
	if (*ptr) { /* css should be a valid one */
		memcg = *ptr;
		if (mem_cgroup_is_root(memcg))
K
KAMEZAWA Hiroyuki 已提交
2374
			goto done;
2375
		if (nr_pages == 1 && consume_stock(memcg))
K
KAMEZAWA Hiroyuki 已提交
2376
			goto done;
2377
		css_get(&memcg->css);
2378
	} else {
K
KAMEZAWA Hiroyuki 已提交
2379
		struct task_struct *p;
2380

K
KAMEZAWA Hiroyuki 已提交
2381 2382 2383
		rcu_read_lock();
		p = rcu_dereference(mm->owner);
		/*
2384
		 * Because we don't have task_lock(), "p" can exit.
2385
		 * In that case, "memcg" can point to root or p can be NULL with
2386 2387 2388 2389 2390 2391
		 * race with swapoff. Then, we have small risk of mis-accouning.
		 * But such kind of mis-account by race always happens because
		 * we don't have cgroup_mutex(). It's overkill and we allo that
		 * small race, here.
		 * (*) swapoff at el will charge against mm-struct not against
		 * task-struct. So, mm->owner can be NULL.
K
KAMEZAWA Hiroyuki 已提交
2392
		 */
2393
		memcg = mem_cgroup_from_task(p);
2394 2395 2396
		if (!memcg)
			memcg = root_mem_cgroup;
		if (mem_cgroup_is_root(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2397 2398 2399
			rcu_read_unlock();
			goto done;
		}
2400
		if (nr_pages == 1 && consume_stock(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412
			/*
			 * It seems dagerous to access memcg without css_get().
			 * But considering how consume_stok works, it's not
			 * necessary. If consume_stock success, some charges
			 * from this memcg are cached on this cpu. So, we
			 * don't need to call css_get()/css_tryget() before
			 * calling consume_stock().
			 */
			rcu_read_unlock();
			goto done;
		}
		/* after here, we may be blocked. we need to get refcnt */
2413
		if (!css_tryget(&memcg->css)) {
K
KAMEZAWA Hiroyuki 已提交
2414 2415 2416 2417 2418
			rcu_read_unlock();
			goto again;
		}
		rcu_read_unlock();
	}
2419

2420 2421
	do {
		bool oom_check;
2422

2423
		/* If killed, bypass charge */
K
KAMEZAWA Hiroyuki 已提交
2424
		if (fatal_signal_pending(current)) {
2425
			css_put(&memcg->css);
2426
			goto bypass;
K
KAMEZAWA Hiroyuki 已提交
2427
		}
2428

2429 2430 2431 2432
		oom_check = false;
		if (oom && !nr_oom_retries) {
			oom_check = true;
			nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2433
		}
2434

2435
		ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, oom_check);
2436 2437 2438 2439
		switch (ret) {
		case CHARGE_OK:
			break;
		case CHARGE_RETRY: /* not in OOM situation but retry */
2440
			batch = nr_pages;
2441 2442
			css_put(&memcg->css);
			memcg = NULL;
K
KAMEZAWA Hiroyuki 已提交
2443
			goto again;
2444
		case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2445
			css_put(&memcg->css);
2446 2447
			goto nomem;
		case CHARGE_NOMEM: /* OOM routine works */
K
KAMEZAWA Hiroyuki 已提交
2448
			if (!oom) {
2449
				css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2450
				goto nomem;
K
KAMEZAWA Hiroyuki 已提交
2451
			}
2452 2453 2454 2455
			/* If oom, we never return -ENOMEM */
			nr_oom_retries--;
			break;
		case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2456
			css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2457
			goto bypass;
2458
		}
2459 2460
	} while (ret != CHARGE_OK);

2461
	if (batch > nr_pages)
2462 2463
		refill_stock(memcg, batch - nr_pages);
	css_put(&memcg->css);
2464
done:
2465
	*ptr = memcg;
2466 2467
	return 0;
nomem:
2468
	*ptr = NULL;
2469
	return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2470
bypass:
2471 2472
	*ptr = root_mem_cgroup;
	return -EINTR;
2473
}
2474

2475 2476 2477 2478 2479
/*
 * Somemtimes we have to undo a charge we got by try_charge().
 * This function is for that and do uncharge, put css's refcnt.
 * gotten by try_charge().
 */
2480
static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2481
				       unsigned int nr_pages)
2482
{
2483
	if (!mem_cgroup_is_root(memcg)) {
2484 2485
		unsigned long bytes = nr_pages * PAGE_SIZE;

2486
		res_counter_uncharge(&memcg->res, bytes);
2487
		if (do_swap_account)
2488
			res_counter_uncharge(&memcg->memsw, bytes);
2489
	}
2490 2491
}

2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
/*
 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
 * This is useful when moving usage to parent cgroup.
 */
static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
					unsigned int nr_pages)
{
	unsigned long bytes = nr_pages * PAGE_SIZE;

	if (mem_cgroup_is_root(memcg))
		return;

	res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
	if (do_swap_account)
		res_counter_uncharge_until(&memcg->memsw,
						memcg->memsw.parent, bytes);
}

2510 2511
/*
 * A helper function to get mem_cgroup from ID. must be called under
T
Tejun Heo 已提交
2512 2513 2514
 * rcu_read_lock().  The caller is responsible for calling css_tryget if
 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
 * called against removed memcg.)
2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
2526
	return mem_cgroup_from_css(css);
2527 2528
}

2529
struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2530
{
2531
	struct mem_cgroup *memcg = NULL;
2532
	struct page_cgroup *pc;
2533
	unsigned short id;
2534 2535
	swp_entry_t ent;

2536 2537 2538
	VM_BUG_ON(!PageLocked(page));

	pc = lookup_page_cgroup(page);
2539
	lock_page_cgroup(pc);
2540
	if (PageCgroupUsed(pc)) {
2541 2542 2543
		memcg = pc->mem_cgroup;
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2544
	} else if (PageSwapCache(page)) {
2545
		ent.val = page_private(page);
2546
		id = lookup_swap_cgroup_id(ent);
2547
		rcu_read_lock();
2548 2549 2550
		memcg = mem_cgroup_lookup(id);
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2551
		rcu_read_unlock();
2552
	}
2553
	unlock_page_cgroup(pc);
2554
	return memcg;
2555 2556
}

2557
static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2558
				       struct page *page,
2559
				       unsigned int nr_pages,
2560 2561
				       enum charge_type ctype,
				       bool lrucare)
2562
{
2563
	struct page_cgroup *pc = lookup_page_cgroup(page);
2564
	struct zone *uninitialized_var(zone);
2565
	struct lruvec *lruvec;
2566
	bool was_on_lru = false;
2567
	bool anon;
2568

2569
	lock_page_cgroup(pc);
2570
	VM_BUG_ON(PageCgroupUsed(pc));
2571 2572 2573 2574
	/*
	 * we don't need page_cgroup_lock about tail pages, becase they are not
	 * accessed by any other context at this point.
	 */
2575 2576 2577 2578 2579 2580 2581 2582 2583

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
	if (lrucare) {
		zone = page_zone(page);
		spin_lock_irq(&zone->lru_lock);
		if (PageLRU(page)) {
2584
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2585
			ClearPageLRU(page);
2586
			del_page_from_lru_list(page, lruvec, page_lru(page));
2587 2588 2589 2590
			was_on_lru = true;
		}
	}

2591
	pc->mem_cgroup = memcg;
2592 2593 2594 2595 2596 2597 2598
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
2599
	smp_wmb();
2600
	SetPageCgroupUsed(pc);
2601

2602 2603
	if (lrucare) {
		if (was_on_lru) {
2604
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2605 2606
			VM_BUG_ON(PageLRU(page));
			SetPageLRU(page);
2607
			add_page_to_lru_list(page, lruvec, page_lru(page));
2608 2609 2610 2611
		}
		spin_unlock_irq(&zone->lru_lock);
	}

2612
	if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2613 2614 2615 2616 2617
		anon = true;
	else
		anon = false;

	mem_cgroup_charge_statistics(memcg, anon, nr_pages);
2618
	unlock_page_cgroup(pc);
2619

2620 2621 2622 2623 2624
	/*
	 * "charge_statistics" updated event counter. Then, check it.
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
	 */
2625
	memcg_check_events(memcg, page);
2626
}
2627

2628 2629
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

2630
#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
2631 2632
/*
 * Because tail pages are not marked as "used", set it. We're under
2633 2634 2635
 * zone->lru_lock, 'splitting on pmd' and compound_lock.
 * charge/uncharge will be never happen and move_account() is done under
 * compound_lock(), so we don't have to take care of races.
2636
 */
2637
void mem_cgroup_split_huge_fixup(struct page *head)
2638 2639
{
	struct page_cgroup *head_pc = lookup_page_cgroup(head);
2640 2641
	struct page_cgroup *pc;
	int i;
2642

2643 2644
	if (mem_cgroup_disabled())
		return;
2645 2646 2647 2648 2649 2650
	for (i = 1; i < HPAGE_PMD_NR; i++) {
		pc = head_pc + i;
		pc->mem_cgroup = head_pc->mem_cgroup;
		smp_wmb();/* see __commit_charge() */
		pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
	}
2651
}
2652
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2653

2654
/**
2655
 * mem_cgroup_move_account - move account of the page
2656
 * @page: the page
2657
 * @nr_pages: number of regular pages (>1 for huge pages)
2658 2659 2660 2661 2662
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
2663
 * - page is not on LRU (isolate_page() is useful.)
2664
 * - compound_lock is held when nr_pages > 1
2665
 *
2666 2667
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
2668
 */
2669 2670 2671 2672
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct page_cgroup *pc,
				   struct mem_cgroup *from,
2673
				   struct mem_cgroup *to)
2674
{
2675 2676
	unsigned long flags;
	int ret;
2677
	bool anon = PageAnon(page);
2678

2679
	VM_BUG_ON(from == to);
2680
	VM_BUG_ON(PageLRU(page));
2681 2682 2683 2684 2685 2686 2687
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
2688
	if (nr_pages > 1 && !PageTransHuge(page))
2689 2690 2691 2692 2693 2694 2695 2696
		goto out;

	lock_page_cgroup(pc);

	ret = -EINVAL;
	if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
		goto unlock;

2697
	move_lock_mem_cgroup(from, &flags);
2698

2699
	if (!anon && page_mapped(page)) {
2700 2701 2702 2703 2704
		/* Update mapped_file data for mem_cgroup */
		preempt_disable();
		__this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		__this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		preempt_enable();
2705
	}
2706
	mem_cgroup_charge_statistics(from, anon, -nr_pages);
2707

2708
	/* caller should have done css_get */
K
KAMEZAWA Hiroyuki 已提交
2709
	pc->mem_cgroup = to;
2710
	mem_cgroup_charge_statistics(to, anon, nr_pages);
2711
	move_unlock_mem_cgroup(from, &flags);
2712 2713
	ret = 0;
unlock:
2714
	unlock_page_cgroup(pc);
2715 2716 2717
	/*
	 * check events
	 */
2718 2719
	memcg_check_events(to, page);
	memcg_check_events(from, page);
2720
out:
2721 2722 2723
	return ret;
}

2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
/**
 * mem_cgroup_move_parent - moves page to the parent group
 * @page: the page to move
 * @pc: page_cgroup of the page
 * @child: page's cgroup
 *
 * move charges to its parent or the root cgroup if the group has no
 * parent (aka use_hierarchy==0).
 * Although this might fail (get_page_unless_zero, isolate_lru_page or
 * mem_cgroup_move_account fails) the failure is always temporary and
 * it signals a race with a page removal/uncharge or migration. In the
 * first case the page is on the way out and it will vanish from the LRU
 * on the next attempt and the call should be retried later.
 * Isolation from the LRU fails only if page has been isolated from
 * the LRU since we looked at it and that usually means either global
 * reclaim or migration going on. The page will either get back to the
 * LRU or vanish.
 * Finaly mem_cgroup_move_account fails only if the page got uncharged
 * (!PageCgroupUsed) or moved to a different group. The page will
 * disappear in the next attempt.
2744
 */
2745 2746
static int mem_cgroup_move_parent(struct page *page,
				  struct page_cgroup *pc,
2747
				  struct mem_cgroup *child)
2748 2749
{
	struct mem_cgroup *parent;
2750
	unsigned int nr_pages;
2751
	unsigned long uninitialized_var(flags);
2752 2753
	int ret;

2754
	VM_BUG_ON(mem_cgroup_is_root(child));
2755

2756 2757 2758 2759 2760
	ret = -EBUSY;
	if (!get_page_unless_zero(page))
		goto out;
	if (isolate_lru_page(page))
		goto put;
2761

2762
	nr_pages = hpage_nr_pages(page);
K
KAMEZAWA Hiroyuki 已提交
2763

2764 2765 2766 2767 2768 2769
	parent = parent_mem_cgroup(child);
	/*
	 * If no parent, move charges to root cgroup.
	 */
	if (!parent)
		parent = root_mem_cgroup;
2770

2771 2772
	if (nr_pages > 1) {
		VM_BUG_ON(!PageTransHuge(page));
2773
		flags = compound_lock_irqsave(page);
2774
	}
2775

2776
	ret = mem_cgroup_move_account(page, nr_pages,
2777
				pc, child, parent);
2778 2779
	if (!ret)
		__mem_cgroup_cancel_local_charge(child, nr_pages);
2780

2781
	if (nr_pages > 1)
2782
		compound_unlock_irqrestore(page, flags);
K
KAMEZAWA Hiroyuki 已提交
2783
	putback_lru_page(page);
2784
put:
2785
	put_page(page);
2786
out:
2787 2788 2789
	return ret;
}

2790 2791 2792 2793 2794 2795 2796
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2797
				gfp_t gfp_mask, enum charge_type ctype)
2798
{
2799
	struct mem_cgroup *memcg = NULL;
2800
	unsigned int nr_pages = 1;
2801
	bool oom = true;
2802
	int ret;
A
Andrea Arcangeli 已提交
2803

A
Andrea Arcangeli 已提交
2804
	if (PageTransHuge(page)) {
2805
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
2806
		VM_BUG_ON(!PageTransHuge(page));
2807 2808 2809 2810 2811
		/*
		 * Never OOM-kill a process for a huge page.  The
		 * fault handler will fall back to regular pages.
		 */
		oom = false;
A
Andrea Arcangeli 已提交
2812
	}
2813

2814
	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
2815
	if (ret == -ENOMEM)
2816
		return ret;
2817
	__mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
2818 2819 2820
	return 0;
}

2821 2822
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
2823
{
2824
	if (mem_cgroup_disabled())
2825
		return 0;
2826 2827 2828
	VM_BUG_ON(page_mapped(page));
	VM_BUG_ON(page->mapping && !PageAnon(page));
	VM_BUG_ON(!mm);
2829
	return mem_cgroup_charge_common(page, mm, gfp_mask,
2830
					MEM_CGROUP_CHARGE_TYPE_ANON);
2831 2832
}

2833 2834 2835
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
2836
 * struct page_cgroup is acquired. This refcnt will be consumed by
2837 2838
 * "commit()" or removed by "cancel()"
 */
2839 2840 2841 2842
static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
					  struct page *page,
					  gfp_t mask,
					  struct mem_cgroup **memcgp)
2843
{
2844
	struct mem_cgroup *memcg;
2845
	struct page_cgroup *pc;
2846
	int ret;
2847

2848 2849 2850 2851 2852 2853 2854 2855 2856 2857
	pc = lookup_page_cgroup(page);
	/*
	 * Every swap fault against a single page tries to charge the
	 * page, bail as early as possible.  shmem_unuse() encounters
	 * already charged pages, too.  The USED bit is protected by
	 * the page lock, which serializes swap cache removal, which
	 * in turn serializes uncharging.
	 */
	if (PageCgroupUsed(pc))
		return 0;
2858 2859
	if (!do_swap_account)
		goto charge_cur_mm;
2860 2861
	memcg = try_get_mem_cgroup_from_page(page);
	if (!memcg)
2862
		goto charge_cur_mm;
2863 2864
	*memcgp = memcg;
	ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
2865
	css_put(&memcg->css);
2866 2867
	if (ret == -EINTR)
		ret = 0;
2868
	return ret;
2869
charge_cur_mm:
2870 2871 2872 2873
	ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
	if (ret == -EINTR)
		ret = 0;
	return ret;
2874 2875
}

2876 2877 2878 2879 2880 2881
int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
				 gfp_t gfp_mask, struct mem_cgroup **memcgp)
{
	*memcgp = NULL;
	if (mem_cgroup_disabled())
		return 0;
2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
	/*
	 * A racing thread's fault, or swapoff, may have already
	 * updated the pte, and even removed page from swap cache: in
	 * those cases unuse_pte()'s pte_same() test will fail; but
	 * there's also a KSM case which does need to charge the page.
	 */
	if (!PageSwapCache(page)) {
		int ret;

		ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
		if (ret == -EINTR)
			ret = 0;
		return ret;
	}
2896 2897 2898
	return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
}

2899 2900 2901 2902 2903 2904 2905 2906 2907
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return;
	if (!memcg)
		return;
	__mem_cgroup_cancel_charge(memcg, 1);
}

D
Daisuke Nishimura 已提交
2908
static void
2909
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
D
Daisuke Nishimura 已提交
2910
					enum charge_type ctype)
2911
{
2912
	if (mem_cgroup_disabled())
2913
		return;
2914
	if (!memcg)
2915
		return;
2916

2917
	__mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
2918 2919 2920
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
2921 2922 2923
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
2924
	 */
2925
	if (do_swap_account && PageSwapCache(page)) {
2926
		swp_entry_t ent = {.val = page_private(page)};
2927
		mem_cgroup_uncharge_swap(ent);
2928
	}
2929 2930
}

2931 2932
void mem_cgroup_commit_charge_swapin(struct page *page,
				     struct mem_cgroup *memcg)
D
Daisuke Nishimura 已提交
2933
{
2934
	__mem_cgroup_commit_charge_swapin(page, memcg,
2935
					  MEM_CGROUP_CHARGE_TYPE_ANON);
D
Daisuke Nishimura 已提交
2936 2937
}

2938 2939
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
2940
{
2941 2942 2943 2944
	struct mem_cgroup *memcg = NULL;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
	int ret;

2945
	if (mem_cgroup_disabled())
2946 2947 2948 2949 2950 2951 2952
		return 0;
	if (PageCompound(page))
		return 0;

	if (!PageSwapCache(page))
		ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
	else { /* page is swapcache/shmem */
2953 2954
		ret = __mem_cgroup_try_charge_swapin(mm, page,
						     gfp_mask, &memcg);
2955 2956 2957 2958
		if (!ret)
			__mem_cgroup_commit_charge_swapin(page, memcg, type);
	}
	return ret;
2959 2960
}

2961
static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
2962 2963
				   unsigned int nr_pages,
				   const enum charge_type ctype)
2964 2965 2966
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
2967

2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
2979
		batch->memcg = memcg;
2980 2981
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
L
Lucas De Marchi 已提交
2982
	 * In those cases, all pages freed continuously can be expected to be in
2983 2984 2985 2986 2987 2988 2989 2990
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */

	if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

2991
	if (nr_pages > 1)
A
Andrea Arcangeli 已提交
2992 2993
		goto direct_uncharge;

2994 2995 2996 2997 2998
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
2999
	if (batch->memcg != memcg)
3000 3001
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
3002
	batch->nr_pages++;
3003
	if (uncharge_memsw)
3004
		batch->memsw_nr_pages++;
3005 3006
	return;
direct_uncharge:
3007
	res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
3008
	if (uncharge_memsw)
3009 3010 3011
		res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
	if (unlikely(batch->memcg != memcg))
		memcg_oom_recover(memcg);
3012
}
3013

3014
/*
3015
 * uncharge if !page_mapped(page)
3016
 */
3017
static struct mem_cgroup *
3018 3019
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
			     bool end_migration)
3020
{
3021
	struct mem_cgroup *memcg = NULL;
3022 3023
	unsigned int nr_pages = 1;
	struct page_cgroup *pc;
3024
	bool anon;
3025

3026
	if (mem_cgroup_disabled())
3027
		return NULL;
3028

3029
	VM_BUG_ON(PageSwapCache(page));
K
KAMEZAWA Hiroyuki 已提交
3030

A
Andrea Arcangeli 已提交
3031
	if (PageTransHuge(page)) {
3032
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
3033 3034
		VM_BUG_ON(!PageTransHuge(page));
	}
3035
	/*
3036
	 * Check if our page_cgroup is valid
3037
	 */
3038
	pc = lookup_page_cgroup(page);
3039
	if (unlikely(!PageCgroupUsed(pc)))
3040
		return NULL;
3041

3042
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
3043

3044
	memcg = pc->mem_cgroup;
3045

K
KAMEZAWA Hiroyuki 已提交
3046 3047 3048
	if (!PageCgroupUsed(pc))
		goto unlock_out;

3049 3050
	anon = PageAnon(page);

K
KAMEZAWA Hiroyuki 已提交
3051
	switch (ctype) {
3052
	case MEM_CGROUP_CHARGE_TYPE_ANON:
3053 3054 3055 3056 3057
		/*
		 * Generally PageAnon tells if it's the anon statistics to be
		 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
		 * used before page reached the stage of being marked PageAnon.
		 */
3058 3059
		anon = true;
		/* fallthrough */
K
KAMEZAWA Hiroyuki 已提交
3060
	case MEM_CGROUP_CHARGE_TYPE_DROP:
3061
		/* See mem_cgroup_prepare_migration() */
3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
		if (page_mapped(page))
			goto unlock_out;
		/*
		 * Pages under migration may not be uncharged.  But
		 * end_migration() /must/ be the one uncharging the
		 * unused post-migration page and so it has to call
		 * here with the migration bit still set.  See the
		 * res_counter handling below.
		 */
		if (!end_migration && PageCgroupMigration(pc))
K
KAMEZAWA Hiroyuki 已提交
3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
3083
	}
K
KAMEZAWA Hiroyuki 已提交
3084

3085
	mem_cgroup_charge_statistics(memcg, anon, -nr_pages);
K
KAMEZAWA Hiroyuki 已提交
3086

3087
	ClearPageCgroupUsed(pc);
3088 3089 3090 3091 3092 3093
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
3094

3095
	unlock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
3096
	/*
3097
	 * even after unlock, we have memcg->res.usage here and this memcg
K
KAMEZAWA Hiroyuki 已提交
3098 3099
	 * will never be freed.
	 */
3100
	memcg_check_events(memcg, page);
K
KAMEZAWA Hiroyuki 已提交
3101
	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
3102 3103
		mem_cgroup_swap_statistics(memcg, true);
		mem_cgroup_get(memcg);
K
KAMEZAWA Hiroyuki 已提交
3104
	}
3105 3106 3107 3108 3109 3110
	/*
	 * Migration does not charge the res_counter for the
	 * replacement page, so leave it alone when phasing out the
	 * page that is unused after the migration.
	 */
	if (!end_migration && !mem_cgroup_is_root(memcg))
3111
		mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
3112

3113
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
3114 3115 3116

unlock_out:
	unlock_page_cgroup(pc);
3117
	return NULL;
3118 3119
}

3120 3121
void mem_cgroup_uncharge_page(struct page *page)
{
3122 3123 3124
	/* early check. */
	if (page_mapped(page))
		return;
3125
	VM_BUG_ON(page->mapping && !PageAnon(page));
3126 3127
	if (PageSwapCache(page))
		return;
3128
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
3129 3130 3131 3132 3133
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
3134
	VM_BUG_ON(page->mapping);
3135
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
3136 3137
}

3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
3152 3153
		current->memcg_batch.nr_pages = 0;
		current->memcg_batch.memsw_nr_pages = 0;
3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
3174 3175 3176 3177 3178 3179
	if (batch->nr_pages)
		res_counter_uncharge(&batch->memcg->res,
				     batch->nr_pages * PAGE_SIZE);
	if (batch->memsw_nr_pages)
		res_counter_uncharge(&batch->memcg->memsw,
				     batch->memsw_nr_pages * PAGE_SIZE);
3180
	memcg_oom_recover(batch->memcg);
3181 3182 3183 3184
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

3185
#ifdef CONFIG_SWAP
3186
/*
3187
 * called after __delete_from_swap_cache() and drop "page" account.
3188 3189
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
3190 3191
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
3192 3193
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
3194 3195 3196 3197 3198
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

3199
	memcg = __mem_cgroup_uncharge_common(page, ctype, false);
3200

K
KAMEZAWA Hiroyuki 已提交
3201 3202 3203 3204 3205
	/*
	 * record memcg information,  if swapout && memcg != NULL,
	 * mem_cgroup_get() was called in uncharge().
	 */
	if (do_swap_account && swapout && memcg)
3206
		swap_cgroup_record(ent, css_id(&memcg->css));
3207
}
3208
#endif
3209

A
Andrew Morton 已提交
3210
#ifdef CONFIG_MEMCG_SWAP
3211 3212 3213 3214 3215
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
3216
{
3217
	struct mem_cgroup *memcg;
3218
	unsigned short id;
3219 3220 3221 3222

	if (!do_swap_account)
		return;

3223 3224 3225
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
3226
	if (memcg) {
3227 3228 3229 3230
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
3231
		if (!mem_cgroup_is_root(memcg))
3232
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
3233
		mem_cgroup_swap_statistics(memcg, false);
3234 3235
		mem_cgroup_put(memcg);
	}
3236
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
3237
}
3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
 * The caller must have charged to @to, IOW, called res_counter_charge() about
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
3254
				struct mem_cgroup *from, struct mem_cgroup *to)
3255 3256 3257 3258 3259 3260 3261 3262
{
	unsigned short old_id, new_id;

	old_id = css_id(&from->css);
	new_id = css_id(&to->css);

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
3263
		mem_cgroup_swap_statistics(to, true);
3264
		/*
3265 3266 3267 3268 3269 3270
		 * This function is only called from task migration context now.
		 * It postpones res_counter and refcount handling till the end
		 * of task migration(mem_cgroup_clear_mc()) for performance
		 * improvement. But we cannot postpone mem_cgroup_get(to)
		 * because if the process that has been moved to @to does
		 * swap-in, the refcount of @to might be decreased to 0.
3271 3272 3273 3274 3275 3276 3277 3278
		 */
		mem_cgroup_get(to);
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3279
				struct mem_cgroup *from, struct mem_cgroup *to)
3280 3281 3282
{
	return -EINVAL;
}
3283
#endif
K
KAMEZAWA Hiroyuki 已提交
3284

3285
/*
3286 3287
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
3288
 */
3289 3290
void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
				  struct mem_cgroup **memcgp)
3291
{
3292
	struct mem_cgroup *memcg = NULL;
3293
	struct page_cgroup *pc;
3294
	enum charge_type ctype;
3295

3296
	*memcgp = NULL;
3297

A
Andrea Arcangeli 已提交
3298
	VM_BUG_ON(PageTransHuge(page));
3299
	if (mem_cgroup_disabled())
3300
		return;
3301

3302 3303 3304
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
3305 3306
		memcg = pc->mem_cgroup;
		css_get(&memcg->css);
3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337
		/*
		 * At migrating an anonymous page, its mapcount goes down
		 * to 0 and uncharge() will be called. But, even if it's fully
		 * unmapped, migration may fail and this page has to be
		 * charged again. We set MIGRATION flag here and delay uncharge
		 * until end_migration() is called
		 *
		 * Corner Case Thinking
		 * A)
		 * When the old page was mapped as Anon and it's unmap-and-freed
		 * while migration was ongoing.
		 * If unmap finds the old page, uncharge() of it will be delayed
		 * until end_migration(). If unmap finds a new page, it's
		 * uncharged when it make mapcount to be 1->0. If unmap code
		 * finds swap_migration_entry, the new page will not be mapped
		 * and end_migration() will find it(mapcount==0).
		 *
		 * B)
		 * When the old page was mapped but migraion fails, the kernel
		 * remaps it. A charge for it is kept by MIGRATION flag even
		 * if mapcount goes down to 0. We can do remap successfully
		 * without charging it again.
		 *
		 * C)
		 * The "old" page is under lock_page() until the end of
		 * migration, so, the old page itself will not be swapped-out.
		 * If the new page is swapped out before end_migraton, our
		 * hook to usual swap-out path will catch the event.
		 */
		if (PageAnon(page))
			SetPageCgroupMigration(pc);
3338
	}
3339
	unlock_page_cgroup(pc);
3340 3341 3342 3343
	/*
	 * If the page is not charged at this point,
	 * we return here.
	 */
3344
	if (!memcg)
3345
		return;
3346

3347
	*memcgp = memcg;
3348 3349 3350 3351 3352 3353 3354
	/*
	 * We charge new page before it's used/mapped. So, even if unlock_page()
	 * is called before end_migration, we can catch all events on this new
	 * page. In the case new page is migrated but not remapped, new page's
	 * mapcount will be finally 0 and we call uncharge in end_migration().
	 */
	if (PageAnon(page))
3355
		ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
3356
	else
3357
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
3358 3359 3360 3361 3362
	/*
	 * The page is committed to the memcg, but it's not actually
	 * charged to the res_counter since we plan on replacing the
	 * old one and only one page is going to be left afterwards.
	 */
3363
	__mem_cgroup_commit_charge(memcg, newpage, 1, ctype, false);
3364
}
3365

3366
/* remove redundant charge if migration failed*/
3367
void mem_cgroup_end_migration(struct mem_cgroup *memcg,
3368
	struct page *oldpage, struct page *newpage, bool migration_ok)
3369
{
3370
	struct page *used, *unused;
3371
	struct page_cgroup *pc;
3372
	bool anon;
3373

3374
	if (!memcg)
3375
		return;
3376

3377
	if (!migration_ok) {
3378 3379
		used = oldpage;
		unused = newpage;
3380
	} else {
3381
		used = newpage;
3382 3383
		unused = oldpage;
	}
3384
	anon = PageAnon(used);
3385 3386 3387 3388
	__mem_cgroup_uncharge_common(unused,
				     anon ? MEM_CGROUP_CHARGE_TYPE_ANON
				     : MEM_CGROUP_CHARGE_TYPE_CACHE,
				     true);
3389
	css_put(&memcg->css);
3390
	/*
3391 3392 3393
	 * We disallowed uncharge of pages under migration because mapcount
	 * of the page goes down to zero, temporarly.
	 * Clear the flag and check the page should be charged.
3394
	 */
3395 3396 3397 3398 3399
	pc = lookup_page_cgroup(oldpage);
	lock_page_cgroup(pc);
	ClearPageCgroupMigration(pc);
	unlock_page_cgroup(pc);

3400
	/*
3401 3402 3403 3404 3405 3406
	 * If a page is a file cache, radix-tree replacement is very atomic
	 * and we can skip this check. When it was an Anon page, its mapcount
	 * goes down to 0. But because we added MIGRATION flage, it's not
	 * uncharged yet. There are several case but page->mapcount check
	 * and USED bit check in mem_cgroup_uncharge_page() will do enough
	 * check. (see prepare_charge() also)
3407
	 */
3408
	if (anon)
3409
		mem_cgroup_uncharge_page(used);
3410
}
3411

3412 3413 3414 3415 3416 3417 3418 3419
/*
 * At replace page cache, newpage is not under any memcg but it's on
 * LRU. So, this function doesn't touch res_counter but handles LRU
 * in correct way. Both pages are locked so we cannot race with uncharge.
 */
void mem_cgroup_replace_page_cache(struct page *oldpage,
				  struct page *newpage)
{
3420
	struct mem_cgroup *memcg = NULL;
3421 3422 3423 3424 3425 3426 3427 3428 3429
	struct page_cgroup *pc;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;

	if (mem_cgroup_disabled())
		return;

	pc = lookup_page_cgroup(oldpage);
	/* fix accounting on old pages */
	lock_page_cgroup(pc);
3430 3431 3432 3433 3434
	if (PageCgroupUsed(pc)) {
		memcg = pc->mem_cgroup;
		mem_cgroup_charge_statistics(memcg, false, -1);
		ClearPageCgroupUsed(pc);
	}
3435 3436
	unlock_page_cgroup(pc);

3437 3438 3439 3440 3441 3442
	/*
	 * When called from shmem_replace_page(), in some cases the
	 * oldpage has already been charged, and in some cases not.
	 */
	if (!memcg)
		return;
3443 3444 3445 3446 3447
	/*
	 * Even if newpage->mapping was NULL before starting replacement,
	 * the newpage may be on LRU(or pagevec for LRU) already. We lock
	 * LRU while we overwrite pc->mem_cgroup.
	 */
3448
	__mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
3449 3450
}

3451 3452 3453 3454 3455 3456
#ifdef CONFIG_DEBUG_VM
static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
3457 3458 3459 3460 3461
	/*
	 * Can be NULL while feeding pages into the page allocator for
	 * the first time, i.e. during boot or memory hotplug;
	 * or when mem_cgroup_disabled().
	 */
3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480
	if (likely(pc) && PageCgroupUsed(pc))
		return pc;
	return NULL;
}

bool mem_cgroup_bad_page_check(struct page *page)
{
	if (mem_cgroup_disabled())
		return false;

	return lookup_page_cgroup_used(page) != NULL;
}

void mem_cgroup_print_bad_page(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup_used(page);
	if (pc) {
3481
		printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
3482 3483 3484 3485 3486
		       pc, pc->flags, pc->mem_cgroup);
	}
}
#endif

3487 3488
static DEFINE_MUTEX(set_limit_mutex);

3489
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
3490
				unsigned long long val)
3491
{
3492
	int retry_count;
3493
	u64 memswlimit, memlimit;
3494
	int ret = 0;
3495 3496
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;
3497
	int enlarge;
3498 3499 3500 3501 3502 3503 3504 3505 3506

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3507

3508
	enlarge = 0;
3509
	while (retry_count) {
3510 3511 3512 3513
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
3514 3515 3516
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
3517
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
3518 3519 3520 3521 3522 3523
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
3524 3525
			break;
		}
3526 3527 3528 3529 3530

		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit < val)
			enlarge = 1;

3531
		ret = res_counter_set_limit(&memcg->res, val);
3532 3533 3534 3535 3536 3537
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3538 3539 3540 3541 3542
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3543 3544
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_SHRINK);
3545 3546 3547 3548 3549 3550
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
3551
	}
3552 3553
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3554

3555 3556 3557
	return ret;
}

L
Li Zefan 已提交
3558 3559
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
3560
{
3561
	int retry_count;
3562
	u64 memlimit, memswlimit, oldusage, curusage;
3563 3564
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
3565
	int enlarge = 0;
3566

3567 3568 3569
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3570 3571 3572 3573 3574 3575 3576 3577
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
3578
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
3579 3580 3581 3582 3583 3584 3585 3586
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
3587 3588 3589
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val)
			enlarge = 1;
3590
		ret = res_counter_set_limit(&memcg->memsw, val);
3591 3592 3593 3594 3595 3596
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
3597 3598 3599 3600 3601
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

3602 3603 3604
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_NOSWAP |
				   MEM_CGROUP_RECLAIM_SHRINK);
3605
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3606
		/* Usage is reduced ? */
3607
		if (curusage >= oldusage)
3608
			retry_count--;
3609 3610
		else
			oldusage = curusage;
3611
	}
3612 3613
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3614 3615 3616
	return ret;
}

3617
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3618 3619
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
3620 3621 3622 3623 3624 3625
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
3626
	unsigned long long excess;
3627
	unsigned long nr_scanned;
3628 3629 3630 3631

	if (order > 0)
		return 0;

3632
	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

3646
		nr_scanned = 0;
3647
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
3648
						    gfp_mask, &nr_scanned);
3649
		nr_reclaimed += reclaimed;
3650
		*total_scanned += nr_scanned;
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
3673
				if (next_mz == mz)
3674
					css_put(&next_mz->memcg->css);
3675
				else /* next_mz == NULL or other memcg */
3676 3677 3678
					break;
			} while (1);
		}
3679 3680
		__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
		excess = res_counter_soft_limit_excess(&mz->memcg->res);
3681 3682 3683 3684 3685 3686 3687 3688
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
3689
		/* If excess == 0, no tree ops */
3690
		__mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
3691
		spin_unlock(&mctz->lock);
3692
		css_put(&mz->memcg->css);
3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
3705
		css_put(&next_mz->memcg->css);
3706 3707 3708
	return nr_reclaimed;
}

3709 3710 3711 3712 3713 3714 3715
/**
 * mem_cgroup_force_empty_list - clears LRU of a group
 * @memcg: group to clear
 * @node: NUMA node
 * @zid: zone id
 * @lru: lru to to clear
 *
3716
 * Traverse a specified page_cgroup list and try to drop them all.  This doesn't
3717 3718
 * reclaim the pages page themselves - pages are moved to the parent (or root)
 * group.
3719
 */
3720
static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
K
KAMEZAWA Hiroyuki 已提交
3721
				int node, int zid, enum lru_list lru)
3722
{
3723
	struct lruvec *lruvec;
3724
	unsigned long flags;
3725
	struct list_head *list;
3726 3727
	struct page *busy;
	struct zone *zone;
3728

K
KAMEZAWA Hiroyuki 已提交
3729
	zone = &NODE_DATA(node)->node_zones[zid];
3730 3731
	lruvec = mem_cgroup_zone_lruvec(zone, memcg);
	list = &lruvec->lists[lru];
3732

3733
	busy = NULL;
3734
	do {
3735
		struct page_cgroup *pc;
3736 3737
		struct page *page;

K
KAMEZAWA Hiroyuki 已提交
3738
		spin_lock_irqsave(&zone->lru_lock, flags);
3739
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
3740
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3741
			break;
3742
		}
3743 3744 3745
		page = list_entry(list->prev, struct page, lru);
		if (busy == page) {
			list_move(&page->lru, list);
3746
			busy = NULL;
K
KAMEZAWA Hiroyuki 已提交
3747
			spin_unlock_irqrestore(&zone->lru_lock, flags);
3748 3749
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
3750
		spin_unlock_irqrestore(&zone->lru_lock, flags);
3751

3752
		pc = lookup_page_cgroup(page);
3753

3754
		if (mem_cgroup_move_parent(page, pc, memcg)) {
3755
			/* found lock contention or "pc" is obsolete. */
3756
			busy = page;
3757 3758 3759
			cond_resched();
		} else
			busy = NULL;
3760
	} while (!list_empty(list));
3761 3762 3763
}

/*
3764 3765
 * make mem_cgroup's charge to be 0 if there is no task by moving
 * all the charges and pages to the parent.
3766
 * This enables deleting this mem_cgroup.
3767 3768
 *
 * Caller is responsible for holding css reference on the memcg.
3769
 */
3770
static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
3771
{
3772
	int node, zid;
3773

3774
	do {
3775 3776
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
3777 3778
		drain_all_stock_sync(memcg);
		mem_cgroup_start_move(memcg);
3779
		for_each_node_state(node, N_HIGH_MEMORY) {
3780
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
H
Hugh Dickins 已提交
3781 3782
				enum lru_list lru;
				for_each_lru(lru) {
3783
					mem_cgroup_force_empty_list(memcg,
H
Hugh Dickins 已提交
3784
							node, zid, lru);
3785
				}
3786
			}
3787
		}
3788 3789
		mem_cgroup_end_move(memcg);
		memcg_oom_recover(memcg);
3790
		cond_resched();
3791

3792 3793 3794 3795 3796 3797 3798 3799
		/*
		 * This is a safety check because mem_cgroup_force_empty_list
		 * could have raced with mem_cgroup_replace_page_cache callers
		 * so the lru seemed empty but the page could have been added
		 * right after the check. RES_USAGE should be safe as we always
		 * charge before adding to the LRU.
		 */
	} while (res_counter_read_u64(&memcg->res, RES_USAGE) > 0);
3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811
}

/*
 * Reclaims as many pages from the given memcg as possible and moves
 * the rest to the parent.
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
	struct cgroup *cgrp = memcg->css.cgroup;
3812

3813
	/* returns EBUSY if there is a task or if we come here twice. */
3814 3815 3816
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
		return -EBUSY;

3817 3818
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
3819
	/* try to free all pages in this cgroup */
3820
	while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
3821
		int progress;
3822

3823 3824 3825
		if (signal_pending(current))
			return -EINTR;

3826
		progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
3827
						false);
3828
		if (!progress) {
3829
			nr_retries--;
3830
			/* maybe some writeback is necessary */
3831
			congestion_wait(BLK_RW_ASYNC, HZ/10);
3832
		}
3833 3834

	}
K
KAMEZAWA Hiroyuki 已提交
3835
	lru_add_drain();
3836 3837 3838
	mem_cgroup_reparent_charges(memcg);

	return 0;
3839 3840
}

3841
static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3842
{
3843 3844 3845
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
	int ret;

3846 3847
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
3848 3849 3850 3851 3852
	css_get(&memcg->css);
	ret = mem_cgroup_force_empty(memcg);
	css_put(&memcg->css);

	return ret;
3853 3854 3855
}


3856 3857 3858 3859 3860 3861 3862 3863 3864
static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
{
	return mem_cgroup_from_cont(cont)->use_hierarchy;
}

static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
					u64 val)
{
	int retval = 0;
3865
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3866
	struct cgroup *parent = cont->parent;
3867
	struct mem_cgroup *parent_memcg = NULL;
3868 3869

	if (parent)
3870
		parent_memcg = mem_cgroup_from_cont(parent);
3871 3872

	cgroup_lock();
3873 3874 3875 3876

	if (memcg->use_hierarchy == val)
		goto out;

3877
	/*
3878
	 * If parent's use_hierarchy is set, we can't make any modifications
3879 3880 3881 3882 3883 3884
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
3885
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3886 3887
				(val == 1 || val == 0)) {
		if (list_empty(&cont->children))
3888
			memcg->use_hierarchy = val;
3889 3890 3891 3892
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
3893 3894

out:
3895 3896 3897 3898 3899
	cgroup_unlock();

	return retval;
}

3900

3901
static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
3902
					       enum mem_cgroup_stat_index idx)
3903
{
K
KAMEZAWA Hiroyuki 已提交
3904
	struct mem_cgroup *iter;
3905
	long val = 0;
3906

3907
	/* Per-cpu values can be negative, use a signed accumulator */
3908
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
3909 3910 3911 3912 3913
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
3914 3915
}

3916
static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3917
{
K
KAMEZAWA Hiroyuki 已提交
3918
	u64 val;
3919

3920
	if (!mem_cgroup_is_root(memcg)) {
3921
		if (!swap)
3922
			return res_counter_read_u64(&memcg->res, RES_USAGE);
3923
		else
3924
			return res_counter_read_u64(&memcg->memsw, RES_USAGE);
3925 3926
	}

3927 3928
	val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
3929

K
KAMEZAWA Hiroyuki 已提交
3930
	if (swap)
3931
		val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
3932 3933 3934 3935

	return val << PAGE_SHIFT;
}

3936 3937 3938
static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
			       struct file *file, char __user *buf,
			       size_t nbytes, loff_t *ppos)
B
Balbir Singh 已提交
3939
{
3940
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3941
	char str[64];
3942
	u64 val;
3943
	int type, name, len;
3944 3945 3946

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
3947 3948 3949 3950

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

3951 3952
	switch (type) {
	case _MEM:
3953
		if (name == RES_USAGE)
3954
			val = mem_cgroup_usage(memcg, false);
3955
		else
3956
			val = res_counter_read_u64(&memcg->res, name);
3957 3958
		break;
	case _MEMSWAP:
3959
		if (name == RES_USAGE)
3960
			val = mem_cgroup_usage(memcg, true);
3961
		else
3962
			val = res_counter_read_u64(&memcg->memsw, name);
3963 3964 3965 3966
		break;
	default:
		BUG();
	}
3967 3968 3969

	len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
	return simple_read_from_buffer(buf, nbytes, ppos, str, len);
B
Balbir Singh 已提交
3970
}
3971 3972 3973 3974
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3975 3976
static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
			    const char *buffer)
B
Balbir Singh 已提交
3977
{
3978
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3979
	int type, name;
3980 3981 3982
	unsigned long long val;
	int ret;

3983 3984
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
3985 3986 3987 3988

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

3989
	switch (name) {
3990
	case RES_LIMIT:
3991 3992 3993 3994
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3995 3996
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
3997 3998 3999
		if (ret)
			break;
		if (type == _MEM)
4000
			ret = mem_cgroup_resize_limit(memcg, val);
4001 4002
		else
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
4003
		break;
4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
4018 4019 4020 4021 4022
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
4023 4024
}

4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	struct cgroup *cgroup;
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	cgroup = memcg->css.cgroup;
	if (!memcg->use_hierarchy)
		goto out;

	while (cgroup->parent) {
		cgroup = cgroup->parent;
		memcg = mem_cgroup_from_cont(cgroup);
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
}

4052
static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
4053
{
4054
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4055
	int type, name;
4056

4057 4058
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
4059 4060 4061 4062

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

4063
	switch (name) {
4064
	case RES_MAX_USAGE:
4065
		if (type == _MEM)
4066
			res_counter_reset_max(&memcg->res);
4067
		else
4068
			res_counter_reset_max(&memcg->memsw);
4069 4070
		break;
	case RES_FAILCNT:
4071
		if (type == _MEM)
4072
			res_counter_reset_failcnt(&memcg->res);
4073
		else
4074
			res_counter_reset_failcnt(&memcg->memsw);
4075 4076
		break;
	}
4077

4078
	return 0;
4079 4080
}

4081 4082 4083 4084 4085 4086
static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
					struct cftype *cft)
{
	return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
}

4087
#ifdef CONFIG_MMU
4088 4089 4090
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
4091
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4092 4093 4094 4095 4096 4097 4098 4099 4100

	if (val >= (1 << NR_MOVE_TYPE))
		return -EINVAL;
	/*
	 * We check this value several times in both in can_attach() and
	 * attach(), so we need cgroup lock to prevent this value from being
	 * inconsistent.
	 */
	cgroup_lock();
4101
	memcg->move_charge_at_immigrate = val;
4102 4103 4104 4105
	cgroup_unlock();

	return 0;
}
4106 4107 4108 4109 4110 4111 4112
#else
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
4113

4114
#ifdef CONFIG_NUMA
4115
static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
4116
				      struct seq_file *m)
4117 4118 4119 4120
{
	int nid;
	unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
	unsigned long node_nr;
4121
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4122

4123
	total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
4124 4125
	seq_printf(m, "total=%lu", total_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4126
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
4127 4128 4129 4130
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4131
	file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
4132 4133
	seq_printf(m, "file=%lu", file_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4134
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
4135
				LRU_ALL_FILE);
4136 4137 4138 4139
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4140
	anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
4141 4142
	seq_printf(m, "anon=%lu", anon_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4143
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
4144
				LRU_ALL_ANON);
4145 4146 4147 4148
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

4149
	unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
4150 4151
	seq_printf(m, "unevictable=%lu", unevictable_nr);
	for_each_node_state(nid, N_HIGH_MEMORY) {
4152
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
4153
				BIT(LRU_UNEVICTABLE));
4154 4155 4156 4157 4158 4159 4160
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');
	return 0;
}
#endif /* CONFIG_NUMA */

4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173
static const char * const mem_cgroup_lru_names[] = {
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

static inline void mem_cgroup_lru_names_not_uptodate(void)
{
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
}

4174
static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
4175
				 struct seq_file *m)
4176
{
4177
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4178 4179
	struct mem_cgroup *mi;
	unsigned int i;
4180

4181
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
4182
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
4183
			continue;
4184 4185
		seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
4186
	}
L
Lee Schermerhorn 已提交
4187

4188 4189 4190 4191 4192 4193 4194 4195
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
			   mem_cgroup_read_events(memcg, i));

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
4196
	/* Hierarchical information */
4197 4198
	{
		unsigned long long limit, memsw_limit;
4199
		memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
4200
		seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
4201
		if (do_swap_account)
4202 4203
			seq_printf(m, "hierarchical_memsw_limit %llu\n",
				   memsw_limit);
4204
	}
K
KOSAKI Motohiro 已提交
4205

4206 4207 4208
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
		long long val = 0;

4209
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
4210
			continue;
4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230
		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
		seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
	}

	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_events(mi, i);
		seq_printf(m, "total_%s %llu\n",
			   mem_cgroup_events_names[i], val);
	}

	for (i = 0; i < NR_LRU_LISTS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
4231
	}
K
KAMEZAWA Hiroyuki 已提交
4232

K
KOSAKI Motohiro 已提交
4233 4234 4235 4236
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
4237
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
4238 4239 4240 4241 4242
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4243
				mz = mem_cgroup_zoneinfo(memcg, nid, zid);
4244
				rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
4245

4246 4247 4248 4249
				recent_rotated[0] += rstat->recent_rotated[0];
				recent_rotated[1] += rstat->recent_rotated[1];
				recent_scanned[0] += rstat->recent_scanned[0];
				recent_scanned[1] += rstat->recent_scanned[1];
K
KOSAKI Motohiro 已提交
4250
			}
4251 4252 4253 4254
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
4255 4256 4257
	}
#endif

4258 4259 4260
	return 0;
}

K
KOSAKI Motohiro 已提交
4261 4262 4263 4264
static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);

4265
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
4266 4267 4268 4269 4270 4271 4272
}

static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
				       u64 val)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;
4273

K
KOSAKI Motohiro 已提交
4274 4275 4276 4277 4278 4279 4280
	if (val > 100)
		return -EINVAL;

	if (cgrp->parent == NULL)
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);
4281 4282 4283

	cgroup_lock();

K
KOSAKI Motohiro 已提交
4284 4285
	/* If under hierarchy, only empty-root can set this value */
	if ((parent->use_hierarchy) ||
4286 4287
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
		cgroup_unlock();
K
KOSAKI Motohiro 已提交
4288
		return -EINVAL;
4289
	}
K
KOSAKI Motohiro 已提交
4290 4291 4292

	memcg->swappiness = val;

4293 4294
	cgroup_unlock();

K
KOSAKI Motohiro 已提交
4295 4296 4297
	return 0;
}

4298 4299 4300 4301 4302 4303 4304 4305
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
	u64 usage;
	int i;

	rcu_read_lock();
	if (!swap)
4306
		t = rcu_dereference(memcg->thresholds.primary);
4307
	else
4308
		t = rcu_dereference(memcg->memsw_thresholds.primary);
4309 4310 4311 4312 4313 4314 4315

	if (!t)
		goto unlock;

	usage = mem_cgroup_usage(memcg, swap);

	/*
4316
	 * current_threshold points to threshold just below or equal to usage.
4317 4318 4319
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
4320
	i = t->current_threshold;
4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
4344
	t->current_threshold = i - 1;
4345 4346 4347 4348 4349 4350
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
4351 4352 4353 4354 4355 4356 4357
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
4358 4359 4360 4361 4362 4363 4364 4365 4366 4367
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

	return _a->threshold - _b->threshold;
}

4368
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4369 4370 4371
{
	struct mem_cgroup_eventfd_list *ev;

4372
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
4373 4374 4375 4376
		eventfd_signal(ev->eventfd, 1);
	return 0;
}

4377
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4378
{
K
KAMEZAWA Hiroyuki 已提交
4379 4380
	struct mem_cgroup *iter;

4381
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
4382
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
4383 4384 4385 4386
}

static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4387 4388
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4389 4390
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4391 4392
	int type = MEMFILE_TYPE(cft->private);
	u64 threshold, usage;
4393
	int i, size, ret;
4394 4395 4396 4397 4398 4399

	ret = res_counter_memparse_write_strategy(args, &threshold);
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
4400

4401
	if (type == _MEM)
4402
		thresholds = &memcg->thresholds;
4403
	else if (type == _MEMSWAP)
4404
		thresholds = &memcg->memsw_thresholds;
4405 4406 4407 4408 4409 4410
	else
		BUG();

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before adding a new one */
4411
	if (thresholds->primary)
4412 4413
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

4414
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4415 4416

	/* Allocate memory for new array of thresholds */
4417
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
4418
			GFP_KERNEL);
4419
	if (!new) {
4420 4421 4422
		ret = -ENOMEM;
		goto unlock;
	}
4423
	new->size = size;
4424 4425

	/* Copy thresholds (if any) to new array */
4426 4427
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
4428
				sizeof(struct mem_cgroup_threshold));
4429 4430
	}

4431
	/* Add new threshold */
4432 4433
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
4434 4435

	/* Sort thresholds. Registering of new threshold isn't time-critical */
4436
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
4437 4438 4439
			compare_thresholds, NULL);

	/* Find current threshold */
4440
	new->current_threshold = -1;
4441
	for (i = 0; i < size; i++) {
4442
		if (new->entries[i].threshold <= usage) {
4443
			/*
4444 4445
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
4446 4447
			 * it here.
			 */
4448
			++new->current_threshold;
4449 4450
		} else
			break;
4451 4452
	}

4453 4454 4455 4456 4457
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
4458

4459
	/* To be sure that nobody uses thresholds */
4460 4461 4462 4463 4464 4465 4466 4467
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

4468
static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4469
	struct cftype *cft, struct eventfd_ctx *eventfd)
4470 4471
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4472 4473
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4474 4475
	int type = MEMFILE_TYPE(cft->private);
	u64 usage;
4476
	int i, j, size;
4477 4478 4479

	mutex_lock(&memcg->thresholds_lock);
	if (type == _MEM)
4480
		thresholds = &memcg->thresholds;
4481
	else if (type == _MEMSWAP)
4482
		thresholds = &memcg->memsw_thresholds;
4483 4484 4485
	else
		BUG();

4486 4487 4488
	if (!thresholds->primary)
		goto unlock;

4489 4490 4491 4492 4493 4494
	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
4495 4496 4497
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
4498 4499 4500
			size++;
	}

4501
	new = thresholds->spare;
4502

4503 4504
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
4505 4506
		kfree(new);
		new = NULL;
4507
		goto swap_buffers;
4508 4509
	}

4510
	new->size = size;
4511 4512

	/* Copy thresholds and find current threshold */
4513 4514 4515
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
4516 4517
			continue;

4518
		new->entries[j] = thresholds->primary->entries[i];
4519
		if (new->entries[j].threshold <= usage) {
4520
			/*
4521
			 * new->current_threshold will not be used
4522 4523 4524
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
4525
			++new->current_threshold;
4526 4527 4528 4529
		}
		j++;
	}

4530
swap_buffers:
4531 4532
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
4533 4534 4535 4536 4537 4538
	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}

4539
	rcu_assign_pointer(thresholds->primary, new);
4540

4541
	/* To be sure that nobody uses thresholds */
4542
	synchronize_rcu();
4543
unlock:
4544 4545
	mutex_unlock(&memcg->thresholds_lock);
}
4546

K
KAMEZAWA Hiroyuki 已提交
4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558
static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *event;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);
	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

4559
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4560 4561 4562 4563 4564

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
4565
	if (atomic_read(&memcg->under_oom))
K
KAMEZAWA Hiroyuki 已提交
4566
		eventfd_signal(eventfd, 1);
4567
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4568 4569 4570 4571

	return 0;
}

4572
static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
4573 4574
	struct cftype *cft, struct eventfd_ctx *eventfd)
{
4575
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
K
KAMEZAWA Hiroyuki 已提交
4576 4577 4578 4579 4580
	struct mem_cgroup_eventfd_list *ev, *tmp;
	int type = MEMFILE_TYPE(cft->private);

	BUG_ON(type != _OOM_TYPE);

4581
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4582

4583
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
4584 4585 4586 4587 4588 4589
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

4590
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4591 4592
}

4593 4594 4595
static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
	struct cftype *cft,  struct cgroup_map_cb *cb)
{
4596
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4597

4598
	cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
4599

4600
	if (atomic_read(&memcg->under_oom))
4601 4602 4603 4604 4605 4606 4607 4608 4609
		cb->fill(cb, "under_oom", 1);
	else
		cb->fill(cb, "under_oom", 0);
	return 0;
}

static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
	struct cftype *cft, u64 val)
{
4610
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
	struct mem_cgroup *parent;

	/* cannot set to root cgroup and only 0 and 1 are allowed */
	if (!cgrp->parent || !((val == 0) || (val == 1)))
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);

	cgroup_lock();
	/* oom-kill-disable is a flag for subhierarchy. */
	if ((parent->use_hierarchy) ||
4622
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
4623 4624 4625
		cgroup_unlock();
		return -EINVAL;
	}
4626
	memcg->oom_kill_disable = val;
4627
	if (!val)
4628
		memcg_oom_recover(memcg);
4629 4630 4631 4632
	cgroup_unlock();
	return 0;
}

A
Andrew Morton 已提交
4633
#ifdef CONFIG_MEMCG_KMEM
4634
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
4635
{
4636
	return mem_cgroup_sockets_init(memcg, ss);
4637 4638
};

4639
static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
4640
{
4641
	mem_cgroup_sockets_destroy(memcg);
G
Glauber Costa 已提交
4642
}
4643
#else
4644
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
4645 4646 4647
{
	return 0;
}
G
Glauber Costa 已提交
4648

4649
static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
4650 4651
{
}
4652 4653
#endif

B
Balbir Singh 已提交
4654 4655
static struct cftype mem_cgroup_files[] = {
	{
4656
		.name = "usage_in_bytes",
4657
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4658
		.read = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4659 4660
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
B
Balbir Singh 已提交
4661
	},
4662 4663
	{
		.name = "max_usage_in_bytes",
4664
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4665
		.trigger = mem_cgroup_reset,
4666
		.read = mem_cgroup_read,
4667
	},
B
Balbir Singh 已提交
4668
	{
4669
		.name = "limit_in_bytes",
4670
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4671
		.write_string = mem_cgroup_write,
4672
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
4673
	},
4674 4675 4676 4677
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
4678
		.read = mem_cgroup_read,
4679
	},
B
Balbir Singh 已提交
4680 4681
	{
		.name = "failcnt",
4682
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4683
		.trigger = mem_cgroup_reset,
4684
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
4685
	},
4686 4687
	{
		.name = "stat",
4688
		.read_seq_string = memcg_stat_show,
4689
	},
4690 4691 4692 4693
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
4694 4695 4696 4697 4698
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
4699 4700 4701 4702 4703
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4704 4705 4706 4707 4708
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4709 4710
	{
		.name = "oom_control",
4711 4712
		.read_map = mem_cgroup_oom_control_read,
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4713 4714 4715 4716
		.register_event = mem_cgroup_oom_register_event,
		.unregister_event = mem_cgroup_oom_unregister_event,
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
4717 4718 4719
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
4720
		.read_seq_string = memcg_numa_stat_show,
4721 4722
	},
#endif
A
Andrew Morton 已提交
4723
#ifdef CONFIG_MEMCG_SWAP
4724 4725 4726
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4727
		.read = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
4728 4729
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
4730 4731 4732 4733 4734
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
4735
		.read = mem_cgroup_read,
4736 4737 4738 4739 4740
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
4741
		.read = mem_cgroup_read,
4742 4743 4744 4745 4746
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
4747
		.read = mem_cgroup_read,
4748 4749
	},
#endif
4750
	{ },	/* terminate */
4751
};
4752

4753
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4754 4755
{
	struct mem_cgroup_per_node *pn;
4756
	struct mem_cgroup_per_zone *mz;
4757
	int zone, tmp = node;
4758 4759 4760 4761 4762 4763 4764 4765
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4766 4767
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4768
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4769 4770
	if (!pn)
		return 1;
4771 4772 4773

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
4774
		lruvec_init(&mz->lruvec);
4775
		mz->usage_in_excess = 0;
4776
		mz->on_tree = false;
4777
		mz->memcg = memcg;
4778
	}
4779
	memcg->info.nodeinfo[node] = pn;
4780 4781 4782
	return 0;
}

4783
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4784
{
4785
	kfree(memcg->info.nodeinfo[node]);
4786 4787
}

4788 4789
static struct mem_cgroup *mem_cgroup_alloc(void)
{
4790
	struct mem_cgroup *memcg;
4791
	int size = sizeof(struct mem_cgroup);
4792

4793
	/* Can be very big if MAX_NUMNODES is very big */
4794
	if (size < PAGE_SIZE)
4795
		memcg = kzalloc(size, GFP_KERNEL);
4796
	else
4797
		memcg = vzalloc(size);
4798

4799
	if (!memcg)
4800 4801
		return NULL;

4802 4803
	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat)
4804
		goto out_free;
4805 4806
	spin_lock_init(&memcg->pcp_counter_lock);
	return memcg;
4807 4808 4809

out_free:
	if (size < PAGE_SIZE)
4810
		kfree(memcg);
4811
	else
4812
		vfree(memcg);
4813
	return NULL;
4814 4815
}

4816
/*
4817
 * Helpers for freeing a kmalloc()ed/vzalloc()ed mem_cgroup by RCU,
4818 4819 4820
 * but in process context.  The work_freeing structure is overlaid
 * on the rcu_freeing structure, which itself is overlaid on memsw.
 */
4821
static void free_work(struct work_struct *work)
4822 4823
{
	struct mem_cgroup *memcg;
4824
	int size = sizeof(struct mem_cgroup);
4825 4826

	memcg = container_of(work, struct mem_cgroup, work_freeing);
4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838
	/*
	 * We need to make sure that (at least for now), the jump label
	 * destruction code runs outside of the cgroup lock. This is because
	 * get_online_cpus(), which is called from the static_branch update,
	 * can't be called inside the cgroup_lock. cpusets are the ones
	 * enforcing this dependency, so if they ever change, we might as well.
	 *
	 * schedule_work() will guarantee this happens. Be careful if you need
	 * to move this code around, and make sure it is outside
	 * the cgroup_lock.
	 */
	disarm_sock_keys(memcg);
4839 4840 4841 4842
	if (size < PAGE_SIZE)
		kfree(memcg);
	else
		vfree(memcg);
4843
}
4844 4845

static void free_rcu(struct rcu_head *rcu_head)
4846 4847 4848 4849
{
	struct mem_cgroup *memcg;

	memcg = container_of(rcu_head, struct mem_cgroup, rcu_freeing);
4850
	INIT_WORK(&memcg->work_freeing, free_work);
4851 4852 4853
	schedule_work(&memcg->work_freeing);
}

4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864
/*
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
 */

4865
static void __mem_cgroup_free(struct mem_cgroup *memcg)
4866
{
K
KAMEZAWA Hiroyuki 已提交
4867 4868
	int node;

4869 4870
	mem_cgroup_remove_from_trees(memcg);
	free_css_id(&mem_cgroup_subsys, &memcg->css);
K
KAMEZAWA Hiroyuki 已提交
4871

B
Bob Liu 已提交
4872
	for_each_node(node)
4873
		free_mem_cgroup_per_zone_info(memcg, node);
K
KAMEZAWA Hiroyuki 已提交
4874

4875
	free_percpu(memcg->stat);
4876
	call_rcu(&memcg->rcu_freeing, free_rcu);
4877 4878
}

4879
static void mem_cgroup_get(struct mem_cgroup *memcg)
4880
{
4881
	atomic_inc(&memcg->refcnt);
4882 4883
}

4884
static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
4885
{
4886 4887 4888
	if (atomic_sub_and_test(count, &memcg->refcnt)) {
		struct mem_cgroup *parent = parent_mem_cgroup(memcg);
		__mem_cgroup_free(memcg);
4889 4890 4891
		if (parent)
			mem_cgroup_put(parent);
	}
4892 4893
}

4894
static void mem_cgroup_put(struct mem_cgroup *memcg)
4895
{
4896
	__mem_cgroup_put(memcg, 1);
4897 4898
}

4899 4900 4901
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
G
Glauber Costa 已提交
4902
struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
4903
{
4904
	if (!memcg->res.parent)
4905
		return NULL;
4906
	return mem_cgroup_from_res_counter(memcg->res.parent, res);
4907
}
G
Glauber Costa 已提交
4908
EXPORT_SYMBOL(parent_mem_cgroup);
4909

A
Andrew Morton 已提交
4910
#ifdef CONFIG_MEMCG_SWAP
4911 4912
static void __init enable_swap_cgroup(void)
{
4913
	if (!mem_cgroup_disabled() && really_do_swap_account)
4914 4915 4916 4917 4918 4919 4920 4921
		do_swap_account = 1;
}
#else
static void __init enable_swap_cgroup(void)
{
}
#endif

4922 4923 4924 4925 4926 4927
static int mem_cgroup_soft_limit_tree_init(void)
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

B
Bob Liu 已提交
4928
	for_each_node(node) {
4929 4930 4931 4932 4933
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
		if (!rtpn)
4934
			goto err_cleanup;
4935 4936 4937 4938 4939 4940 4941 4942 4943 4944

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
	return 0;
4945 4946

err_cleanup:
B
Bob Liu 已提交
4947
	for_each_node(node) {
4948 4949 4950 4951 4952 4953 4954
		if (!soft_limit_tree.rb_tree_per_node[node])
			break;
		kfree(soft_limit_tree.rb_tree_per_node[node]);
		soft_limit_tree.rb_tree_per_node[node] = NULL;
	}
	return 1;

4955 4956
}

L
Li Zefan 已提交
4957
static struct cgroup_subsys_state * __ref
4958
mem_cgroup_css_alloc(struct cgroup *cont)
B
Balbir Singh 已提交
4959
{
4960
	struct mem_cgroup *memcg, *parent;
K
KAMEZAWA Hiroyuki 已提交
4961
	long error = -ENOMEM;
4962
	int node;
B
Balbir Singh 已提交
4963

4964 4965
	memcg = mem_cgroup_alloc();
	if (!memcg)
K
KAMEZAWA Hiroyuki 已提交
4966
		return ERR_PTR(error);
4967

B
Bob Liu 已提交
4968
	for_each_node(node)
4969
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
4970
			goto free_out;
4971

4972
	/* root ? */
4973
	if (cont->parent == NULL) {
4974
		int cpu;
4975
		enable_swap_cgroup();
4976
		parent = NULL;
4977 4978
		if (mem_cgroup_soft_limit_tree_init())
			goto free_out;
4979
		root_mem_cgroup = memcg;
4980 4981 4982 4983 4984
		for_each_possible_cpu(cpu) {
			struct memcg_stock_pcp *stock =
						&per_cpu(memcg_stock, cpu);
			INIT_WORK(&stock->work, drain_local_stock);
		}
4985
		hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
4986
	} else {
4987
		parent = mem_cgroup_from_cont(cont->parent);
4988 4989
		memcg->use_hierarchy = parent->use_hierarchy;
		memcg->oom_kill_disable = parent->oom_kill_disable;
4990
	}
4991

4992
	if (parent && parent->use_hierarchy) {
4993 4994
		res_counter_init(&memcg->res, &parent->res);
		res_counter_init(&memcg->memsw, &parent->memsw);
4995 4996 4997 4998 4999 5000 5001
		/*
		 * We increment refcnt of the parent to ensure that we can
		 * safely access it on res_counter_charge/uncharge.
		 * This refcnt will be decremented when freeing this
		 * mem_cgroup(see mem_cgroup_put).
		 */
		mem_cgroup_get(parent);
5002
	} else {
5003 5004
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
5005 5006 5007 5008 5009 5010 5011
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
		if (parent && parent != root_mem_cgroup)
			mem_cgroup_subsys.broken_hierarchy = true;
5012
	}
5013 5014
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
5015

K
KOSAKI Motohiro 已提交
5016
	if (parent)
5017 5018 5019 5020
		memcg->swappiness = mem_cgroup_swappiness(parent);
	atomic_set(&memcg->refcnt, 1);
	memcg->move_charge_at_immigrate = 0;
	mutex_init(&memcg->thresholds_lock);
5021
	spin_lock_init(&memcg->move_lock);
5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032

	error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
	if (error) {
		/*
		 * We call put now because our (and parent's) refcnts
		 * are already in place. mem_cgroup_put() will internally
		 * call __mem_cgroup_free, so return directly
		 */
		mem_cgroup_put(memcg);
		return ERR_PTR(error);
	}
5033
	return &memcg->css;
5034
free_out:
5035
	__mem_cgroup_free(memcg);
K
KAMEZAWA Hiroyuki 已提交
5036
	return ERR_PTR(error);
B
Balbir Singh 已提交
5037 5038
}

5039
static void mem_cgroup_css_offline(struct cgroup *cont)
5040
{
5041
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5042

5043
	mem_cgroup_reparent_charges(memcg);
5044 5045
}

5046
static void mem_cgroup_css_free(struct cgroup *cont)
B
Balbir Singh 已提交
5047
{
5048
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5049

5050
	kmem_cgroup_destroy(memcg);
G
Glauber Costa 已提交
5051

5052
	mem_cgroup_put(memcg);
B
Balbir Singh 已提交
5053 5054
}

5055
#ifdef CONFIG_MMU
5056
/* Handlers for move charge at task migration. */
5057 5058
#define PRECHARGE_COUNT_AT_ONCE	256
static int mem_cgroup_do_precharge(unsigned long count)
5059
{
5060 5061
	int ret = 0;
	int batch_count = PRECHARGE_COUNT_AT_ONCE;
5062
	struct mem_cgroup *memcg = mc.to;
5063

5064
	if (mem_cgroup_is_root(memcg)) {
5065 5066 5067 5068 5069 5070 5071 5072
		mc.precharge += count;
		/* we don't need css_get for root */
		return ret;
	}
	/* try to charge at once */
	if (count > 1) {
		struct res_counter *dummy;
		/*
5073
		 * "memcg" cannot be under rmdir() because we've already checked
5074 5075 5076 5077
		 * by cgroup_lock_live_cgroup() that it is not removed and we
		 * are still under the same cgroup_mutex. So we can postpone
		 * css_get().
		 */
5078
		if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
5079
			goto one_by_one;
5080
		if (do_swap_account && res_counter_charge(&memcg->memsw,
5081
						PAGE_SIZE * count, &dummy)) {
5082
			res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
			goto one_by_one;
		}
		mc.precharge += count;
		return ret;
	}
one_by_one:
	/* fall back to one by one charge */
	while (count--) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		if (!batch_count--) {
			batch_count = PRECHARGE_COUNT_AT_ONCE;
			cond_resched();
		}
5099 5100
		ret = __mem_cgroup_try_charge(NULL,
					GFP_KERNEL, 1, &memcg, false);
5101
		if (ret)
5102
			/* mem_cgroup_clear_mc() will do uncharge later */
5103
			return ret;
5104 5105
		mc.precharge++;
	}
5106 5107 5108 5109
	return ret;
}

/**
5110
 * get_mctgt_type - get target type of moving charge
5111 5112 5113
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
5114
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
5115 5116 5117 5118 5119 5120
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
5121 5122 5123
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
5124 5125 5126 5127 5128
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
5129
	swp_entry_t	ent;
5130 5131 5132
};

enum mc_target_type {
5133
	MC_TARGET_NONE = 0,
5134
	MC_TARGET_PAGE,
5135
	MC_TARGET_SWAP,
5136 5137
};

D
Daisuke Nishimura 已提交
5138 5139
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
5140
{
D
Daisuke Nishimura 已提交
5141
	struct page *page = vm_normal_page(vma, addr, ptent);
5142

D
Daisuke Nishimura 已提交
5143 5144 5145 5146
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
		/* we don't move shared anon */
5147
		if (!move_anon())
D
Daisuke Nishimura 已提交
5148
			return NULL;
5149 5150
	} else if (!move_file())
		/* we ignore mapcount for file pages */
D
Daisuke Nishimura 已提交
5151 5152 5153 5154 5155 5156 5157
		return NULL;
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

5158
#ifdef CONFIG_SWAP
D
Daisuke Nishimura 已提交
5159 5160 5161 5162 5163 5164 5165 5166
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

	if (!move_anon() || non_swap_entry(ent))
		return NULL;
5167 5168 5169 5170 5171
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
	page = find_get_page(&swapper_space, ent.val);
D
Daisuke Nishimura 已提交
5172 5173 5174 5175 5176
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}
5177 5178 5179 5180 5181 5182 5183
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
5184

5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
	if (!move_file())
		return NULL;

	mapping = vma->vm_file->f_mapping;
	if (pte_none(ptent))
		pgoff = linear_page_index(vma, addr);
	else /* pte_file(ptent) is true */
		pgoff = pte_to_pgoff(ptent);

	/* page is moved even if it's not RSS of this task(page-faulted). */
5204 5205 5206 5207 5208 5209
	page = find_get_page(mapping, pgoff);

#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
	if (radix_tree_exceptional_entry(page)) {
		swp_entry_t swap = radix_to_swp_entry(page);
5210
		if (do_swap_account)
5211 5212
			*entry = swap;
		page = find_get_page(&swapper_space, swap.val);
5213
	}
5214
#endif
5215 5216 5217
	return page;
}

5218
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
5219 5220 5221 5222
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
5223
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
5224 5225 5226 5227 5228 5229
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
5230 5231
	else if (pte_none(ptent) || pte_file(ptent))
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
5232 5233

	if (!page && !ent.val)
5234
		return ret;
5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249
	if (page) {
		pc = lookup_page_cgroup(page);
		/*
		 * Do only loose check w/o page_cgroup lock.
		 * mem_cgroup_move_account() checks the pc is valid or not under
		 * the lock.
		 */
		if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
5250 5251
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
5252
			css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
5253 5254 5255
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
5256 5257 5258 5259
	}
	return ret;
}

5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
 * We don't consider swapping or file mapped pages because THP does not
 * support them for now.
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
	enum mc_target_type ret = MC_TARGET_NONE;

	page = pmd_page(pmd);
	VM_BUG_ON(!page || !PageHead(page));
	if (!move_anon())
		return ret;
	pc = lookup_page_cgroup(page);
	if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

5295 5296 5297 5298 5299 5300 5301 5302
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

5303 5304 5305 5306
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
		spin_unlock(&vma->vm_mm->page_table_lock);
5307
		return 0;
5308
	}
5309

5310 5311
	if (pmd_trans_unstable(pmd))
		return 0;
5312 5313
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
5314
		if (get_mctgt_type(vma, addr, *pte, NULL))
5315 5316 5317 5318
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

5319 5320 5321
	return 0;
}

5322 5323 5324 5325 5326
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;
	struct vm_area_struct *vma;

5327
	down_read(&mm->mmap_sem);
5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		struct mm_walk mem_cgroup_count_precharge_walk = {
			.pmd_entry = mem_cgroup_count_precharge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		walk_page_range(vma->vm_start, vma->vm_end,
					&mem_cgroup_count_precharge_walk);
	}
5339
	up_read(&mm->mmap_sem);
5340 5341 5342 5343 5344 5345 5346 5347 5348

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
5349 5350 5351 5352 5353
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
5354 5355
}

5356 5357
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
5358
{
5359 5360 5361
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

5362
	/* we must uncharge all the leftover precharges from mc.to */
5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373
	if (mc.precharge) {
		__mem_cgroup_cancel_charge(mc.to, mc.precharge);
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
		mc.moved_charge = 0;
5374
	}
5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
		if (!mem_cgroup_is_root(mc.from))
			res_counter_uncharge(&mc.from->memsw,
						PAGE_SIZE * mc.moved_swap);
		__mem_cgroup_put(mc.from, mc.moved_swap);

		if (!mem_cgroup_is_root(mc.to)) {
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			res_counter_uncharge(&mc.to->res,
						PAGE_SIZE * mc.moved_swap);
		}
		/* we've already done mem_cgroup_get(mc.to) */
		mc.moved_swap = 0;
	}
5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	struct mem_cgroup *from = mc.from;

	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
5409
	spin_lock(&mc.lock);
5410 5411
	mc.from = NULL;
	mc.to = NULL;
5412
	spin_unlock(&mc.lock);
5413
	mem_cgroup_end_move(from);
5414 5415
}

5416 5417
static int mem_cgroup_can_attach(struct cgroup *cgroup,
				 struct cgroup_taskset *tset)
5418
{
5419
	struct task_struct *p = cgroup_taskset_first(tset);
5420
	int ret = 0;
5421
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
5422

5423
	if (memcg->move_charge_at_immigrate) {
5424 5425 5426
		struct mm_struct *mm;
		struct mem_cgroup *from = mem_cgroup_from_task(p);

5427
		VM_BUG_ON(from == memcg);
5428 5429 5430 5431 5432

		mm = get_task_mm(p);
		if (!mm)
			return 0;
		/* We move charges only when we move a owner of the mm */
5433 5434 5435 5436
		if (mm->owner == p) {
			VM_BUG_ON(mc.from);
			VM_BUG_ON(mc.to);
			VM_BUG_ON(mc.precharge);
5437
			VM_BUG_ON(mc.moved_charge);
5438
			VM_BUG_ON(mc.moved_swap);
5439
			mem_cgroup_start_move(from);
5440
			spin_lock(&mc.lock);
5441
			mc.from = from;
5442
			mc.to = memcg;
5443
			spin_unlock(&mc.lock);
5444
			/* We set mc.moving_task later */
5445 5446 5447 5448

			ret = mem_cgroup_precharge_mc(mm);
			if (ret)
				mem_cgroup_clear_mc();
5449 5450
		}
		mmput(mm);
5451 5452 5453 5454
	}
	return ret;
}

5455 5456
static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
				     struct cgroup_taskset *tset)
5457
{
5458
	mem_cgroup_clear_mc();
5459 5460
}

5461 5462 5463
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
5464
{
5465 5466 5467 5468
	int ret = 0;
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;
5469 5470 5471 5472
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
	struct page_cgroup *pc;
5473

5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484
	/*
	 * We don't take compound_lock() here but no race with splitting thp
	 * happens because:
	 *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
	 *    under splitting, which means there's no concurrent thp split,
	 *  - if another thread runs into split_huge_page() just after we
	 *    entered this if-block, the thread must wait for page table lock
	 *    to be unlocked in __split_huge_page_splitting(), where the main
	 *    part of thp split is not executed yet.
	 */
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
5485
		if (mc.precharge < HPAGE_PMD_NR) {
5486 5487 5488 5489 5490 5491 5492 5493 5494
			spin_unlock(&vma->vm_mm->page_table_lock);
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
				pc = lookup_page_cgroup(page);
				if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
5495
							pc, mc.from, mc.to)) {
5496 5497 5498 5499 5500 5501 5502 5503
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
		}
		spin_unlock(&vma->vm_mm->page_table_lock);
5504
		return 0;
5505 5506
	}

5507 5508
	if (pmd_trans_unstable(pmd))
		return 0;
5509 5510 5511 5512
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
5513
		swp_entry_t ent;
5514 5515 5516 5517

		if (!mc.precharge)
			break;

5518
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
5519 5520 5521 5522 5523
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
			pc = lookup_page_cgroup(page);
5524
			if (!mem_cgroup_move_account(page, 1, pc,
5525
						     mc.from, mc.to)) {
5526
				mc.precharge--;
5527 5528
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
5529 5530
			}
			putback_lru_page(page);
5531
put:			/* get_mctgt_type() gets the page */
5532 5533
			put_page(page);
			break;
5534 5535
		case MC_TARGET_SWAP:
			ent = target.ent;
5536
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5537
				mc.precharge--;
5538 5539 5540
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
5541
			break;
5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
5556
		ret = mem_cgroup_do_precharge(1);
5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	lru_add_drain_all();
5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		int ret;
		struct mm_walk mem_cgroup_move_charge_walk = {
			.pmd_entry = mem_cgroup_move_charge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		ret = walk_page_range(vma->vm_start, vma->vm_end,
						&mem_cgroup_move_charge_walk);
		if (ret)
			/*
			 * means we have consumed all precharges and failed in
			 * doing additional charge. Just abandon here.
			 */
			break;
	}
5600
	up_read(&mm->mmap_sem);
5601 5602
}

5603 5604
static void mem_cgroup_move_task(struct cgroup *cont,
				 struct cgroup_taskset *tset)
B
Balbir Singh 已提交
5605
{
5606
	struct task_struct *p = cgroup_taskset_first(tset);
5607
	struct mm_struct *mm = get_task_mm(p);
5608 5609

	if (mm) {
5610 5611
		if (mc.to)
			mem_cgroup_move_charge(mm);
5612 5613
		mmput(mm);
	}
5614 5615
	if (mc.to)
		mem_cgroup_clear_mc();
B
Balbir Singh 已提交
5616
}
5617
#else	/* !CONFIG_MMU */
5618 5619
static int mem_cgroup_can_attach(struct cgroup *cgroup,
				 struct cgroup_taskset *tset)
5620 5621 5622
{
	return 0;
}
5623 5624
static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
				     struct cgroup_taskset *tset)
5625 5626
{
}
5627 5628
static void mem_cgroup_move_task(struct cgroup *cont,
				 struct cgroup_taskset *tset)
5629 5630 5631
{
}
#endif
B
Balbir Singh 已提交
5632

B
Balbir Singh 已提交
5633 5634 5635
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
5636 5637 5638
	.css_alloc = mem_cgroup_css_alloc,
	.css_offline = mem_cgroup_css_offline,
	.css_free = mem_cgroup_css_free,
5639 5640
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
5641
	.attach = mem_cgroup_move_task,
5642
	.base_cftypes = mem_cgroup_files,
5643
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
5644
	.use_id = 1,
B
Balbir Singh 已提交
5645
};
5646

A
Andrew Morton 已提交
5647
#ifdef CONFIG_MEMCG_SWAP
5648 5649 5650
static int __init enable_swap_account(char *s)
{
	/* consider enabled if no parameter or 1 is given */
5651
	if (!strcmp(s, "1"))
5652
		really_do_swap_account = 1;
5653
	else if (!strcmp(s, "0"))
5654 5655 5656
		really_do_swap_account = 0;
	return 1;
}
5657
__setup("swapaccount=", enable_swap_account);
5658 5659

#endif