memcontrol.c 85.9 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
23
#include <linux/mm.h>
K
KAMEZAWA Hiroyuki 已提交
24
#include <linux/pagemap.h>
25
#include <linux/smp.h>
26
#include <linux/page-flags.h>
27
#include <linux/backing-dev.h>
28 29
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
30
#include <linux/limits.h>
31
#include <linux/mutex.h>
32
#include <linux/rbtree.h>
33
#include <linux/slab.h>
34 35 36
#include <linux/swap.h>
#include <linux/spinlock.h>
#include <linux/fs.h>
37
#include <linux/seq_file.h>
38
#include <linux/vmalloc.h>
39
#include <linux/mm_inline.h>
40
#include <linux/page_cgroup.h>
41
#include <linux/cpu.h>
K
KAMEZAWA Hiroyuki 已提交
42
#include "internal.h"
B
Balbir Singh 已提交
43

44 45
#include <asm/uaccess.h>

46 47
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
#define MEM_CGROUP_RECLAIM_RETRIES	5
48
struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
49

50
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
L
Li Zefan 已提交
51
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
52 53 54 55 56 57
int do_swap_account __read_mostly;
static int really_do_swap_account __initdata = 1; /* for remember boot option*/
#else
#define do_swap_account		(0)
#endif

58
static DEFINE_MUTEX(memcg_tasklist);	/* can be hold under cgroup_mutex */
59
#define SOFTLIMIT_EVENTS_THRESH (1000)
60

61 62 63 64 65 66 67 68
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
69
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as anon rss */
70
	MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
71 72
	MEM_CGROUP_STAT_PGPGIN_COUNT,	/* # of pages paged in */
	MEM_CGROUP_STAT_PGPGOUT_COUNT,	/* # of pages paged out */
73
	MEM_CGROUP_STAT_EVENTS,	/* sum of pagein + pageout for internal use */
74
	MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
75 76 77 78 79 80 81 82 83

	MEM_CGROUP_STAT_NSTATS,
};

struct mem_cgroup_stat_cpu {
	s64 count[MEM_CGROUP_STAT_NSTATS];
} ____cacheline_aligned_in_smp;

struct mem_cgroup_stat {
84
	struct mem_cgroup_stat_cpu cpustat[0];
85 86
};

87 88 89 90 91 92 93 94 95 96 97 98 99 100
static inline void
__mem_cgroup_stat_reset_safe(struct mem_cgroup_stat_cpu *stat,
				enum mem_cgroup_stat_index idx)
{
	stat->count[idx] = 0;
}

static inline s64
__mem_cgroup_stat_read_local(struct mem_cgroup_stat_cpu *stat,
				enum mem_cgroup_stat_index idx)
{
	return stat->count[idx];
}

101 102 103
/*
 * For accounting under irq disable, no need for increment preempt count.
 */
104
static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
105 106
		enum mem_cgroup_stat_index idx, int val)
{
107
	stat->count[idx] += val;
108 109 110 111 112 113 114 115 116 117 118 119
}

static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
		enum mem_cgroup_stat_index idx)
{
	int cpu;
	s64 ret = 0;
	for_each_possible_cpu(cpu)
		ret += stat->cpustat[cpu].count[idx];
	return ret;
}

K
KAMEZAWA Hiroyuki 已提交
120 121 122 123 124 125 126 127 128
static s64 mem_cgroup_local_usage(struct mem_cgroup_stat *stat)
{
	s64 ret;

	ret = mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_CACHE);
	ret += mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_RSS);
	return ret;
}

129 130 131 132
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
133 134 135
	/*
	 * spin_lock to protect the per cgroup LRU
	 */
136 137
	struct list_head	lists[NR_LRU_LISTS];
	unsigned long		count[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
138 139

	struct zone_reclaim_stat reclaim_stat;
140 141 142 143
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
144 145
	struct mem_cgroup	*mem;		/* Back pointer, we cannot */
						/* use container_of	   */
146 147 148 149 150 151 152 153 154 155 156 157
};
/* Macro for accessing counter */
#define MEM_CGROUP_ZSTAT(mz, idx)	((mz)->count[(idx)])

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
	struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
};

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

B
Balbir Singh 已提交
178 179 180 181 182 183 184
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
185 186 187
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
188 189 190 191 192 193 194
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
195 196 197 198
	/*
	 * the counter to account for mem+swap usage.
	 */
	struct res_counter memsw;
199 200 201 202
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 */
203
	struct mem_cgroup_lru_info info;
204

K
KOSAKI Motohiro 已提交
205 206 207 208 209
	/*
	  protect against reclaim related member.
	*/
	spinlock_t reclaim_param_lock;

210
	int	prev_priority;	/* for recording reclaim priority */
211 212

	/*
213
	 * While reclaiming in a hierarchy, we cache the last child we
K
KAMEZAWA Hiroyuki 已提交
214
	 * reclaimed from.
215
	 */
K
KAMEZAWA Hiroyuki 已提交
216
	int last_scanned_child;
217 218 219 220
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
221
	unsigned long	last_oom_jiffies;
222
	atomic_t	refcnt;
223

K
KOSAKI Motohiro 已提交
224 225
	unsigned int	swappiness;

226 227 228
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

229
	/*
230
	 * statistics. This must be placed at the end of memcg.
231 232
	 */
	struct mem_cgroup_stat stat;
B
Balbir Singh 已提交
233 234
};

235 236 237 238 239 240 241
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		(100)
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	(2)

242 243 244
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
	MEM_CGROUP_CHARGE_TYPE_MAPPED,
245
	MEM_CGROUP_CHARGE_TYPE_SHMEM,	/* used by page migration of shmem */
246
	MEM_CGROUP_CHARGE_TYPE_FORCE,	/* used by force_empty */
K
KAMEZAWA Hiroyuki 已提交
247
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
248
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
249 250 251
	NR_CHARGE_TYPE,
};

252 253 254 255
/* only for here (for easy reading.) */
#define PCGF_CACHE	(1UL << PCG_CACHE)
#define PCGF_USED	(1UL << PCG_USED)
#define PCGF_LOCK	(1UL << PCG_LOCK)
256 257
/* Not used, but added here for completeness */
#define PCGF_ACCT	(1UL << PCG_ACCT)
258

259 260 261 262 263 264 265
/* for encoding cft->private value on file */
#define _MEM			(0)
#define _MEMSWAP		(1)
#define MEMFILE_PRIVATE(x, val)	(((x) << 16) | (val))
#define MEMFILE_TYPE(val)	(((val) >> 16) & 0xffff)
#define MEMFILE_ATTR(val)	((val) & 0xffff)

266 267 268 269 270 271 272
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
273 274
#define MEM_CGROUP_RECLAIM_SOFT_BIT	0x2
#define MEM_CGROUP_RECLAIM_SOFT		(1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
275

276 277
static void mem_cgroup_get(struct mem_cgroup *mem);
static void mem_cgroup_put(struct mem_cgroup *mem);
278
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
279
static void drain_all_stock_async(void);
280

281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
static struct mem_cgroup_per_zone *
mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
{
	return &mem->info.nodeinfo[nid]->zoneinfo[zid];
}

static struct mem_cgroup_per_zone *
page_cgroup_zoneinfo(struct page_cgroup *pc)
{
	struct mem_cgroup *mem = pc->mem_cgroup;
	int nid = page_cgroup_nid(pc);
	int zid = page_cgroup_zid(pc);

	if (!mem)
		return NULL;

	return mem_cgroup_zoneinfo(mem, nid, zid);
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
316
__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
317
				struct mem_cgroup_per_zone *mz,
318 319
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
320 321 322 323 324 325 326 327
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

328 329 330
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
347 348 349 350 351 352 353 354 355 356 357 358 359
}

static void
__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

360 361 362 363 364 365
static void
mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
366
	__mem_cgroup_remove_exceeded(mem, mz, mctz);
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
	spin_unlock(&mctz->lock);
}

static bool mem_cgroup_soft_limit_check(struct mem_cgroup *mem)
{
	bool ret = false;
	int cpu;
	s64 val;
	struct mem_cgroup_stat_cpu *cpustat;

	cpu = get_cpu();
	cpustat = &mem->stat.cpustat[cpu];
	val = __mem_cgroup_stat_read_local(cpustat, MEM_CGROUP_STAT_EVENTS);
	if (unlikely(val > SOFTLIMIT_EVENTS_THRESH)) {
		__mem_cgroup_stat_reset_safe(cpustat, MEM_CGROUP_STAT_EVENTS);
		ret = true;
	}
	put_cpu();
	return ret;
}

static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
{
390
	unsigned long long excess;
391 392
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
393 394
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
395 396 397
	mctz = soft_limit_tree_from_page(page);

	/*
398 399
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
400
	 */
401 402
	for (; mem; mem = parent_mem_cgroup(mem)) {
		mz = mem_cgroup_zoneinfo(mem, nid, zid);
403
		excess = res_counter_soft_limit_excess(&mem->res);
404 405 406 407
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
408
		if (excess || mz->on_tree) {
409 410 411 412 413
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
				__mem_cgroup_remove_exceeded(mem, mz, mctz);
			/*
414 415
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
416
			 */
417
			__mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
418 419
			spin_unlock(&mctz->lock);
		}
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

	for_each_node_state(node, N_POSSIBLE) {
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			mz = mem_cgroup_zoneinfo(mem, node, zone);
			mctz = soft_limit_tree_node_zone(node, zone);
			mem_cgroup_remove_exceeded(mem, mz, mctz);
		}
	}
}

438 439 440 441 442 443 444 445 446
static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
{
	return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
}

static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
447
	struct mem_cgroup_per_zone *mz;
448 449

retry:
450
	mz = NULL;
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
	__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->mem->res) ||
		!css_tryget(&mz->mem->css))
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

480 481 482 483 484 485 486 487 488 489 490 491 492
static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
					 bool charge)
{
	int val = (charge) ? 1 : -1;
	struct mem_cgroup_stat *stat = &mem->stat;
	struct mem_cgroup_stat_cpu *cpustat;
	int cpu = get_cpu();

	cpustat = &stat->cpustat[cpu];
	__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_SWAPOUT, val);
	put_cpu();
}

493 494 495
static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
					 struct page_cgroup *pc,
					 bool charge)
496
{
497
	int val = (charge) ? 1 : -1;
498
	struct mem_cgroup_stat *stat = &mem->stat;
499
	struct mem_cgroup_stat_cpu *cpustat;
K
KAMEZAWA Hiroyuki 已提交
500
	int cpu = get_cpu();
501

K
KAMEZAWA Hiroyuki 已提交
502
	cpustat = &stat->cpustat[cpu];
503
	if (PageCgroupCache(pc))
504
		__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
505
	else
506
		__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
507 508

	if (charge)
509
		__mem_cgroup_stat_add_safe(cpustat,
510 511
				MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
	else
512
		__mem_cgroup_stat_add_safe(cpustat,
513
				MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
514
	__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_EVENTS, 1);
K
KAMEZAWA Hiroyuki 已提交
515
	put_cpu();
516 517
}

K
KAMEZAWA Hiroyuki 已提交
518
static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
519
					enum lru_list idx)
520 521 522 523 524 525 526 527 528 529 530
{
	int nid, zid;
	struct mem_cgroup_per_zone *mz;
	u64 total = 0;

	for_each_online_node(nid)
		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			mz = mem_cgroup_zoneinfo(mem, nid, zid);
			total += MEM_CGROUP_ZSTAT(mz, idx);
		}
	return total;
531 532
}

533
static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
534 535 536 537 538 539
{
	return container_of(cgroup_subsys_state(cont,
				mem_cgroup_subsys_id), struct mem_cgroup,
				css);
}

540
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
541
{
542 543 544 545 546 547 548 549
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

550 551 552 553
	return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
				struct mem_cgroup, css);
}

554 555 556
static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
{
	struct mem_cgroup *mem = NULL;
557 558 559

	if (!mm)
		return NULL;
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
		mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!mem))
			break;
	} while (!css_tryget(&mem->css));
	rcu_read_unlock();
	return mem;
}

K
KAMEZAWA Hiroyuki 已提交
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
/*
 * Call callback function against all cgroup under hierarchy tree.
 */
static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
			  int (*func)(struct mem_cgroup *, void *))
{
	int found, ret, nextid;
	struct cgroup_subsys_state *css;
	struct mem_cgroup *mem;

	if (!root->use_hierarchy)
		return (*func)(root, data);

	nextid = 1;
	do {
		ret = 0;
		mem = NULL;

		rcu_read_lock();
		css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
				   &found);
		if (css && css_tryget(css))
			mem = container_of(css, struct mem_cgroup, css);
		rcu_read_unlock();

		if (mem) {
			ret = (*func)(mem, data);
			css_put(&mem->css);
		}
		nextid = found + 1;
	} while (!ret && css);

	return ret;
}

610 611 612 613 614
static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
{
	return (mem == root_mem_cgroup);
}

K
KAMEZAWA Hiroyuki 已提交
615 616 617 618 619 620 621 622 623 624 625 626 627
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
628

K
KAMEZAWA Hiroyuki 已提交
629 630 631 632
void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
633

634
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
635 636 637
		return;
	pc = lookup_page_cgroup(page);
	/* can happen while we handle swapcache. */
638
	if (!TestClearPageCgroupAcctLRU(pc))
K
KAMEZAWA Hiroyuki 已提交
639
		return;
640
	VM_BUG_ON(!pc->mem_cgroup);
641 642 643 644
	/*
	 * We don't check PCG_USED bit. It's cleared when the "page" is finally
	 * removed from global LRU.
	 */
K
KAMEZAWA Hiroyuki 已提交
645
	mz = page_cgroup_zoneinfo(pc);
646
	MEM_CGROUP_ZSTAT(mz, lru) -= 1;
647 648 649
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
	VM_BUG_ON(list_empty(&pc->lru));
K
KAMEZAWA Hiroyuki 已提交
650 651
	list_del_init(&pc->lru);
	return;
652 653
}

K
KAMEZAWA Hiroyuki 已提交
654
void mem_cgroup_del_lru(struct page *page)
655
{
K
KAMEZAWA Hiroyuki 已提交
656 657
	mem_cgroup_del_lru_list(page, page_lru(page));
}
658

K
KAMEZAWA Hiroyuki 已提交
659 660 661 662
void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;
	struct page_cgroup *pc;
663

664
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
665
		return;
666

K
KAMEZAWA Hiroyuki 已提交
667
	pc = lookup_page_cgroup(page);
668 669 670 671
	/*
	 * Used bit is set without atomic ops but after smp_wmb().
	 * For making pc->mem_cgroup visible, insert smp_rmb() here.
	 */
K
KAMEZAWA Hiroyuki 已提交
672
	smp_rmb();
673 674
	/* unused or root page is not rotated. */
	if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
K
KAMEZAWA Hiroyuki 已提交
675 676 677
		return;
	mz = page_cgroup_zoneinfo(pc);
	list_move(&pc->lru, &mz->lists[lru]);
678 679
}

K
KAMEZAWA Hiroyuki 已提交
680
void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
681
{
K
KAMEZAWA Hiroyuki 已提交
682 683
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;
684

685
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
686 687
		return;
	pc = lookup_page_cgroup(page);
688
	VM_BUG_ON(PageCgroupAcctLRU(pc));
689 690 691 692
	/*
	 * Used bit is set without atomic ops but after smp_wmb().
	 * For making pc->mem_cgroup visible, insert smp_rmb() here.
	 */
K
KAMEZAWA Hiroyuki 已提交
693 694
	smp_rmb();
	if (!PageCgroupUsed(pc))
L
Lee Schermerhorn 已提交
695
		return;
696

K
KAMEZAWA Hiroyuki 已提交
697
	mz = page_cgroup_zoneinfo(pc);
698
	MEM_CGROUP_ZSTAT(mz, lru) += 1;
699 700 701
	SetPageCgroupAcctLRU(pc);
	if (mem_cgroup_is_root(pc->mem_cgroup))
		return;
K
KAMEZAWA Hiroyuki 已提交
702 703
	list_add(&pc->lru, &mz->lists[lru]);
}
704

K
KAMEZAWA Hiroyuki 已提交
705
/*
706 707 708 709 710
 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
 * lru because the page may.be reused after it's fully uncharged (because of
 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
 * it again. This function is only used to charge SwapCache. It's done under
 * lock_page and expected that zone->lru_lock is never held.
K
KAMEZAWA Hiroyuki 已提交
711
 */
712
static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
K
KAMEZAWA Hiroyuki 已提交
713
{
714 715 716 717 718 719 720 721 722 723 724 725
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);

	spin_lock_irqsave(&zone->lru_lock, flags);
	/*
	 * Forget old LRU when this page_cgroup is *not* used. This Used bit
	 * is guarded by lock_page() because the page is SwapCache.
	 */
	if (!PageCgroupUsed(pc))
		mem_cgroup_del_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
K
KAMEZAWA Hiroyuki 已提交
726 727
}

728 729 730 731 732 733 734 735
static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
{
	unsigned long flags;
	struct zone *zone = page_zone(page);
	struct page_cgroup *pc = lookup_page_cgroup(page);

	spin_lock_irqsave(&zone->lru_lock, flags);
	/* link when the page is linked to LRU but page_cgroup isn't */
736
	if (PageLRU(page) && !PageCgroupAcctLRU(pc))
737 738 739 740 741
		mem_cgroup_add_lru_list(page, page_lru(page));
	spin_unlock_irqrestore(&zone->lru_lock, flags);
}


K
KAMEZAWA Hiroyuki 已提交
742 743 744
void mem_cgroup_move_lists(struct page *page,
			   enum lru_list from, enum lru_list to)
{
745
	if (mem_cgroup_disabled())
K
KAMEZAWA Hiroyuki 已提交
746 747 748
		return;
	mem_cgroup_del_lru_list(page, from);
	mem_cgroup_add_lru_list(page, to);
749 750
}

751 752 753
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
{
	int ret;
754
	struct mem_cgroup *curr = NULL;
755 756

	task_lock(task);
757 758 759
	rcu_read_lock();
	curr = try_get_mem_cgroup_from_mm(task->mm);
	rcu_read_unlock();
760
	task_unlock(task);
761 762 763 764 765 766 767
	if (!curr)
		return 0;
	if (curr->use_hierarchy)
		ret = css_is_ancestor(&curr->css, &mem->css);
	else
		ret = (curr == mem);
	css_put(&curr->css);
768 769 770
	return ret;
}

771 772 773 774 775
/*
 * prev_priority control...this will be used in memory reclaim path.
 */
int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
{
K
KOSAKI Motohiro 已提交
776 777 778 779 780 781 782
	int prev_priority;

	spin_lock(&mem->reclaim_param_lock);
	prev_priority = mem->prev_priority;
	spin_unlock(&mem->reclaim_param_lock);

	return prev_priority;
783 784 785 786
}

void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
{
K
KOSAKI Motohiro 已提交
787
	spin_lock(&mem->reclaim_param_lock);
788 789
	if (priority < mem->prev_priority)
		mem->prev_priority = priority;
K
KOSAKI Motohiro 已提交
790
	spin_unlock(&mem->reclaim_param_lock);
791 792 793 794
}

void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
{
K
KOSAKI Motohiro 已提交
795
	spin_lock(&mem->reclaim_param_lock);
796
	mem->prev_priority = priority;
K
KOSAKI Motohiro 已提交
797
	spin_unlock(&mem->reclaim_param_lock);
798 799
}

800
static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
801 802 803
{
	unsigned long active;
	unsigned long inactive;
804 805
	unsigned long gb;
	unsigned long inactive_ratio;
806

K
KAMEZAWA Hiroyuki 已提交
807 808
	inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
809

810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

	if (present_pages) {
		present_pages[0] = inactive;
		present_pages[1] = active;
	}

	return inactive_ratio;
}

int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
{
	unsigned long active;
	unsigned long inactive;
	unsigned long present_pages[2];
	unsigned long inactive_ratio;

	inactive_ratio = calc_inactive_ratio(memcg, present_pages);

	inactive = present_pages[0];
	active = present_pages[1];

	if (inactive * inactive_ratio < active)
837 838 839 840 841
		return 1;

	return 0;
}

842 843 844 845 846 847 848 849 850 851 852
int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
{
	unsigned long active;
	unsigned long inactive;

	inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
	active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);

	return (active > inactive);
}

853 854 855 856 857 858 859 860 861 862 863
unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
				       struct zone *zone,
				       enum lru_list lru)
{
	int nid = zone->zone_pgdat->node_id;
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

	return MEM_CGROUP_ZSTAT(mz, lru);
}

K
KOSAKI Motohiro 已提交
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
						      struct zone *zone)
{
	int nid = zone->zone_pgdat->node_id;
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

	return &mz->reclaim_stat;
}

struct zone_reclaim_stat *
mem_cgroup_get_reclaim_stat_from_page(struct page *page)
{
	struct page_cgroup *pc;
	struct mem_cgroup_per_zone *mz;

	if (mem_cgroup_disabled())
		return NULL;

	pc = lookup_page_cgroup(page);
884 885 886 887 888 889 890 891
	/*
	 * Used bit is set without atomic ops but after smp_wmb().
	 * For making pc->mem_cgroup visible, insert smp_rmb() here.
	 */
	smp_rmb();
	if (!PageCgroupUsed(pc))
		return NULL;

K
KOSAKI Motohiro 已提交
892 893 894 895 896 897 898
	mz = page_cgroup_zoneinfo(pc);
	if (!mz)
		return NULL;

	return &mz->reclaim_stat;
}

899 900 901 902 903
unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
					struct list_head *dst,
					unsigned long *scanned, int order,
					int mode, struct zone *z,
					struct mem_cgroup *mem_cont,
904
					int active, int file)
905 906 907 908 909 910
{
	unsigned long nr_taken = 0;
	struct page *page;
	unsigned long scan;
	LIST_HEAD(pc_list);
	struct list_head *src;
911
	struct page_cgroup *pc, *tmp;
912 913 914
	int nid = z->zone_pgdat->node_id;
	int zid = zone_idx(z);
	struct mem_cgroup_per_zone *mz;
915
	int lru = LRU_FILE * file + active;
916
	int ret;
917

918
	BUG_ON(!mem_cont);
919
	mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
920
	src = &mz->lists[lru];
921

922 923
	scan = 0;
	list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
H
Hugh Dickins 已提交
924
		if (scan >= nr_to_scan)
925
			break;
K
KAMEZAWA Hiroyuki 已提交
926 927

		page = pc->page;
928 929
		if (unlikely(!PageCgroupUsed(pc)))
			continue;
H
Hugh Dickins 已提交
930
		if (unlikely(!PageLRU(page)))
931 932
			continue;

H
Hugh Dickins 已提交
933
		scan++;
934 935 936
		ret = __isolate_lru_page(page, mode, file);
		switch (ret) {
		case 0:
937
			list_move(&page->lru, dst);
938
			mem_cgroup_del_lru(page);
939
			nr_taken++;
940 941 942 943 944 945 946
			break;
		case -EBUSY:
			/* we don't affect global LRU but rotate in our LRU */
			mem_cgroup_rotate_lru_list(page, page_lru(page));
			break;
		default:
			break;
947 948 949 950 951 952 953
		}
	}

	*scanned = scan;
	return nr_taken;
}

954 955 956
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

957 958 959 960 961 962 963 964 965 966 967 968
static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
{
	if (do_swap_account) {
		if (res_counter_check_under_limit(&mem->res) &&
			res_counter_check_under_limit(&mem->memsw))
			return true;
	} else
		if (res_counter_check_under_limit(&mem->res))
			return true;
	return false;
}

K
KOSAKI Motohiro 已提交
969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
static unsigned int get_swappiness(struct mem_cgroup *memcg)
{
	struct cgroup *cgrp = memcg->css.cgroup;
	unsigned int swappiness;

	/* root ? */
	if (cgrp->parent == NULL)
		return vm_swappiness;

	spin_lock(&memcg->reclaim_param_lock);
	swappiness = memcg->swappiness;
	spin_unlock(&memcg->reclaim_param_lock);

	return swappiness;
}

985 986 987 988 989 990
static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
{
	int *val = data;
	(*val)++;
	return 0;
}
991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058

/**
 * mem_cgroup_print_mem_info: Called from OOM with tasklist_lock held in read mode.
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;

	if (!memcg)
		return;


	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	printk(KERN_INFO "Task in %s killed", memcg_name);

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
	printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
done:

	printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
	printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
		"failcnt %llu\n",
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
}

1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
static int mem_cgroup_count_children(struct mem_cgroup *mem)
{
	int num = 0;
 	mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
	return num;
}

1070
/*
K
KAMEZAWA Hiroyuki 已提交
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
 * Visit the first child (need not be the first child as per the ordering
 * of the cgroup list, since we track last_scanned_child) of @mem and use
 * that to reclaim free pages from.
 */
static struct mem_cgroup *
mem_cgroup_select_victim(struct mem_cgroup *root_mem)
{
	struct mem_cgroup *ret = NULL;
	struct cgroup_subsys_state *css;
	int nextid, found;

	if (!root_mem->use_hierarchy) {
		css_get(&root_mem->css);
		ret = root_mem;
	}

	while (!ret) {
		rcu_read_lock();
		nextid = root_mem->last_scanned_child + 1;
		css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
				   &found);
		if (css && css_tryget(css))
			ret = container_of(css, struct mem_cgroup, css);

		rcu_read_unlock();
		/* Updates scanning parameter */
		spin_lock(&root_mem->reclaim_param_lock);
		if (!css) {
			/* this means start scan from ID:1 */
			root_mem->last_scanned_child = 0;
		} else
			root_mem->last_scanned_child = found;
		spin_unlock(&root_mem->reclaim_param_lock);
	}

	return ret;
}

/*
 * Scan the hierarchy if needed to reclaim memory. We remember the last child
 * we reclaimed from, so that we don't end up penalizing one child extensively
 * based on its position in the children list.
1113 1114
 *
 * root_mem is the original ancestor that we've been reclaim from.
K
KAMEZAWA Hiroyuki 已提交
1115 1116 1117
 *
 * We give up and return to the caller when we visit root_mem twice.
 * (other groups can be removed while we're walking....)
1118 1119
 *
 * If shrink==true, for avoiding to free too much, this returns immedieately.
1120 1121
 */
static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
1122
						struct zone *zone,
1123 1124
						gfp_t gfp_mask,
						unsigned long reclaim_options)
1125
{
K
KAMEZAWA Hiroyuki 已提交
1126 1127 1128
	struct mem_cgroup *victim;
	int ret, total = 0;
	int loop = 0;
1129 1130
	bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
	bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1131 1132
	bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
	unsigned long excess = mem_cgroup_get_excess(root_mem);
K
KAMEZAWA Hiroyuki 已提交
1133

1134 1135 1136 1137
	/* If memsw_is_minimum==1, swap-out is of-no-use. */
	if (root_mem->memsw_is_minimum)
		noswap = true;

1138
	while (1) {
K
KAMEZAWA Hiroyuki 已提交
1139
		victim = mem_cgroup_select_victim(root_mem);
1140
		if (victim == root_mem) {
K
KAMEZAWA Hiroyuki 已提交
1141
			loop++;
1142 1143
			if (loop >= 1)
				drain_all_stock_async();
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!check_soft || !total) {
					css_put(&victim->css);
					break;
				}
				/*
				 * We want to do more targetted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
					css_put(&victim->css);
					break;
				}
			}
		}
K
KAMEZAWA Hiroyuki 已提交
1167 1168 1169
		if (!mem_cgroup_local_usage(&victim->stat)) {
			/* this cgroup's local usage == 0 */
			css_put(&victim->css);
1170 1171
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
1172
		/* we use swappiness of local cgroup */
1173 1174 1175 1176 1177 1178 1179
		if (check_soft)
			ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
				noswap, get_swappiness(victim), zone,
				zone->zone_pgdat->node_id);
		else
			ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
						noswap, get_swappiness(victim));
K
KAMEZAWA Hiroyuki 已提交
1180
		css_put(&victim->css);
1181 1182 1183 1184 1185 1186 1187
		/*
		 * At shrinking usage, we can't check we should stop here or
		 * reclaim more. It's depends on callers. last_scanned_child
		 * will work enough for keeping fairness under tree.
		 */
		if (shrink)
			return ret;
K
KAMEZAWA Hiroyuki 已提交
1188
		total += ret;
1189 1190 1191 1192
		if (check_soft) {
			if (res_counter_check_under_soft_limit(&root_mem->res))
				return total;
		} else if (mem_cgroup_check_under_limit(root_mem))
K
KAMEZAWA Hiroyuki 已提交
1193
			return 1 + total;
1194
	}
K
KAMEZAWA Hiroyuki 已提交
1195
	return total;
1196 1197
}

1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
bool mem_cgroup_oom_called(struct task_struct *task)
{
	bool ret = false;
	struct mem_cgroup *mem;
	struct mm_struct *mm;

	rcu_read_lock();
	mm = task->mm;
	if (!mm)
		mm = &init_mm;
	mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (mem && time_before(jiffies, mem->last_oom_jiffies + HZ/10))
		ret = true;
	rcu_read_unlock();
	return ret;
}
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225

static int record_last_oom_cb(struct mem_cgroup *mem, void *data)
{
	mem->last_oom_jiffies = jiffies;
	return 0;
}

static void record_last_oom(struct mem_cgroup *mem)
{
	mem_cgroup_walk_tree(mem, NULL, record_last_oom_cb);
}

1226 1227 1228 1229
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
 */
1230
void mem_cgroup_update_file_mapped(struct page *page, int val)
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256
{
	struct mem_cgroup *mem;
	struct mem_cgroup_stat *stat;
	struct mem_cgroup_stat_cpu *cpustat;
	int cpu;
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
	if (unlikely(!pc))
		return;

	lock_page_cgroup(pc);
	mem = pc->mem_cgroup;
	if (!mem)
		goto done;

	if (!PageCgroupUsed(pc))
		goto done;

	/*
	 * Preemption is already disabled, we don't need get_cpu()
	 */
	cpu = smp_processor_id();
	stat = &mem->stat;
	cpustat = &stat->cpustat[cpu];

1257
	__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED, val);
1258 1259 1260
done:
	unlock_page_cgroup(pc);
}
1261

1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
#define CHARGE_SIZE	(32 * PAGE_SIZE)
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
	int charge;
	struct work_struct work;
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
static atomic_t memcg_drain_count;

/*
 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
 * from local stock and true is returned. If the stock is 0 or charges from a
 * cgroup which is not current target, returns false. This stock will be
 * refilled.
 */
static bool consume_stock(struct mem_cgroup *mem)
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

	stock = &get_cpu_var(memcg_stock);
	if (mem == stock->cached && stock->charge)
		stock->charge -= PAGE_SIZE;
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

	if (stock->charge) {
		res_counter_uncharge(&old->res, stock->charge);
		if (do_swap_account)
			res_counter_uncharge(&old->memsw, stock->charge);
	}
	stock->cached = NULL;
	stock->charge = 0;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
}

/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
 * This will be consumed by consumt_stock() function, later.
 */
static void refill_stock(struct mem_cgroup *mem, int val)
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

	if (stock->cached != mem) { /* reset if necessary */
		drain_stock(stock);
		stock->cached = mem;
	}
	stock->charge += val;
	put_cpu_var(memcg_stock);
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
static void drain_all_stock_async(void)
{
	int cpu;
	/* This function is for scheduling "drain" in asynchronous way.
	 * The result of "drain" is not directly handled by callers. Then,
	 * if someone is calling drain, we don't have to call drain more.
	 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
	 * there is a race. We just do loose check here.
	 */
	if (atomic_read(&memcg_drain_count))
		return;
	/* Notify other cpus that system-wide "drain" is running */
	atomic_inc(&memcg_drain_count);
	get_online_cpus();
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
		schedule_work_on(cpu, &stock->work);
	}
 	put_online_cpus();
	atomic_dec(&memcg_drain_count);
	/* We don't wait for flush_work */
}

/* This is a synchronous drain interface. */
static void drain_all_stock_sync(void)
{
	/* called when force_empty is called */
	atomic_inc(&memcg_drain_count);
	schedule_on_each_cpu(drain_local_stock);
	atomic_dec(&memcg_drain_count);
}

static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb,
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;

	if (action != CPU_DEAD)
		return NOTIFY_OK;
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

1389 1390 1391
/*
 * Unlike exported interface, "oom" parameter is added. if oom==true,
 * oom-killer can be invoked.
1392
 */
1393
static int __mem_cgroup_try_charge(struct mm_struct *mm,
1394
			gfp_t gfp_mask, struct mem_cgroup **memcg,
1395
			bool oom, struct page *page)
1396
{
1397
	struct mem_cgroup *mem, *mem_over_limit;
1398
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1399
	struct res_counter *fail_res;
1400
	int csize = CHARGE_SIZE;
1401 1402 1403 1404 1405 1406 1407

	if (unlikely(test_thread_flag(TIF_MEMDIE))) {
		/* Don't account this! */
		*memcg = NULL;
		return 0;
	}

1408
	/*
1409 1410
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
1411 1412 1413
	 * thread group leader migrates. It's possible that mm is not
	 * set, if so charge the init_mm (happens for pagecache usage).
	 */
1414 1415 1416
	mem = *memcg;
	if (likely(!mem)) {
		mem = try_get_mem_cgroup_from_mm(mm);
1417
		*memcg = mem;
1418
	} else {
1419
		css_get(&mem->css);
1420
	}
1421 1422 1423
	if (unlikely(!mem))
		return 0;

1424
	VM_BUG_ON(css_is_removed(&mem->css));
1425 1426
	if (mem_cgroup_is_root(mem))
		goto done;
1427

1428
	while (1) {
1429
		int ret = 0;
1430
		unsigned long flags = 0;
1431

1432 1433 1434 1435
		if (consume_stock(mem))
			goto charged;

		ret = res_counter_charge(&mem->res, csize, &fail_res);
1436 1437 1438
		if (likely(!ret)) {
			if (!do_swap_account)
				break;
1439
			ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1440 1441 1442
			if (likely(!ret))
				break;
			/* mem+swap counter fails */
1443
			res_counter_uncharge(&mem->res, csize);
1444
			flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1445 1446 1447 1448 1449 1450 1451
			mem_over_limit = mem_cgroup_from_res_counter(fail_res,
									memsw);
		} else
			/* mem counter fails */
			mem_over_limit = mem_cgroup_from_res_counter(fail_res,
									res);

1452 1453 1454 1455 1456
		/* reduce request size and retry */
		if (csize > PAGE_SIZE) {
			csize = PAGE_SIZE;
			continue;
		}
1457
		if (!(gfp_mask & __GFP_WAIT))
1458
			goto nomem;
1459

1460 1461
		ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
						gfp_mask, flags);
1462 1463
		if (ret)
			continue;
1464 1465

		/*
1466 1467 1468 1469 1470
		 * try_to_free_mem_cgroup_pages() might not give us a full
		 * picture of reclaim. Some pages are reclaimed and might be
		 * moved to swap cache or just unmapped from the cgroup.
		 * Check the limit again to see if the reclaim reduced the
		 * current usage of the cgroup before giving up
1471
		 *
1472
		 */
1473 1474
		if (mem_cgroup_check_under_limit(mem_over_limit))
			continue;
1475 1476

		if (!nr_retries--) {
1477
			if (oom) {
1478
				mutex_lock(&memcg_tasklist);
1479
				mem_cgroup_out_of_memory(mem_over_limit, gfp_mask);
1480
				mutex_unlock(&memcg_tasklist);
1481
				record_last_oom(mem_over_limit);
1482
			}
1483
			goto nomem;
1484
		}
1485
	}
1486 1487 1488
	if (csize > PAGE_SIZE)
		refill_stock(mem, csize - PAGE_SIZE);
charged:
1489
	/*
1490 1491
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
1492
	 */
1493 1494
	if (mem_cgroup_soft_limit_check(mem))
		mem_cgroup_update_tree(mem, page);
1495
done:
1496 1497 1498 1499 1500
	return 0;
nomem:
	css_put(&mem->css);
	return -ENOMEM;
}
1501

1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
/*
 * A helper function to get mem_cgroup from ID. must be called under
 * rcu_read_lock(). The caller must check css_is_removed() or some if
 * it's concern. (dropping refcnt from swap can be called against removed
 * memcg.)
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
	return container_of(css, struct mem_cgroup, css);
}

1521 1522 1523
static struct mem_cgroup *try_get_mem_cgroup_from_swapcache(struct page *page)
{
	struct mem_cgroup *mem;
1524
	struct page_cgroup *pc;
1525
	unsigned short id;
1526 1527
	swp_entry_t ent;

1528 1529
	VM_BUG_ON(!PageLocked(page));

1530 1531 1532
	if (!PageSwapCache(page))
		return NULL;

1533
	pc = lookup_page_cgroup(page);
1534
	lock_page_cgroup(pc);
1535
	if (PageCgroupUsed(pc)) {
1536
		mem = pc->mem_cgroup;
1537 1538 1539
		if (mem && !css_tryget(&mem->css))
			mem = NULL;
	} else {
1540
		ent.val = page_private(page);
1541 1542 1543 1544 1545 1546
		id = lookup_swap_cgroup(ent);
		rcu_read_lock();
		mem = mem_cgroup_lookup(id);
		if (mem && !css_tryget(&mem->css))
			mem = NULL;
		rcu_read_unlock();
1547
	}
1548
	unlock_page_cgroup(pc);
1549 1550 1551
	return mem;
}

1552
/*
1553
 * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
 * USED state. If already USED, uncharge and return.
 */

static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
				     struct page_cgroup *pc,
				     enum charge_type ctype)
{
	/* try_charge() can return NULL to *memcg, taking care of it. */
	if (!mem)
		return;
1564 1565 1566 1567

	lock_page_cgroup(pc);
	if (unlikely(PageCgroupUsed(pc))) {
		unlock_page_cgroup(pc);
1568
		if (!mem_cgroup_is_root(mem)) {
1569
			res_counter_uncharge(&mem->res, PAGE_SIZE);
1570
			if (do_swap_account)
1571
				res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1572
		}
1573
		css_put(&mem->css);
1574
		return;
1575
	}
1576

1577
	pc->mem_cgroup = mem;
1578 1579 1580 1581 1582 1583 1584
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
1585
	smp_wmb();
1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_CACHE:
	case MEM_CGROUP_CHARGE_TYPE_SHMEM:
		SetPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
		ClearPageCgroupCache(pc);
		SetPageCgroupUsed(pc);
		break;
	default:
		break;
	}
1599

K
KAMEZAWA Hiroyuki 已提交
1600
	mem_cgroup_charge_statistics(mem, pc, true);
1601 1602

	unlock_page_cgroup(pc);
1603
}
1604

1605 1606 1607 1608 1609 1610 1611
/**
 * mem_cgroup_move_account - move account of the page
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
1612
 * - page is not on LRU (isolate_page() is useful.)
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
 *
 * returns 0 at success,
 * returns -EBUSY when lock is busy or "pc" is unstable.
 *
 * This function does "uncharge" from old cgroup but doesn't do "charge" to
 * new cgroup. It should be done by a caller.
 */

static int mem_cgroup_move_account(struct page_cgroup *pc,
	struct mem_cgroup *from, struct mem_cgroup *to)
{
	struct mem_cgroup_per_zone *from_mz, *to_mz;
	int nid, zid;
	int ret = -EBUSY;
1627 1628 1629 1630
	struct page *page;
	int cpu;
	struct mem_cgroup_stat *stat;
	struct mem_cgroup_stat_cpu *cpustat;
1631 1632

	VM_BUG_ON(from == to);
K
KAMEZAWA Hiroyuki 已提交
1633
	VM_BUG_ON(PageLRU(pc->page));
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648

	nid = page_cgroup_nid(pc);
	zid = page_cgroup_zid(pc);
	from_mz =  mem_cgroup_zoneinfo(from, nid, zid);
	to_mz =  mem_cgroup_zoneinfo(to, nid, zid);

	if (!trylock_page_cgroup(pc))
		return ret;

	if (!PageCgroupUsed(pc))
		goto out;

	if (pc->mem_cgroup != from)
		goto out;

1649
	if (!mem_cgroup_is_root(from))
1650
		res_counter_uncharge(&from->res, PAGE_SIZE);
K
KAMEZAWA Hiroyuki 已提交
1651
	mem_cgroup_charge_statistics(from, pc, false);
1652 1653

	page = pc->page;
1654
	if (page_mapped(page) && !PageAnon(page)) {
1655 1656 1657 1658
		cpu = smp_processor_id();
		/* Update mapped_file data for mem_cgroup "from" */
		stat = &from->stat;
		cpustat = &stat->cpustat[cpu];
1659
		__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED,
1660 1661 1662 1663 1664
						-1);

		/* Update mapped_file data for mem_cgroup "to" */
		stat = &to->stat;
		cpustat = &stat->cpustat[cpu];
1665
		__mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_FILE_MAPPED,
1666 1667 1668
						1);
	}

1669
	if (do_swap_account && !mem_cgroup_is_root(from))
1670
		res_counter_uncharge(&from->memsw, PAGE_SIZE);
1671 1672 1673
	css_put(&from->css);

	css_get(&to->css);
K
KAMEZAWA Hiroyuki 已提交
1674 1675 1676
	pc->mem_cgroup = to;
	mem_cgroup_charge_statistics(to, pc, true);
	ret = 0;
1677 1678
out:
	unlock_page_cgroup(pc);
1679 1680 1681 1682 1683 1684
	/*
	 * We charges against "to" which may not have any tasks. Then, "to"
	 * can be under rmdir(). But in current implementation, caller of
	 * this function is just force_empty() and it's garanteed that
	 * "to" is never removed. So, we don't check rmdir status here.
	 */
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
	return ret;
}

/*
 * move charges to its parent.
 */

static int mem_cgroup_move_parent(struct page_cgroup *pc,
				  struct mem_cgroup *child,
				  gfp_t gfp_mask)
{
K
KAMEZAWA Hiroyuki 已提交
1696
	struct page *page = pc->page;
1697 1698 1699 1700 1701 1702 1703 1704 1705
	struct cgroup *cg = child->css.cgroup;
	struct cgroup *pcg = cg->parent;
	struct mem_cgroup *parent;
	int ret;

	/* Is ROOT ? */
	if (!pcg)
		return -EINVAL;

K
KAMEZAWA Hiroyuki 已提交
1706

1707 1708
	parent = mem_cgroup_from_cont(pcg);

K
KAMEZAWA Hiroyuki 已提交
1709

1710
	ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, page);
1711
	if (ret || !parent)
1712 1713
		return ret;

1714 1715 1716 1717
	if (!get_page_unless_zero(page)) {
		ret = -EBUSY;
		goto uncharge;
	}
K
KAMEZAWA Hiroyuki 已提交
1718 1719 1720 1721 1722

	ret = isolate_lru_page(page);

	if (ret)
		goto cancel;
1723 1724 1725

	ret = mem_cgroup_move_account(pc, child, parent);

K
KAMEZAWA Hiroyuki 已提交
1726 1727 1728
	putback_lru_page(page);
	if (!ret) {
		put_page(page);
1729 1730
		/* drop extra refcnt by try_charge() */
		css_put(&parent->css);
K
KAMEZAWA Hiroyuki 已提交
1731
		return 0;
1732
	}
1733

K
KAMEZAWA Hiroyuki 已提交
1734
cancel:
1735 1736 1737 1738 1739
	put_page(page);
uncharge:
	/* drop extra refcnt by try_charge() */
	css_put(&parent->css);
	/* uncharge if move fails */
1740
	if (!mem_cgroup_is_root(parent)) {
1741
		res_counter_uncharge(&parent->res, PAGE_SIZE);
1742
		if (do_swap_account)
1743
			res_counter_uncharge(&parent->memsw, PAGE_SIZE);
1744
	}
1745 1746 1747
	return ret;
}

1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask, enum charge_type ctype,
				struct mem_cgroup *memcg)
{
	struct mem_cgroup *mem;
	struct page_cgroup *pc;
	int ret;

	pc = lookup_page_cgroup(page);
	/* can happen at boot */
	if (unlikely(!pc))
		return 0;
	prefetchw(pc);

	mem = memcg;
1769
	ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page);
1770
	if (ret || !mem)
1771 1772 1773
		return ret;

	__mem_cgroup_commit_charge(mem, pc, ctype);
1774 1775 1776
	return 0;
}

1777 1778
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
1779
{
1780
	if (mem_cgroup_disabled())
1781
		return 0;
1782 1783
	if (PageCompound(page))
		return 0;
1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
	/*
	 * If already mapped, we don't have to account.
	 * If page cache, page->mapping has address_space.
	 * But page->mapping may have out-of-use anon_vma pointer,
	 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
	 * is NULL.
  	 */
	if (page_mapped(page) || (page->mapping && !PageAnon(page)))
		return 0;
	if (unlikely(!mm))
		mm = &init_mm;
1795
	return mem_cgroup_charge_common(page, mm, gfp_mask,
1796
				MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
1797 1798
}

D
Daisuke Nishimura 已提交
1799 1800 1801 1802
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype);

1803 1804
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
1805
{
1806 1807 1808
	struct mem_cgroup *mem = NULL;
	int ret;

1809
	if (mem_cgroup_disabled())
1810
		return 0;
1811 1812
	if (PageCompound(page))
		return 0;
1813 1814 1815 1816 1817 1818 1819 1820
	/*
	 * Corner case handling. This is called from add_to_page_cache()
	 * in usual. But some FS (shmem) precharges this page before calling it
	 * and call add_to_page_cache() with GFP_NOWAIT.
	 *
	 * For GFP_NOWAIT case, the page may be pre-charged before calling
	 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
	 * charge twice. (It works but has to pay a bit larger cost.)
1821 1822
	 * And when the page is SwapCache, it should take swap information
	 * into account. This is under lock_page() now.
1823 1824 1825 1826
	 */
	if (!(gfp_mask & __GFP_WAIT)) {
		struct page_cgroup *pc;

1827 1828 1829 1830 1831 1832 1833

		pc = lookup_page_cgroup(page);
		if (!pc)
			return 0;
		lock_page_cgroup(pc);
		if (PageCgroupUsed(pc)) {
			unlock_page_cgroup(pc);
1834 1835
			return 0;
		}
1836
		unlock_page_cgroup(pc);
1837 1838
	}

1839
	if (unlikely(!mm && !mem))
1840
		mm = &init_mm;
1841

1842 1843
	if (page_is_file_cache(page))
		return mem_cgroup_charge_common(page, mm, gfp_mask,
1844
				MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
1845

D
Daisuke Nishimura 已提交
1846 1847 1848 1849 1850 1851 1852 1853 1854
	/* shmem */
	if (PageSwapCache(page)) {
		ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
		if (!ret)
			__mem_cgroup_commit_charge_swapin(page, mem,
					MEM_CGROUP_CHARGE_TYPE_SHMEM);
	} else
		ret = mem_cgroup_charge_common(page, mm, gfp_mask,
					MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
1855 1856

	return ret;
1857 1858
}

1859 1860 1861
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
1862
 * struct page_cgroup is acquired. This refcnt will be consumed by
1863 1864
 * "commit()" or removed by "cancel()"
 */
1865 1866 1867 1868 1869
int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
				 struct page *page,
				 gfp_t mask, struct mem_cgroup **ptr)
{
	struct mem_cgroup *mem;
1870
	int ret;
1871

1872
	if (mem_cgroup_disabled())
1873 1874 1875 1876 1877 1878
		return 0;

	if (!do_swap_account)
		goto charge_cur_mm;
	/*
	 * A racing thread's fault, or swapoff, may have already updated
H
Hugh Dickins 已提交
1879 1880 1881
	 * the pte, and even removed page from swap cache: in those cases
	 * do_swap_page()'s pte_same() test will fail; but there's also a
	 * KSM case which does need to charge the page.
1882 1883
	 */
	if (!PageSwapCache(page))
H
Hugh Dickins 已提交
1884
		goto charge_cur_mm;
1885
	mem = try_get_mem_cgroup_from_swapcache(page);
1886 1887
	if (!mem)
		goto charge_cur_mm;
1888
	*ptr = mem;
1889
	ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, page);
1890 1891 1892
	/* drop extra refcnt from tryget */
	css_put(&mem->css);
	return ret;
1893 1894 1895
charge_cur_mm:
	if (unlikely(!mm))
		mm = &init_mm;
1896
	return __mem_cgroup_try_charge(mm, mask, ptr, true, page);
1897 1898
}

D
Daisuke Nishimura 已提交
1899 1900 1901
static void
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
					enum charge_type ctype)
1902 1903 1904
{
	struct page_cgroup *pc;

1905
	if (mem_cgroup_disabled())
1906 1907 1908
		return;
	if (!ptr)
		return;
1909
	cgroup_exclude_rmdir(&ptr->css);
1910
	pc = lookup_page_cgroup(page);
1911
	mem_cgroup_lru_del_before_commit_swapcache(page);
D
Daisuke Nishimura 已提交
1912
	__mem_cgroup_commit_charge(ptr, pc, ctype);
1913
	mem_cgroup_lru_add_after_commit_swapcache(page);
1914 1915 1916
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
1917 1918 1919
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
1920
	 */
1921
	if (do_swap_account && PageSwapCache(page)) {
1922
		swp_entry_t ent = {.val = page_private(page)};
1923
		unsigned short id;
1924
		struct mem_cgroup *memcg;
1925 1926 1927 1928

		id = swap_cgroup_record(ent, 0);
		rcu_read_lock();
		memcg = mem_cgroup_lookup(id);
1929
		if (memcg) {
1930 1931 1932 1933
			/*
			 * This recorded memcg can be obsolete one. So, avoid
			 * calling css_tryget
			 */
1934
			if (!mem_cgroup_is_root(memcg))
1935
				res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1936
			mem_cgroup_swap_statistics(memcg, false);
1937 1938
			mem_cgroup_put(memcg);
		}
1939
		rcu_read_unlock();
1940
	}
1941 1942 1943 1944 1945 1946
	/*
	 * At swapin, we may charge account against cgroup which has no tasks.
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
	cgroup_release_and_wakeup_rmdir(&ptr->css);
1947 1948
}

D
Daisuke Nishimura 已提交
1949 1950 1951 1952 1953 1954
void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
{
	__mem_cgroup_commit_charge_swapin(page, ptr,
					MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

1955 1956
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
{
1957
	if (mem_cgroup_disabled())
1958 1959 1960
		return;
	if (!mem)
		return;
1961
	if (!mem_cgroup_is_root(mem)) {
1962
		res_counter_uncharge(&mem->res, PAGE_SIZE);
1963
		if (do_swap_account)
1964
			res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1965
	}
1966 1967 1968
	css_put(&mem->css);
}

1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
static void
__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
	 * In those cases, all pages freed continously can be expected to be in
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */
	if (!current->memcg_batch.do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
		batch->memcg = mem;
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
	if (batch->memcg != mem)
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
	batch->bytes += PAGE_SIZE;
	if (uncharge_memsw)
		batch->memsw_bytes += PAGE_SIZE;
	return;
direct_uncharge:
	res_counter_uncharge(&mem->res, PAGE_SIZE);
	if (uncharge_memsw)
		res_counter_uncharge(&mem->memsw, PAGE_SIZE);
	return;
}
2013

2014
/*
2015
 * uncharge if !page_mapped(page)
2016
 */
2017
static struct mem_cgroup *
2018
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2019
{
H
Hugh Dickins 已提交
2020
	struct page_cgroup *pc;
2021
	struct mem_cgroup *mem = NULL;
2022
	struct mem_cgroup_per_zone *mz;
2023

2024
	if (mem_cgroup_disabled())
2025
		return NULL;
2026

K
KAMEZAWA Hiroyuki 已提交
2027
	if (PageSwapCache(page))
2028
		return NULL;
K
KAMEZAWA Hiroyuki 已提交
2029

2030
	/*
2031
	 * Check if our page_cgroup is valid
2032
	 */
2033 2034
	pc = lookup_page_cgroup(page);
	if (unlikely(!pc || !PageCgroupUsed(pc)))
2035
		return NULL;
2036

2037
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
2038

2039 2040
	mem = pc->mem_cgroup;

K
KAMEZAWA Hiroyuki 已提交
2041 2042 2043 2044 2045
	if (!PageCgroupUsed(pc))
		goto unlock_out;

	switch (ctype) {
	case MEM_CGROUP_CHARGE_TYPE_MAPPED:
K
KAMEZAWA Hiroyuki 已提交
2046
	case MEM_CGROUP_CHARGE_TYPE_DROP:
K
KAMEZAWA Hiroyuki 已提交
2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058
		if (page_mapped(page))
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
2059
	}
K
KAMEZAWA Hiroyuki 已提交
2060

2061 2062
	if (!mem_cgroup_is_root(mem))
		__do_uncharge(mem, ctype);
2063 2064
	if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		mem_cgroup_swap_statistics(mem, true);
K
KAMEZAWA Hiroyuki 已提交
2065
	mem_cgroup_charge_statistics(mem, pc, false);
K
KAMEZAWA Hiroyuki 已提交
2066

2067
	ClearPageCgroupUsed(pc);
2068 2069 2070 2071 2072 2073
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
2074

2075
	mz = page_cgroup_zoneinfo(pc);
2076
	unlock_page_cgroup(pc);
H
Hugh Dickins 已提交
2077

2078
	if (mem_cgroup_soft_limit_check(mem))
2079
		mem_cgroup_update_tree(mem, page);
K
KAMEZAWA Hiroyuki 已提交
2080 2081 2082
	/* at swapout, this memcg will be accessed to record to swap */
	if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		css_put(&mem->css);
2083

2084
	return mem;
K
KAMEZAWA Hiroyuki 已提交
2085 2086 2087

unlock_out:
	unlock_page_cgroup(pc);
2088
	return NULL;
2089 2090
}

2091 2092
void mem_cgroup_uncharge_page(struct page *page)
{
2093 2094 2095 2096 2097
	/* early check. */
	if (page_mapped(page))
		return;
	if (page->mapping && !PageAnon(page))
		return;
2098 2099 2100 2101 2102 2103
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
2104
	VM_BUG_ON(page->mapping);
2105 2106 2107
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
}

2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
		current->memcg_batch.bytes = 0;
		current->memcg_batch.memsw_bytes = 0;
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
	if (batch->bytes)
		res_counter_uncharge(&batch->memcg->res, batch->bytes);
	if (batch->memsw_bytes)
		res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

2152
#ifdef CONFIG_SWAP
2153
/*
2154
 * called after __delete_from_swap_cache() and drop "page" account.
2155 2156
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
2157 2158
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
2159 2160
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
2161 2162 2163 2164 2165 2166
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

	memcg = __mem_cgroup_uncharge_common(page, ctype);
2167 2168

	/* record memcg information */
K
KAMEZAWA Hiroyuki 已提交
2169
	if (do_swap_account && swapout && memcg) {
2170
		swap_cgroup_record(ent, css_id(&memcg->css));
2171 2172
		mem_cgroup_get(memcg);
	}
K
KAMEZAWA Hiroyuki 已提交
2173
	if (swapout && memcg)
K
KAMEZAWA Hiroyuki 已提交
2174
		css_put(&memcg->css);
2175
}
2176
#endif
2177 2178 2179 2180 2181 2182 2183

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
2184
{
2185
	struct mem_cgroup *memcg;
2186
	unsigned short id;
2187 2188 2189 2190

	if (!do_swap_account)
		return;

2191 2192 2193
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
2194
	if (memcg) {
2195 2196 2197 2198
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
2199
		if (!mem_cgroup_is_root(memcg))
2200
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2201
		mem_cgroup_swap_statistics(memcg, false);
2202 2203
		mem_cgroup_put(memcg);
	}
2204
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
2205
}
2206
#endif
K
KAMEZAWA Hiroyuki 已提交
2207

2208
/*
2209 2210
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
2211
 */
2212
int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
2213 2214
{
	struct page_cgroup *pc;
2215 2216
	struct mem_cgroup *mem = NULL;
	int ret = 0;
2217

2218
	if (mem_cgroup_disabled())
2219 2220
		return 0;

2221 2222 2223
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
2224 2225 2226
		mem = pc->mem_cgroup;
		css_get(&mem->css);
	}
2227
	unlock_page_cgroup(pc);
2228

2229
	if (mem) {
2230 2231
		ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
						page);
2232 2233
		css_put(&mem->css);
	}
2234
	*ptr = mem;
2235
	return ret;
2236
}
2237

2238
/* remove redundant charge if migration failed*/
2239 2240
void mem_cgroup_end_migration(struct mem_cgroup *mem,
		struct page *oldpage, struct page *newpage)
2241
{
2242 2243 2244 2245 2246 2247
	struct page *target, *unused;
	struct page_cgroup *pc;
	enum charge_type ctype;

	if (!mem)
		return;
2248
	cgroup_exclude_rmdir(&mem->css);
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
	/* at migration success, oldpage->mapping is NULL. */
	if (oldpage->mapping) {
		target = oldpage;
		unused = NULL;
	} else {
		target = newpage;
		unused = oldpage;
	}

	if (PageAnon(target))
		ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
	else if (page_is_file_cache(target))
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
	else
		ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;

	/* unused page is not on radix-tree now. */
K
KAMEZAWA Hiroyuki 已提交
2266
	if (unused)
2267 2268 2269
		__mem_cgroup_uncharge_common(unused, ctype);

	pc = lookup_page_cgroup(target);
2270
	/*
2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
	 * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
	 * So, double-counting is effectively avoided.
	 */
	__mem_cgroup_commit_charge(mem, pc, ctype);

	/*
	 * Both of oldpage and newpage are still under lock_page().
	 * Then, we don't have to care about race in radix-tree.
	 * But we have to be careful that this page is unmapped or not.
	 *
	 * There is a case for !page_mapped(). At the start of
	 * migration, oldpage was mapped. But now, it's zapped.
	 * But we know *target* page is not freed/reused under us.
	 * mem_cgroup_uncharge_page() does all necessary checks.
2285
	 */
2286 2287
	if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
		mem_cgroup_uncharge_page(target);
2288 2289 2290 2291 2292 2293
	/*
	 * At migration, we may charge account against cgroup which has no tasks
	 * So, rmdir()->pre_destroy() can be called while we do this charge.
	 * In that case, we need to call pre_destroy() again. check it here.
	 */
	cgroup_release_and_wakeup_rmdir(&mem->css);
2294
}
2295

2296
/*
2297 2298 2299 2300 2301 2302
 * A call to try to shrink memory usage on charge failure at shmem's swapin.
 * Calling hierarchical_reclaim is not enough because we should update
 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
 * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
 * not from the memcg which this page would be charged to.
 * try_charge_swapin does all of these works properly.
2303
 */
2304
int mem_cgroup_shmem_charge_fallback(struct page *page,
2305 2306
			    struct mm_struct *mm,
			    gfp_t gfp_mask)
2307
{
2308
	struct mem_cgroup *mem = NULL;
2309
	int ret;
2310

2311
	if (mem_cgroup_disabled())
2312
		return 0;
2313

2314 2315 2316
	ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
	if (!ret)
		mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
2317

2318
	return ret;
2319 2320
}

2321 2322
static DEFINE_MUTEX(set_limit_mutex);

2323
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2324
				unsigned long long val)
2325
{
2326
	int retry_count;
2327
	int progress;
2328
	u64 memswlimit;
2329
	int ret = 0;
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2341

2342
	while (retry_count) {
2343 2344 2345 2346
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
		 * We have to guarantee mem->res.limit < mem->memsw.limit.
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
2357 2358
			break;
		}
2359
		ret = res_counter_set_limit(&memcg->res, val);
2360 2361 2362 2363 2364 2365
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
2366 2367 2368 2369 2370
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

2371 2372 2373
		progress = mem_cgroup_hierarchical_reclaim(memcg, NULL,
						GFP_KERNEL,
						MEM_CGROUP_RECLAIM_SHRINK);
2374 2375 2376 2377 2378 2379
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
2380
	}
2381

2382 2383 2384
	return ret;
}

L
Li Zefan 已提交
2385 2386
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
2387
{
2388
	int retry_count;
2389
	u64 memlimit, oldusage, curusage;
2390 2391
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
2392

2393 2394 2395
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
		 * We have to guarantee mem->res.limit < mem->memsw.limit.
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
		ret = res_counter_set_limit(&memcg->memsw, val);
2414 2415 2416 2417 2418 2419
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
2420 2421 2422 2423 2424
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

2425
		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
2426 2427
						MEM_CGROUP_RECLAIM_NOSWAP |
						MEM_CGROUP_RECLAIM_SHRINK);
2428
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
2429
		/* Usage is reduced ? */
2430
		if (curusage >= oldusage)
2431
			retry_count--;
2432 2433
		else
			oldusage = curusage;
2434 2435 2436 2437
	}
	return ret;
}

2438 2439 2440 2441 2442 2443 2444 2445 2446
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
						gfp_t gfp_mask, int nid,
						int zid)
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
2447
	unsigned long long excess;
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499

	if (order > 0)
		return 0;

	mctz = soft_limit_tree_node_zone(nid, zid);
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
						gfp_mask,
						MEM_CGROUP_RECLAIM_SOFT);
		nr_reclaimed += reclaimed;
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
				if (next_mz == mz) {
					css_put(&next_mz->mem->css);
					next_mz = NULL;
				} else /* next_mz == NULL or other memcg */
					break;
			} while (1);
		}
		__mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
2500
		excess = res_counter_soft_limit_excess(&mz->mem->res);
2501 2502 2503 2504 2505 2506 2507 2508
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
2509 2510
		/* If excess == 0, no tree ops */
		__mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
		spin_unlock(&mctz->lock);
		css_put(&mz->mem->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->mem->css);
	return nr_reclaimed;
}

2529 2530 2531 2532
/*
 * This routine traverse page_cgroup in given list and drop them all.
 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
 */
2533
static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
K
KAMEZAWA Hiroyuki 已提交
2534
				int node, int zid, enum lru_list lru)
2535
{
K
KAMEZAWA Hiroyuki 已提交
2536 2537
	struct zone *zone;
	struct mem_cgroup_per_zone *mz;
2538
	struct page_cgroup *pc, *busy;
K
KAMEZAWA Hiroyuki 已提交
2539
	unsigned long flags, loop;
2540
	struct list_head *list;
2541
	int ret = 0;
2542

K
KAMEZAWA Hiroyuki 已提交
2543 2544
	zone = &NODE_DATA(node)->node_zones[zid];
	mz = mem_cgroup_zoneinfo(mem, node, zid);
2545
	list = &mz->lists[lru];
2546

2547 2548 2549 2550 2551 2552
	loop = MEM_CGROUP_ZSTAT(mz, lru);
	/* give some margin against EBUSY etc...*/
	loop += 256;
	busy = NULL;
	while (loop--) {
		ret = 0;
K
KAMEZAWA Hiroyuki 已提交
2553
		spin_lock_irqsave(&zone->lru_lock, flags);
2554
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
2555
			spin_unlock_irqrestore(&zone->lru_lock, flags);
2556
			break;
2557 2558 2559 2560 2561
		}
		pc = list_entry(list->prev, struct page_cgroup, lru);
		if (busy == pc) {
			list_move(&pc->lru, list);
			busy = 0;
K
KAMEZAWA Hiroyuki 已提交
2562
			spin_unlock_irqrestore(&zone->lru_lock, flags);
2563 2564
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
2565
		spin_unlock_irqrestore(&zone->lru_lock, flags);
2566

K
KAMEZAWA Hiroyuki 已提交
2567
		ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
2568
		if (ret == -ENOMEM)
2569
			break;
2570 2571 2572 2573 2574 2575 2576

		if (ret == -EBUSY || ret == -EINVAL) {
			/* found lock contention or "pc" is obsolete. */
			busy = pc;
			cond_resched();
		} else
			busy = NULL;
2577
	}
K
KAMEZAWA Hiroyuki 已提交
2578

2579 2580 2581
	if (!ret && !list_empty(list))
		return -EBUSY;
	return ret;
2582 2583 2584 2585 2586 2587
}

/*
 * make mem_cgroup's charge to be 0 if there is no task.
 * This enables deleting this mem_cgroup.
 */
2588
static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
2589
{
2590 2591 2592
	int ret;
	int node, zid, shrink;
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2593
	struct cgroup *cgrp = mem->css.cgroup;
2594

2595
	css_get(&mem->css);
2596 2597

	shrink = 0;
2598 2599 2600
	/* should free all ? */
	if (free_all)
		goto try_to_free;
2601
move_account:
2602
	while (mem->res.usage > 0) {
2603
		ret = -EBUSY;
2604 2605 2606 2607
		if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
			goto out;
		ret = -EINTR;
		if (signal_pending(current))
2608
			goto out;
2609 2610
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
2611
		drain_all_stock_sync();
2612
		ret = 0;
2613
		for_each_node_state(node, N_HIGH_MEMORY) {
2614
			for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
2615
				enum lru_list l;
2616 2617
				for_each_lru(l) {
					ret = mem_cgroup_force_empty_list(mem,
K
KAMEZAWA Hiroyuki 已提交
2618
							node, zid, l);
2619 2620 2621
					if (ret)
						break;
				}
2622
			}
2623 2624 2625 2626 2627 2628
			if (ret)
				break;
		}
		/* it seems parent cgroup doesn't have enough mem */
		if (ret == -ENOMEM)
			goto try_to_free;
2629
		cond_resched();
2630 2631 2632 2633 2634
	}
	ret = 0;
out:
	css_put(&mem->css);
	return ret;
2635 2636

try_to_free:
2637 2638
	/* returns EBUSY if there is a task or if we come here twice. */
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
2639 2640 2641
		ret = -EBUSY;
		goto out;
	}
2642 2643
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
2644 2645 2646 2647
	/* try to free all pages in this cgroup */
	shrink = 1;
	while (nr_retries && mem->res.usage > 0) {
		int progress;
2648 2649 2650 2651 2652

		if (signal_pending(current)) {
			ret = -EINTR;
			goto out;
		}
K
KOSAKI Motohiro 已提交
2653 2654
		progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
						false, get_swappiness(mem));
2655
		if (!progress) {
2656
			nr_retries--;
2657
			/* maybe some writeback is necessary */
2658
			congestion_wait(BLK_RW_ASYNC, HZ/10);
2659
		}
2660 2661

	}
K
KAMEZAWA Hiroyuki 已提交
2662
	lru_add_drain();
2663 2664 2665 2666 2667
	/* try move_account...there may be some *locked* pages. */
	if (mem->res.usage)
		goto move_account;
	ret = 0;
	goto out;
2668 2669
}

2670 2671 2672 2673 2674 2675
int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
{
	return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
}


2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693
static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
{
	return mem_cgroup_from_cont(cont)->use_hierarchy;
}

static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
					u64 val)
{
	int retval = 0;
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
	struct cgroup *parent = cont->parent;
	struct mem_cgroup *parent_mem = NULL;

	if (parent)
		parent_mem = mem_cgroup_from_cont(parent);

	cgroup_lock();
	/*
2694
	 * If parent's use_hierarchy is set, we can't make any modifications
2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
	if ((!parent_mem || !parent_mem->use_hierarchy) &&
				(val == 1 || val == 0)) {
		if (list_empty(&cont->children))
			mem->use_hierarchy = val;
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
	cgroup_unlock();

	return retval;
}

2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
struct mem_cgroup_idx_data {
	s64 val;
	enum mem_cgroup_stat_index idx;
};

static int
mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
{
	struct mem_cgroup_idx_data *d = data;
	d->val += mem_cgroup_read_stat(&mem->stat, d->idx);
	return 0;
}

static void
mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
				enum mem_cgroup_stat_index idx, s64 *val)
{
	struct mem_cgroup_idx_data d;
	d.idx = idx;
	d.val = 0;
	mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
	*val = d.val;
}

2738
static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
B
Balbir Singh 已提交
2739
{
2740
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
2741
	u64 idx_val, val;
2742 2743 2744 2745 2746 2747
	int type, name;

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (type) {
	case _MEM:
2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
		if (name == RES_USAGE && mem_cgroup_is_root(mem)) {
			mem_cgroup_get_recursive_idx_stat(mem,
				MEM_CGROUP_STAT_CACHE, &idx_val);
			val = idx_val;
			mem_cgroup_get_recursive_idx_stat(mem,
				MEM_CGROUP_STAT_RSS, &idx_val);
			val += idx_val;
			val <<= PAGE_SHIFT;
		} else
			val = res_counter_read_u64(&mem->res, name);
2758 2759
		break;
	case _MEMSWAP:
2760 2761 2762 2763 2764 2765 2766 2767 2768
		if (name == RES_USAGE && mem_cgroup_is_root(mem)) {
			mem_cgroup_get_recursive_idx_stat(mem,
				MEM_CGROUP_STAT_CACHE, &idx_val);
			val = idx_val;
			mem_cgroup_get_recursive_idx_stat(mem,
				MEM_CGROUP_STAT_RSS, &idx_val);
			val += idx_val;
			mem_cgroup_get_recursive_idx_stat(mem,
				MEM_CGROUP_STAT_SWAPOUT, &idx_val);
2769
			val += idx_val;
2770 2771 2772
			val <<= PAGE_SHIFT;
		} else
			val = res_counter_read_u64(&mem->memsw, name);
2773 2774 2775 2776 2777 2778
		break;
	default:
		BUG();
		break;
	}
	return val;
B
Balbir Singh 已提交
2779
}
2780 2781 2782 2783
/*
 * The user of this function is...
 * RES_LIMIT.
 */
2784 2785
static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
			    const char *buffer)
B
Balbir Singh 已提交
2786
{
2787
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
2788
	int type, name;
2789 2790 2791
	unsigned long long val;
	int ret;

2792 2793 2794
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
	switch (name) {
2795
	case RES_LIMIT:
2796 2797 2798 2799
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
2800 2801
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
2802 2803 2804
		if (ret)
			break;
		if (type == _MEM)
2805
			ret = mem_cgroup_resize_limit(memcg, val);
2806 2807
		else
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
2808
		break;
2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
2823 2824 2825 2826 2827
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
2828 2829
}

2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	struct cgroup *cgroup;
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	cgroup = memcg->css.cgroup;
	if (!memcg->use_hierarchy)
		goto out;

	while (cgroup->parent) {
		cgroup = cgroup->parent;
		memcg = mem_cgroup_from_cont(cgroup);
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
	return;
}

2858
static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
2859 2860
{
	struct mem_cgroup *mem;
2861
	int type, name;
2862 2863

	mem = mem_cgroup_from_cont(cont);
2864 2865 2866
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
	switch (name) {
2867
	case RES_MAX_USAGE:
2868 2869 2870 2871
		if (type == _MEM)
			res_counter_reset_max(&mem->res);
		else
			res_counter_reset_max(&mem->memsw);
2872 2873
		break;
	case RES_FAILCNT:
2874 2875 2876 2877
		if (type == _MEM)
			res_counter_reset_failcnt(&mem->res);
		else
			res_counter_reset_failcnt(&mem->memsw);
2878 2879
		break;
	}
2880

2881
	return 0;
2882 2883
}

K
KAMEZAWA Hiroyuki 已提交
2884 2885 2886 2887 2888

/* For read statistics */
enum {
	MCS_CACHE,
	MCS_RSS,
2889
	MCS_FILE_MAPPED,
K
KAMEZAWA Hiroyuki 已提交
2890 2891
	MCS_PGPGIN,
	MCS_PGPGOUT,
2892
	MCS_SWAP,
K
KAMEZAWA Hiroyuki 已提交
2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
	MCS_INACTIVE_ANON,
	MCS_ACTIVE_ANON,
	MCS_INACTIVE_FILE,
	MCS_ACTIVE_FILE,
	MCS_UNEVICTABLE,
	NR_MCS_STAT,
};

struct mcs_total_stat {
	s64 stat[NR_MCS_STAT];
2903 2904
};

K
KAMEZAWA Hiroyuki 已提交
2905 2906 2907 2908 2909 2910
struct {
	char *local_name;
	char *total_name;
} memcg_stat_strings[NR_MCS_STAT] = {
	{"cache", "total_cache"},
	{"rss", "total_rss"},
2911
	{"mapped_file", "total_mapped_file"},
K
KAMEZAWA Hiroyuki 已提交
2912 2913
	{"pgpgin", "total_pgpgin"},
	{"pgpgout", "total_pgpgout"},
2914
	{"swap", "total_swap"},
K
KAMEZAWA Hiroyuki 已提交
2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
	{"inactive_anon", "total_inactive_anon"},
	{"active_anon", "total_active_anon"},
	{"inactive_file", "total_inactive_file"},
	{"active_file", "total_active_file"},
	{"unevictable", "total_unevictable"}
};


static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
{
	struct mcs_total_stat *s = data;
	s64 val;

	/* per cpu stat */
	val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_CACHE);
	s->stat[MCS_CACHE] += val * PAGE_SIZE;
	val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
	s->stat[MCS_RSS] += val * PAGE_SIZE;
2933 2934
	val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_FILE_MAPPED);
	s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
K
KAMEZAWA Hiroyuki 已提交
2935 2936 2937 2938
	val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGIN_COUNT);
	s->stat[MCS_PGPGIN] += val;
	val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGOUT_COUNT);
	s->stat[MCS_PGPGOUT] += val;
2939 2940 2941 2942
	if (do_swap_account) {
		val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_SWAPOUT);
		s->stat[MCS_SWAP] += val * PAGE_SIZE;
	}
K
KAMEZAWA Hiroyuki 已提交
2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963

	/* per zone stat */
	val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
	s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
	s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
	s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
	s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
	val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
	s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
	return 0;
}

static void
mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
{
	mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
}

2964 2965
static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
				 struct cgroup_map_cb *cb)
2966 2967
{
	struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
K
KAMEZAWA Hiroyuki 已提交
2968
	struct mcs_total_stat mystat;
2969 2970
	int i;

K
KAMEZAWA Hiroyuki 已提交
2971 2972
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_local_stat(mem_cont, &mystat);
2973

2974 2975 2976
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
2977
		cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
2978
	}
L
Lee Schermerhorn 已提交
2979

K
KAMEZAWA Hiroyuki 已提交
2980
	/* Hierarchical information */
2981 2982 2983 2984 2985 2986 2987
	{
		unsigned long long limit, memsw_limit;
		memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
		cb->fill(cb, "hierarchical_memory_limit", limit);
		if (do_swap_account)
			cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
	}
K
KOSAKI Motohiro 已提交
2988

K
KAMEZAWA Hiroyuki 已提交
2989 2990
	memset(&mystat, 0, sizeof(mystat));
	mem_cgroup_get_total_stat(mem_cont, &mystat);
2991 2992 2993
	for (i = 0; i < NR_MCS_STAT; i++) {
		if (i == MCS_SWAP && !do_swap_account)
			continue;
K
KAMEZAWA Hiroyuki 已提交
2994
		cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
2995
	}
K
KAMEZAWA Hiroyuki 已提交
2996

K
KOSAKI Motohiro 已提交
2997
#ifdef CONFIG_DEBUG_VM
2998
	cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
K
KOSAKI Motohiro 已提交
2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025

	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
				mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);

				recent_rotated[0] +=
					mz->reclaim_stat.recent_rotated[0];
				recent_rotated[1] +=
					mz->reclaim_stat.recent_rotated[1];
				recent_scanned[0] +=
					mz->reclaim_stat.recent_scanned[0];
				recent_scanned[1] +=
					mz->reclaim_stat.recent_scanned[1];
			}
		cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
		cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
		cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
		cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
	}
#endif

3026 3027 3028
	return 0;
}

K
KOSAKI Motohiro 已提交
3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040
static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);

	return get_swappiness(memcg);
}

static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
				       u64 val)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;
3041

K
KOSAKI Motohiro 已提交
3042 3043 3044 3045 3046 3047 3048
	if (val > 100)
		return -EINVAL;

	if (cgrp->parent == NULL)
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);
3049 3050 3051

	cgroup_lock();

K
KOSAKI Motohiro 已提交
3052 3053
	/* If under hierarchy, only empty-root can set this value */
	if ((parent->use_hierarchy) ||
3054 3055
	    (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
		cgroup_unlock();
K
KOSAKI Motohiro 已提交
3056
		return -EINVAL;
3057
	}
K
KOSAKI Motohiro 已提交
3058 3059 3060 3061 3062

	spin_lock(&memcg->reclaim_param_lock);
	memcg->swappiness = val;
	spin_unlock(&memcg->reclaim_param_lock);

3063 3064
	cgroup_unlock();

K
KOSAKI Motohiro 已提交
3065 3066 3067
	return 0;
}

3068

B
Balbir Singh 已提交
3069 3070
static struct cftype mem_cgroup_files[] = {
	{
3071
		.name = "usage_in_bytes",
3072
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3073
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
3074
	},
3075 3076
	{
		.name = "max_usage_in_bytes",
3077
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3078
		.trigger = mem_cgroup_reset,
3079 3080
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
3081
	{
3082
		.name = "limit_in_bytes",
3083
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3084
		.write_string = mem_cgroup_write,
3085
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
3086
	},
3087 3088 3089 3090 3091 3092
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
B
Balbir Singh 已提交
3093 3094
	{
		.name = "failcnt",
3095
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3096
		.trigger = mem_cgroup_reset,
3097
		.read_u64 = mem_cgroup_read,
B
Balbir Singh 已提交
3098
	},
3099 3100
	{
		.name = "stat",
3101
		.read_map = mem_control_stat_show,
3102
	},
3103 3104 3105 3106
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
3107 3108 3109 3110 3111
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
3112 3113 3114 3115 3116
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
B
Balbir Singh 已提交
3117 3118
};

3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read_u64 = mem_cgroup_read,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read,
	},
};

static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	if (!do_swap_account)
		return 0;
	return cgroup_add_files(cont, ss, memsw_cgroup_files,
				ARRAY_SIZE(memsw_cgroup_files));
};
#else
static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
{
	return 0;
}
#endif

3160 3161 3162
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
{
	struct mem_cgroup_per_node *pn;
3163
	struct mem_cgroup_per_zone *mz;
3164
	enum lru_list l;
3165
	int zone, tmp = node;
3166 3167 3168 3169 3170 3171 3172 3173
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
3174 3175 3176
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
	pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
3177 3178
	if (!pn)
		return 1;
3179

3180 3181
	mem->info.nodeinfo[node] = pn;
	memset(pn, 0, sizeof(*pn));
3182 3183 3184

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
3185 3186
		for_each_lru(l)
			INIT_LIST_HEAD(&mz->lists[l]);
3187
		mz->usage_in_excess = 0;
3188 3189
		mz->on_tree = false;
		mz->mem = mem;
3190
	}
3191 3192 3193
	return 0;
}

3194 3195 3196 3197 3198
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
{
	kfree(mem->info.nodeinfo[node]);
}

3199 3200 3201 3202 3203 3204
static int mem_cgroup_size(void)
{
	int cpustat_size = nr_cpu_ids * sizeof(struct mem_cgroup_stat_cpu);
	return sizeof(struct mem_cgroup) + cpustat_size;
}

3205 3206 3207
static struct mem_cgroup *mem_cgroup_alloc(void)
{
	struct mem_cgroup *mem;
3208
	int size = mem_cgroup_size();
3209

3210 3211
	if (size < PAGE_SIZE)
		mem = kmalloc(size, GFP_KERNEL);
3212
	else
3213
		mem = vmalloc(size);
3214 3215

	if (mem)
3216
		memset(mem, 0, size);
3217 3218 3219
	return mem;
}

3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
/*
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
 */

3231
static void __mem_cgroup_free(struct mem_cgroup *mem)
3232
{
K
KAMEZAWA Hiroyuki 已提交
3233 3234
	int node;

3235
	mem_cgroup_remove_from_trees(mem);
K
KAMEZAWA Hiroyuki 已提交
3236 3237
	free_css_id(&mem_cgroup_subsys, &mem->css);

K
KAMEZAWA Hiroyuki 已提交
3238 3239 3240
	for_each_node_state(node, N_POSSIBLE)
		free_mem_cgroup_per_zone_info(mem, node);

3241
	if (mem_cgroup_size() < PAGE_SIZE)
3242 3243 3244 3245 3246
		kfree(mem);
	else
		vfree(mem);
}

3247 3248 3249 3250 3251 3252 3253
static void mem_cgroup_get(struct mem_cgroup *mem)
{
	atomic_inc(&mem->refcnt);
}

static void mem_cgroup_put(struct mem_cgroup *mem)
{
3254 3255
	if (atomic_dec_and_test(&mem->refcnt)) {
		struct mem_cgroup *parent = parent_mem_cgroup(mem);
3256
		__mem_cgroup_free(mem);
3257 3258 3259
		if (parent)
			mem_cgroup_put(parent);
	}
3260 3261
}

3262 3263 3264 3265 3266 3267 3268 3269 3270
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
{
	if (!mem->res.parent)
		return NULL;
	return mem_cgroup_from_res_counter(mem->res.parent, res);
}
3271

3272 3273 3274
#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
static void __init enable_swap_cgroup(void)
{
3275
	if (!mem_cgroup_disabled() && really_do_swap_account)
3276 3277 3278 3279 3280 3281 3282 3283
		do_swap_account = 1;
}
#else
static void __init enable_swap_cgroup(void)
{
}
#endif

3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308
static int mem_cgroup_soft_limit_tree_init(void)
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

	for_each_node_state(node, N_POSSIBLE) {
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
		if (!rtpn)
			return 1;

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
	return 0;
}

L
Li Zefan 已提交
3309
static struct cgroup_subsys_state * __ref
B
Balbir Singh 已提交
3310 3311
mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
{
3312
	struct mem_cgroup *mem, *parent;
K
KAMEZAWA Hiroyuki 已提交
3313
	long error = -ENOMEM;
3314
	int node;
B
Balbir Singh 已提交
3315

3316 3317
	mem = mem_cgroup_alloc();
	if (!mem)
K
KAMEZAWA Hiroyuki 已提交
3318
		return ERR_PTR(error);
3319

3320 3321 3322
	for_each_node_state(node, N_POSSIBLE)
		if (alloc_mem_cgroup_per_zone_info(mem, node))
			goto free_out;
3323

3324
	/* root ? */
3325
	if (cont->parent == NULL) {
3326
		int cpu;
3327
		enable_swap_cgroup();
3328
		parent = NULL;
3329
		root_mem_cgroup = mem;
3330 3331
		if (mem_cgroup_soft_limit_tree_init())
			goto free_out;
3332 3333 3334 3335 3336 3337
		for_each_possible_cpu(cpu) {
			struct memcg_stock_pcp *stock =
						&per_cpu(memcg_stock, cpu);
			INIT_WORK(&stock->work, drain_local_stock);
		}
		hotcpu_notifier(memcg_stock_cpu_callback, 0);
3338

3339
	} else {
3340
		parent = mem_cgroup_from_cont(cont->parent);
3341 3342
		mem->use_hierarchy = parent->use_hierarchy;
	}
3343

3344 3345 3346
	if (parent && parent->use_hierarchy) {
		res_counter_init(&mem->res, &parent->res);
		res_counter_init(&mem->memsw, &parent->memsw);
3347 3348 3349 3350 3351 3352 3353
		/*
		 * We increment refcnt of the parent to ensure that we can
		 * safely access it on res_counter_charge/uncharge.
		 * This refcnt will be decremented when freeing this
		 * mem_cgroup(see mem_cgroup_put).
		 */
		mem_cgroup_get(parent);
3354 3355 3356 3357
	} else {
		res_counter_init(&mem->res, NULL);
		res_counter_init(&mem->memsw, NULL);
	}
K
KAMEZAWA Hiroyuki 已提交
3358
	mem->last_scanned_child = 0;
K
KOSAKI Motohiro 已提交
3359
	spin_lock_init(&mem->reclaim_param_lock);
3360

K
KOSAKI Motohiro 已提交
3361 3362
	if (parent)
		mem->swappiness = get_swappiness(parent);
3363
	atomic_set(&mem->refcnt, 1);
B
Balbir Singh 已提交
3364
	return &mem->css;
3365
free_out:
3366
	__mem_cgroup_free(mem);
3367
	root_mem_cgroup = NULL;
K
KAMEZAWA Hiroyuki 已提交
3368
	return ERR_PTR(error);
B
Balbir Singh 已提交
3369 3370
}

3371
static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
3372 3373 3374
					struct cgroup *cont)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3375 3376

	return mem_cgroup_force_empty(mem, false);
3377 3378
}

B
Balbir Singh 已提交
3379 3380 3381
static void mem_cgroup_destroy(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
3382 3383 3384
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);

	mem_cgroup_put(mem);
B
Balbir Singh 已提交
3385 3386 3387 3388 3389
}

static int mem_cgroup_populate(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
3390 3391 3392 3393 3394 3395 3396 3397
	int ret;

	ret = cgroup_add_files(cont, ss, mem_cgroup_files,
				ARRAY_SIZE(mem_cgroup_files));

	if (!ret)
		ret = register_memsw_files(cont, ss);
	return ret;
B
Balbir Singh 已提交
3398 3399
}

B
Balbir Singh 已提交
3400 3401 3402
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
3403 3404
				struct task_struct *p,
				bool threadgroup)
B
Balbir Singh 已提交
3405
{
3406
	mutex_lock(&memcg_tasklist);
B
Balbir Singh 已提交
3407
	/*
3408 3409
	 * FIXME: It's better to move charges of this process from old
	 * memcg to new memcg. But it's just on TODO-List now.
B
Balbir Singh 已提交
3410
	 */
3411
	mutex_unlock(&memcg_tasklist);
B
Balbir Singh 已提交
3412 3413
}

B
Balbir Singh 已提交
3414 3415 3416 3417
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
	.create = mem_cgroup_create,
3418
	.pre_destroy = mem_cgroup_pre_destroy,
B
Balbir Singh 已提交
3419 3420
	.destroy = mem_cgroup_destroy,
	.populate = mem_cgroup_populate,
B
Balbir Singh 已提交
3421
	.attach = mem_cgroup_move_task,
3422
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
3423
	.use_id = 1,
B
Balbir Singh 已提交
3424
};
3425 3426 3427 3428 3429 3430 3431 3432 3433 3434

#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP

static int __init disable_swap_account(char *s)
{
	really_do_swap_account = 0;
	return 1;
}
__setup("noswapaccount", disable_swap_account);
#endif