memcontrol.c 29.0 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21 22
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
23
#include <linux/mm.h>
24
#include <linux/smp.h>
25
#include <linux/page-flags.h>
26
#include <linux/backing-dev.h>
27 28
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
29 30 31
#include <linux/swap.h>
#include <linux/spinlock.h>
#include <linux/fs.h>
32
#include <linux/seq_file.h>
B
Balbir Singh 已提交
33

34 35
#include <asm/uaccess.h>

B
Balbir Singh 已提交
36
struct cgroup_subsys mem_cgroup_subsys;
37
static const int MEM_CGROUP_RECLAIM_RETRIES = 5;
B
Balbir Singh 已提交
38

39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as rss */

	MEM_CGROUP_STAT_NSTATS,
};

struct mem_cgroup_stat_cpu {
	s64 count[MEM_CGROUP_STAT_NSTATS];
} ____cacheline_aligned_in_smp;

struct mem_cgroup_stat {
	struct mem_cgroup_stat_cpu cpustat[NR_CPUS];
};

/*
 * For accounting under irq disable, no need for increment preempt count.
 */
static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat,
		enum mem_cgroup_stat_index idx, int val)
{
	int cpu = smp_processor_id();
	stat->cpustat[cpu].count[idx] += val;
}

static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
		enum mem_cgroup_stat_index idx)
{
	int cpu;
	s64 ret = 0;
	for_each_possible_cpu(cpu)
		ret += stat->cpustat[cpu].count[idx];
	return ret;
}

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
/*
 * per-zone information in memory controller.
 */

enum mem_cgroup_zstat_index {
	MEM_CGROUP_ZSTAT_ACTIVE,
	MEM_CGROUP_ZSTAT_INACTIVE,

	NR_MEM_CGROUP_ZSTAT,
};

struct mem_cgroup_per_zone {
	unsigned long count[NR_MEM_CGROUP_ZSTAT];
};
/* Macro for accessing counter */
#define MEM_CGROUP_ZSTAT(mz, idx)	((mz)->count[(idx)])

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
	struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
};

B
Balbir Singh 已提交
105 106 107 108 109 110 111
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
112 113 114
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
115 116 117 118 119 120 121
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
122 123 124 125 126 127 128
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 * TODO: Consider making these lists per zone
	 */
	struct list_head active_list;
	struct list_head inactive_list;
129
	struct mem_cgroup_lru_info info;
130 131 132 133
	/*
	 * spin_lock to protect the per cgroup LRU
	 */
	spinlock_t lru_lock;
134
	unsigned long control_type;	/* control RSS or RSS+Pagecache */
135
	int	prev_priority;	/* for recording reclaim priority */
136 137 138 139
	/*
	 * statistics.
	 */
	struct mem_cgroup_stat stat;
B
Balbir Singh 已提交
140 141
};

142 143 144 145 146 147 148 149
/*
 * We use the lower bit of the page->page_cgroup pointer as a bit spin
 * lock. We need to ensure that page->page_cgroup is atleast two
 * byte aligned (based on comments from Nick Piggin)
 */
#define PAGE_CGROUP_LOCK_BIT 	0x0
#define PAGE_CGROUP_LOCK 		(1 << PAGE_CGROUP_LOCK_BIT)

B
Balbir Singh 已提交
150 151 152 153 154 155 156 157
/*
 * A page_cgroup page is associated with every page descriptor. The
 * page_cgroup helps us identify information about the cgroup
 */
struct page_cgroup {
	struct list_head lru;		/* per cgroup LRU list */
	struct page *page;
	struct mem_cgroup *mem_cgroup;
158 159
	atomic_t ref_cnt;		/* Helpful when pages move b/w  */
					/* mapped and cached states     */
160
	int	 flags;
B
Balbir Singh 已提交
161
};
162
#define PAGE_CGROUP_FLAG_CACHE	(0x1)	/* charged as cache */
163
#define PAGE_CGROUP_FLAG_ACTIVE (0x2)	/* page is active in this cgroup */
B
Balbir Singh 已提交
164

165 166 167 168 169 170 171 172 173 174
static inline int page_cgroup_nid(struct page_cgroup *pc)
{
	return page_to_nid(pc->page);
}

static inline enum zone_type page_cgroup_zid(struct page_cgroup *pc)
{
	return page_zonenum(pc->page);
}

175 176 177 178 179 180 181 182
enum {
	MEM_CGROUP_TYPE_UNSPEC = 0,
	MEM_CGROUP_TYPE_MAPPED,
	MEM_CGROUP_TYPE_CACHED,
	MEM_CGROUP_TYPE_ALL,
	MEM_CGROUP_TYPE_MAX,
};

183 184 185 186 187
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
	MEM_CGROUP_CHARGE_TYPE_MAPPED,
};

188

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
/*
 * Always modified under lru lock. Then, not necessary to preempt_disable()
 */
static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
					bool charge)
{
	int val = (charge)? 1 : -1;
	struct mem_cgroup_stat *stat = &mem->stat;
	VM_BUG_ON(!irqs_disabled());

	if (flags & PAGE_CGROUP_FLAG_CACHE)
		__mem_cgroup_stat_add_safe(stat,
					MEM_CGROUP_STAT_CACHE, val);
	else
		__mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
}

static inline struct mem_cgroup_per_zone *
mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
{
	BUG_ON(!mem->info.nodeinfo[nid]);
	return &mem->info.nodeinfo[nid]->zoneinfo[zid];
}

static inline struct mem_cgroup_per_zone *
page_cgroup_zoneinfo(struct page_cgroup *pc)
{
	struct mem_cgroup *mem = pc->mem_cgroup;
	int nid = page_cgroup_nid(pc);
	int zid = page_cgroup_zid(pc);
219

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
	return mem_cgroup_zoneinfo(mem, nid, zid);
}

static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
					enum mem_cgroup_zstat_index idx)
{
	int nid, zid;
	struct mem_cgroup_per_zone *mz;
	u64 total = 0;

	for_each_online_node(nid)
		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			mz = mem_cgroup_zoneinfo(mem, nid, zid);
			total += MEM_CGROUP_ZSTAT(mz, idx);
		}
	return total;
236 237
}

238
static struct mem_cgroup init_mem_cgroup;
B
Balbir Singh 已提交
239 240 241 242 243 244 245 246 247

static inline
struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
{
	return container_of(cgroup_subsys_state(cont,
				mem_cgroup_subsys_id), struct mem_cgroup,
				css);
}

248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
static inline
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
{
	return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
				struct mem_cgroup, css);
}

void mm_init_cgroup(struct mm_struct *mm, struct task_struct *p)
{
	struct mem_cgroup *mem;

	mem = mem_cgroup_from_task(p);
	css_get(&mem->css);
	mm->mem_cgroup = mem;
}

void mm_free_cgroup(struct mm_struct *mm)
{
	css_put(&mm->mem_cgroup->css);
}

269 270 271 272 273 274
static inline int page_cgroup_locked(struct page *page)
{
	return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT,
					&page->page_cgroup);
}

275 276
void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
{
277 278 279 280 281 282 283 284 285 286 287
	int locked;

	/*
	 * While resetting the page_cgroup we might not hold the
	 * page_cgroup lock. free_hot_cold_page() is an example
	 * of such a scenario
	 */
	if (pc)
		VM_BUG_ON(!page_cgroup_locked(page));
	locked = (page->page_cgroup & PAGE_CGROUP_LOCK);
	page->page_cgroup = ((unsigned long)pc | locked);
288 289 290 291
}

struct page_cgroup *page_get_page_cgroup(struct page *page)
{
292 293 294 295
	return (struct page_cgroup *)
		(page->page_cgroup & ~PAGE_CGROUP_LOCK);
}

296
static void __always_inline lock_page_cgroup(struct page *page)
297 298 299 300 301
{
	bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
	VM_BUG_ON(!page_cgroup_locked(page));
}

302
static void __always_inline unlock_page_cgroup(struct page *page)
303 304 305 306
{
	bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
}

307 308 309 310 311
/*
 * Tie new page_cgroup to struct page under lock_page_cgroup()
 * This can fail if the page has been tied to a page_cgroup.
 * If success, returns 0.
 */
312 313
static int page_cgroup_assign_new_page_cgroup(struct page *page,
						struct page_cgroup *pc)
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
{
	int ret = 0;

	lock_page_cgroup(page);
	if (!page_get_page_cgroup(page))
		page_assign_page_cgroup(page, pc);
	else /* A page is tied to other pc. */
		ret = 1;
	unlock_page_cgroup(page);
	return ret;
}

/*
 * Clear page->page_cgroup member under lock_page_cgroup().
 * If given "pc" value is different from one page->page_cgroup,
 * page->cgroup is not cleared.
 * Returns a value of page->page_cgroup at lock taken.
 * A can can detect failure of clearing by following
 *  clear_page_cgroup(page, pc) == pc
 */

335 336
static struct page_cgroup *clear_page_cgroup(struct page *page,
						struct page_cgroup *pc)
337 338 339 340 341 342 343 344 345 346 347
{
	struct page_cgroup *ret;
	/* lock and clear */
	lock_page_cgroup(page);
	ret = page_get_page_cgroup(page);
	if (likely(ret == pc))
		page_assign_page_cgroup(page, NULL);
	unlock_page_cgroup(page);
	return ret;
}

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
static void __mem_cgroup_remove_list(struct page_cgroup *pc)
{
	int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
	struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);

	if (from)
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
	else
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;

	mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false);
	list_del_init(&pc->lru);
}

static void __mem_cgroup_add_list(struct page_cgroup *pc)
{
	int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
	struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);

	if (!to) {
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
		list_add(&pc->lru, &pc->mem_cgroup->inactive_list);
	} else {
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
		list_add(&pc->lru, &pc->mem_cgroup->active_list);
	}
	mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true);
}

377
static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
378
{
379 380 381 382 383 384 385 386
	int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
	struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);

	if (from)
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
	else
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;

387
	if (active) {
388
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
389
		pc->flags |= PAGE_CGROUP_FLAG_ACTIVE;
390
		list_move(&pc->lru, &pc->mem_cgroup->active_list);
391
	} else {
392
		MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
393
		pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE;
394
		list_move(&pc->lru, &pc->mem_cgroup->inactive_list);
395
	}
396 397
}

398 399 400 401 402 403 404 405 406 407
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
{
	int ret;

	task_lock(task);
	ret = task->mm && mm_cgroup(task->mm) == mem;
	task_unlock(task);
	return ret;
}

408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
/*
 * This routine assumes that the appropriate zone's lru lock is already held
 */
void mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
{
	struct mem_cgroup *mem;
	if (!pc)
		return;

	mem = pc->mem_cgroup;

	spin_lock(&mem->lru_lock);
	__mem_cgroup_move_lists(pc, active);
	spin_unlock(&mem->lru_lock);
}

424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
/*
 * Calculate mapped_ratio under memory controller. This will be used in
 * vmscan.c for deteremining we have to reclaim mapped pages.
 */
int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
{
	long total, rss;

	/*
	 * usage is recorded in bytes. But, here, we assume the number of
	 * physical pages can be represented by "long" on any arch.
	 */
	total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
	rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
	return (int)((rss * 100L) / total);
}
440 441 442 443 444 445 446 447 448 449 450 451 452 453
/*
 * This function is called from vmscan.c. In page reclaiming loop. balance
 * between active and inactive list is calculated. For memory controller
 * page reclaiming, we should use using mem_cgroup's imbalance rather than
 * zone's global lru imbalance.
 */
long mem_cgroup_reclaim_imbalance(struct mem_cgroup *mem)
{
	unsigned long active, inactive;
	/* active and inactive are the number of pages. 'long' is ok.*/
	active = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_ACTIVE);
	inactive = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_INACTIVE);
	return (long) (active / (inactive + 1));
}
454

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
/*
 * prev_priority control...this will be used in memory reclaim path.
 */
int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
{
	return mem->prev_priority;
}

void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
{
	if (priority < mem->prev_priority)
		mem->prev_priority = priority;
}

void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
{
	mem->prev_priority = priority;
}

474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
/*
 * Calculate # of pages to be scanned in this priority/zone.
 * See also vmscan.c
 *
 * priority starts from "DEF_PRIORITY" and decremented in each loop.
 * (see include/linux/mmzone.h)
 */

long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
				   struct zone *zone, int priority)
{
	long nr_active;
	int nid = zone->zone_pgdat->node_id;
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);

	nr_active = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE);
	return (nr_active >> priority);
}

long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
					struct zone *zone, int priority)
{
	long nr_inactive;
	int nid = zone->zone_pgdat->node_id;
	int zid = zone_idx(zone);
	struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);

	nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);

	return (nr_inactive >> priority);
}

507 508 509 510 511 512 513 514 515 516 517 518
unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
					struct list_head *dst,
					unsigned long *scanned, int order,
					int mode, struct zone *z,
					struct mem_cgroup *mem_cont,
					int active)
{
	unsigned long nr_taken = 0;
	struct page *page;
	unsigned long scan;
	LIST_HEAD(pc_list);
	struct list_head *src;
519
	struct page_cgroup *pc, *tmp;
520 521 522 523 524 525 526

	if (active)
		src = &mem_cont->active_list;
	else
		src = &mem_cont->inactive_list;

	spin_lock(&mem_cont->lru_lock);
527 528
	scan = 0;
	list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
H
Hugh Dickins 已提交
529
		if (scan >= nr_to_scan)
530
			break;
531 532 533
		page = pc->page;
		VM_BUG_ON(!pc);

H
Hugh Dickins 已提交
534
		if (unlikely(!PageLRU(page)))
535 536
			continue;

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
		if (PageActive(page) && !active) {
			__mem_cgroup_move_lists(pc, true);
			continue;
		}
		if (!PageActive(page) && active) {
			__mem_cgroup_move_lists(pc, false);
			continue;
		}

		/*
		 * Reclaim, per zone
		 * TODO: make the active/inactive lists per zone
		 */
		if (page_zone(page) != z)
			continue;

H
Hugh Dickins 已提交
553 554
		scan++;
		list_move(&pc->lru, &pc_list);
555 556 557 558 559 560 561 562 563 564 565 566 567 568

		if (__isolate_lru_page(page, mode) == 0) {
			list_move(&page->lru, dst);
			nr_taken++;
		}
	}

	list_splice(&pc_list, src);
	spin_unlock(&mem_cont->lru_lock);

	*scanned = scan;
	return nr_taken;
}

569 570 571 572 573 574
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
575 576
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask, enum charge_type ctype)
577 578
{
	struct mem_cgroup *mem;
579
	struct page_cgroup *pc;
580 581
	unsigned long flags;
	unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
582 583 584 585 586 587 588 589

	/*
	 * Should page_cgroup's go to their own slab?
	 * One could optimize the performance of the charging routine
	 * by saving a bit in the page_flags and using it as a lock
	 * to see if the cgroup page already has a page_cgroup associated
	 * with it
	 */
590
retry:
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
	if (page) {
		lock_page_cgroup(page);
		pc = page_get_page_cgroup(page);
		/*
		 * The page_cgroup exists and
		 * the page has already been accounted.
		 */
		if (pc) {
			if (unlikely(!atomic_inc_not_zero(&pc->ref_cnt))) {
				/* this page is under being uncharged ? */
				unlock_page_cgroup(page);
				cpu_relax();
				goto retry;
			} else {
				unlock_page_cgroup(page);
				goto done;
			}
608
		}
609
		unlock_page_cgroup(page);
610 611
	}

612
	pc = kzalloc(sizeof(struct page_cgroup), gfp_mask);
613 614 615 616
	if (pc == NULL)
		goto err;

	/*
617 618
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
619 620 621 622 623 624
	 * thread group leader migrates. It's possible that mm is not
	 * set, if so charge the init_mm (happens for pagecache usage).
	 */
	if (!mm)
		mm = &init_mm;

625
	rcu_read_lock();
626 627 628 629 630 631 632 633 634 635 636 637
	mem = rcu_dereference(mm->mem_cgroup);
	/*
	 * For every charge from the cgroup, increment reference
	 * count
	 */
	css_get(&mem->css);
	rcu_read_unlock();

	/*
	 * If we created the page_cgroup, we should free it on exceeding
	 * the cgroup limit.
	 */
638
	while (res_counter_charge(&mem->res, PAGE_SIZE)) {
639 640
		if (!(gfp_mask & __GFP_WAIT))
			goto out;
641 642

		if (try_to_free_mem_cgroup_pages(mem, gfp_mask))
643 644 645 646 647 648 649 650 651 652 653
			continue;

		/*
 		 * try_to_free_mem_cgroup_pages() might not give us a full
 		 * picture of reclaim. Some pages are reclaimed and might be
 		 * moved to swap cache or just unmapped from the cgroup.
 		 * Check the limit again to see if the reclaim reduced the
 		 * current usage of the cgroup before giving up
 		 */
		if (res_counter_check_under_limit(&mem->res))
			continue;
654 655 656 657

		if (!nr_retries--) {
			mem_cgroup_out_of_memory(mem, gfp_mask);
			goto out;
658
		}
659
		congestion_wait(WRITE, HZ/10);
660 661 662 663 664
	}

	atomic_set(&pc->ref_cnt, 1);
	pc->mem_cgroup = mem;
	pc->page = page;
665
	pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
666 667
	if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
		pc->flags |= PAGE_CGROUP_FLAG_CACHE;
668

669
	if (!page || page_cgroup_assign_new_page_cgroup(page, pc)) {
670
		/*
671 672
		 * Another charge has been added to this page already.
		 * We take lock_page_cgroup(page) again and read
673 674 675 676 677
		 * page->cgroup, increment refcnt.... just retry is OK.
		 */
		res_counter_uncharge(&mem->res, PAGE_SIZE);
		css_put(&mem->css);
		kfree(pc);
678 679
		if (!page)
			goto done;
680 681
		goto retry;
	}
682

683
	spin_lock_irqsave(&mem->lru_lock, flags);
684
	/* Update statistics vector */
685
	__mem_cgroup_add_list(pc);
686 687
	spin_unlock_irqrestore(&mem->lru_lock, flags);

688 689
done:
	return 0;
690 691
out:
	css_put(&mem->css);
692 693 694 695 696
	kfree(pc);
err:
	return -ENOMEM;
}

697 698 699 700 701 702 703
int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
			gfp_t gfp_mask)
{
	return mem_cgroup_charge_common(page, mm, gfp_mask,
			MEM_CGROUP_CHARGE_TYPE_MAPPED);
}

704 705 706
/*
 * See if the cached pages should be charged at all?
 */
707 708
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
709
{
710
	int ret = 0;
711 712 713 714
	struct mem_cgroup *mem;
	if (!mm)
		mm = &init_mm;

715
	rcu_read_lock();
716
	mem = rcu_dereference(mm->mem_cgroup);
717 718
	css_get(&mem->css);
	rcu_read_unlock();
719
	if (mem->control_type == MEM_CGROUP_TYPE_ALL)
720
		ret = mem_cgroup_charge_common(page, mm, gfp_mask,
721
				MEM_CGROUP_CHARGE_TYPE_CACHE);
722 723
	css_put(&mem->css);
	return ret;
724 725
}

726 727 728 729 730 731 732 733
/*
 * Uncharging is always a welcome operation, we never complain, simply
 * uncharge.
 */
void mem_cgroup_uncharge(struct page_cgroup *pc)
{
	struct mem_cgroup *mem;
	struct page *page;
734
	unsigned long flags;
735

736 737 738 739
	/*
	 * This can handle cases when a page is not charged at all and we
	 * are switching between handling the control_type.
	 */
740 741 742 743 744
	if (!pc)
		return;

	if (atomic_dec_and_test(&pc->ref_cnt)) {
		page = pc->page;
745 746
		/*
		 * get page->cgroup and clear it under lock.
747
		 * force_empty can drop page->cgroup without checking refcnt.
748 749 750 751 752 753
		 */
		if (clear_page_cgroup(page, pc) == pc) {
			mem = pc->mem_cgroup;
			css_put(&mem->css);
			res_counter_uncharge(&mem->res, PAGE_SIZE);
			spin_lock_irqsave(&mem->lru_lock, flags);
754
			__mem_cgroup_remove_list(pc);
755 756 757
			spin_unlock_irqrestore(&mem->lru_lock, flags);
			kfree(pc);
		}
758
	}
759
}
760

761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
/*
 * Returns non-zero if a page (under migration) has valid page_cgroup member.
 * Refcnt of page_cgroup is incremented.
 */

int mem_cgroup_prepare_migration(struct page *page)
{
	struct page_cgroup *pc;
	int ret = 0;
	lock_page_cgroup(page);
	pc = page_get_page_cgroup(page);
	if (pc && atomic_inc_not_zero(&pc->ref_cnt))
		ret = 1;
	unlock_page_cgroup(page);
	return ret;
}

void mem_cgroup_end_migration(struct page *page)
{
	struct page_cgroup *pc = page_get_page_cgroup(page);
	mem_cgroup_uncharge(pc);
}
/*
 * We know both *page* and *newpage* are now not-on-LRU and Pg_locked.
 * And no race with uncharge() routines because page_cgroup for *page*
 * has extra one reference by mem_cgroup_prepare_migration.
 */

void mem_cgroup_page_migration(struct page *page, struct page *newpage)
{
	struct page_cgroup *pc;
792 793
	struct mem_cgroup *mem;
	unsigned long flags;
794 795 796 797
retry:
	pc = page_get_page_cgroup(page);
	if (!pc)
		return;
798
	mem = pc->mem_cgroup;
799 800
	if (clear_page_cgroup(page, pc) != pc)
		goto retry;
801 802 803 804

	spin_lock_irqsave(&mem->lru_lock, flags);

	__mem_cgroup_remove_list(pc);
805 806 807 808
	pc->page = newpage;
	lock_page_cgroup(newpage);
	page_assign_page_cgroup(newpage, pc);
	unlock_page_cgroup(newpage);
809 810 811
	__mem_cgroup_add_list(pc);

	spin_unlock_irqrestore(&mem->lru_lock, flags);
812 813
	return;
}
814

815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
/*
 * This routine traverse page_cgroup in given list and drop them all.
 * This routine ignores page_cgroup->ref_cnt.
 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
 */
#define FORCE_UNCHARGE_BATCH	(128)
static void
mem_cgroup_force_empty_list(struct mem_cgroup *mem, struct list_head *list)
{
	struct page_cgroup *pc;
	struct page *page;
	int count;
	unsigned long flags;

retry:
	count = FORCE_UNCHARGE_BATCH;
	spin_lock_irqsave(&mem->lru_lock, flags);

	while (--count && !list_empty(list)) {
		pc = list_entry(list->prev, struct page_cgroup, lru);
		page = pc->page;
		/* Avoid race with charge */
		atomic_set(&pc->ref_cnt, 0);
		if (clear_page_cgroup(page, pc) == pc) {
			css_put(&mem->css);
			res_counter_uncharge(&mem->res, PAGE_SIZE);
841
			__mem_cgroup_remove_list(pc);
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
			kfree(pc);
		} else 	/* being uncharged ? ...do relax */
			break;
	}
	spin_unlock_irqrestore(&mem->lru_lock, flags);
	if (!list_empty(list)) {
		cond_resched();
		goto retry;
	}
	return;
}

/*
 * make mem_cgroup's charge to be 0 if there is no task.
 * This enables deleting this mem_cgroup.
 */

int mem_cgroup_force_empty(struct mem_cgroup *mem)
{
	int ret = -EBUSY;
	css_get(&mem->css);
	/*
	 * page reclaim code (kswapd etc..) will move pages between
`	 * active_list <-> inactive_list while we don't take a lock.
	 * So, we have to do loop here until all lists are empty.
	 */
	while (!(list_empty(&mem->active_list) &&
		 list_empty(&mem->inactive_list))) {
		if (atomic_read(&mem->css.cgroup->count) > 0)
			goto out;
		/* drop all page_cgroup in active_list */
		mem_cgroup_force_empty_list(mem, &mem->active_list);
		/* drop all page_cgroup in inactive_list */
		mem_cgroup_force_empty_list(mem, &mem->inactive_list);
	}
	ret = 0;
out:
	css_put(&mem->css);
	return ret;
}



885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
{
	*tmp = memparse(buf, &buf);
	if (*buf != '\0')
		return -EINVAL;

	/*
	 * Round up the value to the closest page size
	 */
	*tmp = ((*tmp + PAGE_SIZE - 1) >> PAGE_SHIFT) << PAGE_SHIFT;
	return 0;
}

static ssize_t mem_cgroup_read(struct cgroup *cont,
			struct cftype *cft, struct file *file,
			char __user *userbuf, size_t nbytes, loff_t *ppos)
B
Balbir Singh 已提交
901 902
{
	return res_counter_read(&mem_cgroup_from_cont(cont)->res,
903 904
				cft->private, userbuf, nbytes, ppos,
				NULL);
B
Balbir Singh 已提交
905 906 907 908 909 910 911
}

static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
				struct file *file, const char __user *userbuf,
				size_t nbytes, loff_t *ppos)
{
	return res_counter_write(&mem_cgroup_from_cont(cont)->res,
912 913
				cft->private, userbuf, nbytes, ppos,
				mem_cgroup_write_strategy);
B
Balbir Singh 已提交
914 915
}

916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
static ssize_t mem_control_type_write(struct cgroup *cont,
			struct cftype *cft, struct file *file,
			const char __user *userbuf,
			size_t nbytes, loff_t *pos)
{
	int ret;
	char *buf, *end;
	unsigned long tmp;
	struct mem_cgroup *mem;

	mem = mem_cgroup_from_cont(cont);
	buf = kmalloc(nbytes + 1, GFP_KERNEL);
	ret = -ENOMEM;
	if (buf == NULL)
		goto out;

	buf[nbytes] = 0;
	ret = -EFAULT;
	if (copy_from_user(buf, userbuf, nbytes))
		goto out_free;

	ret = -EINVAL;
	tmp = simple_strtoul(buf, &end, 10);
	if (*end != '\0')
		goto out_free;

	if (tmp <= MEM_CGROUP_TYPE_UNSPEC || tmp >= MEM_CGROUP_TYPE_MAX)
		goto out_free;

	mem->control_type = tmp;
	ret = nbytes;
out_free:
	kfree(buf);
out:
	return ret;
}

static ssize_t mem_control_type_read(struct cgroup *cont,
				struct cftype *cft,
				struct file *file, char __user *userbuf,
				size_t nbytes, loff_t *ppos)
{
	unsigned long val;
	char buf[64], *s;
	struct mem_cgroup *mem;

	mem = mem_cgroup_from_cont(cont);
	s = buf;
	val = mem->control_type;
	s += sprintf(s, "%lu\n", val);
	return simple_read_from_buffer((void __user *)userbuf, nbytes,
			ppos, buf, s - buf);
}

970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996

static ssize_t mem_force_empty_write(struct cgroup *cont,
				struct cftype *cft, struct file *file,
				const char __user *userbuf,
				size_t nbytes, loff_t *ppos)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
	int ret;
	ret = mem_cgroup_force_empty(mem);
	if (!ret)
		ret = nbytes;
	return ret;
}

/*
 * Note: This should be removed if cgroup supports write-only file.
 */

static ssize_t mem_force_empty_read(struct cgroup *cont,
				struct cftype *cft,
				struct file *file, char __user *userbuf,
				size_t nbytes, loff_t *ppos)
{
	return -EINVAL;
}


997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
static const struct mem_cgroup_stat_desc {
	const char *msg;
	u64 unit;
} mem_cgroup_stat_desc[] = {
	[MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
	[MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
};

static int mem_control_stat_show(struct seq_file *m, void *arg)
{
	struct cgroup *cont = m->private;
	struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
	struct mem_cgroup_stat *stat = &mem_cont->stat;
	int i;

	for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
		s64 val;

		val = mem_cgroup_read_stat(stat, i);
		val *= mem_cgroup_stat_desc[i].unit;
		seq_printf(m, "%s %lld\n", mem_cgroup_stat_desc[i].msg,
				(long long)val);
	}
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
	/* showing # of active pages */
	{
		unsigned long active, inactive;

		inactive = mem_cgroup_get_all_zonestat(mem_cont,
						MEM_CGROUP_ZSTAT_INACTIVE);
		active = mem_cgroup_get_all_zonestat(mem_cont,
						MEM_CGROUP_ZSTAT_ACTIVE);
		seq_printf(m, "active %ld\n", (active) * PAGE_SIZE);
		seq_printf(m, "inactive %ld\n", (inactive) * PAGE_SIZE);
	}
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
	return 0;
}

static const struct file_operations mem_control_stat_file_operations = {
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int mem_control_stat_open(struct inode *unused, struct file *file)
{
	/* XXX __d_cont */
	struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;

	file->f_op = &mem_control_stat_file_operations;
	return single_open(file, mem_control_stat_show, cont);
}



B
Balbir Singh 已提交
1051 1052
static struct cftype mem_cgroup_files[] = {
	{
1053
		.name = "usage_in_bytes",
B
Balbir Singh 已提交
1054 1055 1056 1057
		.private = RES_USAGE,
		.read = mem_cgroup_read,
	},
	{
1058
		.name = "limit_in_bytes",
B
Balbir Singh 已提交
1059 1060 1061 1062 1063 1064 1065 1066 1067
		.private = RES_LIMIT,
		.write = mem_cgroup_write,
		.read = mem_cgroup_read,
	},
	{
		.name = "failcnt",
		.private = RES_FAILCNT,
		.read = mem_cgroup_read,
	},
1068 1069 1070 1071 1072
	{
		.name = "control_type",
		.write = mem_control_type_write,
		.read = mem_control_type_read,
	},
1073 1074 1075 1076 1077
	{
		.name = "force_empty",
		.write = mem_force_empty_write,
		.read = mem_force_empty_read,
	},
1078 1079 1080 1081
	{
		.name = "stat",
		.open = mem_control_stat_open,
	},
B
Balbir Singh 已提交
1082 1083
};

1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
{
	struct mem_cgroup_per_node *pn;

	pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, node);
	if (!pn)
		return 1;
	mem->info.nodeinfo[node] = pn;
	memset(pn, 0, sizeof(*pn));
	return 0;
}

1096 1097
static struct mem_cgroup init_mem_cgroup;

B
Balbir Singh 已提交
1098 1099 1100 1101
static struct cgroup_subsys_state *
mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
{
	struct mem_cgroup *mem;
1102
	int node;
B
Balbir Singh 已提交
1103

1104 1105 1106 1107 1108 1109 1110 1111
	if (unlikely((cont->parent) == NULL)) {
		mem = &init_mem_cgroup;
		init_mm.mem_cgroup = mem;
	} else
		mem = kzalloc(sizeof(struct mem_cgroup), GFP_KERNEL);

	if (mem == NULL)
		return NULL;
B
Balbir Singh 已提交
1112 1113

	res_counter_init(&mem->res);
1114 1115
	INIT_LIST_HEAD(&mem->active_list);
	INIT_LIST_HEAD(&mem->inactive_list);
1116
	spin_lock_init(&mem->lru_lock);
1117
	mem->control_type = MEM_CGROUP_TYPE_ALL;
1118 1119 1120 1121 1122 1123
	memset(&mem->info, 0, sizeof(mem->info));

	for_each_node_state(node, N_POSSIBLE)
		if (alloc_mem_cgroup_per_zone_info(mem, node))
			goto free_out;

B
Balbir Singh 已提交
1124
	return &mem->css;
1125 1126 1127 1128 1129 1130
free_out:
	for_each_node_state(node, N_POSSIBLE)
		kfree(mem->info.nodeinfo[node]);
	if (cont->parent != NULL)
		kfree(mem);
	return NULL;
B
Balbir Singh 已提交
1131 1132
}

1133 1134 1135 1136 1137 1138 1139
static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
					struct cgroup *cont)
{
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
	mem_cgroup_force_empty(mem);
}

B
Balbir Singh 已提交
1140 1141 1142
static void mem_cgroup_destroy(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
1143 1144 1145 1146 1147 1148
	int node;
	struct mem_cgroup *mem = mem_cgroup_from_cont(cont);

	for_each_node_state(node, N_POSSIBLE)
		kfree(mem->info.nodeinfo[node]);

B
Balbir Singh 已提交
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158
	kfree(mem_cgroup_from_cont(cont));
}

static int mem_cgroup_populate(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
	return cgroup_add_files(cont, ss, mem_cgroup_files,
					ARRAY_SIZE(mem_cgroup_files));
}

B
Balbir Singh 已提交
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
static void mem_cgroup_move_task(struct cgroup_subsys *ss,
				struct cgroup *cont,
				struct cgroup *old_cont,
				struct task_struct *p)
{
	struct mm_struct *mm;
	struct mem_cgroup *mem, *old_mem;

	mm = get_task_mm(p);
	if (mm == NULL)
		return;

	mem = mem_cgroup_from_cont(cont);
	old_mem = mem_cgroup_from_cont(old_cont);

	if (mem == old_mem)
		goto out;

	/*
	 * Only thread group leaders are allowed to migrate, the mm_struct is
	 * in effect owned by the leader
	 */
	if (p->tgid != p->pid)
		goto out;

	css_get(&mem->css);
	rcu_assign_pointer(mm->mem_cgroup, mem);
	css_put(&old_mem->css);

out:
	mmput(mm);
	return;
}

B
Balbir Singh 已提交
1193 1194 1195 1196
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
	.create = mem_cgroup_create,
1197
	.pre_destroy = mem_cgroup_pre_destroy,
B
Balbir Singh 已提交
1198 1199
	.destroy = mem_cgroup_destroy,
	.populate = mem_cgroup_populate,
B
Balbir Singh 已提交
1200
	.attach = mem_cgroup_move_task,
1201
	.early_init = 0,
B
Balbir Singh 已提交
1202
};