memcontrol.c 146.3 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
13 14 15 16
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
17 18 19 20 21 22
 * Native page reclaim
 * Charge lifetime sanitation
 * Lockless page tracking & accounting
 * Unified hierarchy configuration model
 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
 *
B
Balbir Singh 已提交
23 24 25 26 27 28 29 30 31 32 33
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

34
#include <linux/page_counter.h>
B
Balbir Singh 已提交
35 36
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
37
#include <linux/mm.h>
38
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
39
#include <linux/pagemap.h>
40
#include <linux/smp.h>
41
#include <linux/page-flags.h>
42
#include <linux/backing-dev.h>
43 44
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
45
#include <linux/limits.h>
46
#include <linux/export.h>
47
#include <linux/mutex.h>
48
#include <linux/rbtree.h>
49
#include <linux/slab.h>
50
#include <linux/swap.h>
51
#include <linux/swapops.h>
52
#include <linux/spinlock.h>
53
#include <linux/eventfd.h>
54
#include <linux/poll.h>
55
#include <linux/sort.h>
56
#include <linux/fs.h>
57
#include <linux/seq_file.h>
58
#include <linux/vmpressure.h>
59
#include <linux/mm_inline.h>
60
#include <linux/swap_cgroup.h>
61
#include <linux/cpu.h>
62
#include <linux/oom.h>
63
#include <linux/lockdep.h>
64
#include <linux/file.h>
K
KAMEZAWA Hiroyuki 已提交
65
#include "internal.h"
G
Glauber Costa 已提交
66
#include <net/sock.h>
M
Michal Hocko 已提交
67
#include <net/ip.h>
G
Glauber Costa 已提交
68
#include <net/tcp_memcontrol.h>
69
#include "slab.h"
B
Balbir Singh 已提交
70

71 72
#include <asm/uaccess.h>

73 74
#include <trace/events/vmscan.h>

75 76
struct cgroup_subsys memory_cgrp_subsys __read_mostly;
EXPORT_SYMBOL(memory_cgrp_subsys);
77

78
#define MEM_CGROUP_RECLAIM_RETRIES	5
79
static struct mem_cgroup *root_mem_cgroup __read_mostly;
T
Tejun Heo 已提交
80
struct cgroup_subsys_state *mem_cgroup_root_css __read_mostly;
B
Balbir Singh 已提交
81

82
/* Whether the swap controller is active */
A
Andrew Morton 已提交
83
#ifdef CONFIG_MEMCG_SWAP
84 85
int do_swap_account __read_mostly;
#else
86
#define do_swap_account		0
87 88
#endif

89 90 91
static const char * const mem_cgroup_stat_names[] = {
	"cache",
	"rss",
92
	"rss_huge",
93
	"mapped_file",
94
	"dirty",
95
	"writeback",
96 97 98 99 100 101 102 103 104 105
	"swap",
};

static const char * const mem_cgroup_events_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

106 107 108 109 110 111 112 113
static const char * const mem_cgroup_lru_names[] = {
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

114 115 116
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
117

118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

K
KAMEZAWA Hiroyuki 已提交
138 139 140 141 142
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
143

144 145 146
/*
 * cgroup_event represents events which userspace want to receive.
 */
147
struct mem_cgroup_event {
148
	/*
149
	 * memcg which the event belongs to.
150
	 */
151
	struct mem_cgroup *memcg;
152 153 154 155 156 157 158 159
	/*
	 * eventfd to signal userspace about the event.
	 */
	struct eventfd_ctx *eventfd;
	/*
	 * Each of these stored in a list by the cgroup.
	 */
	struct list_head list;
160 161 162 163 164
	/*
	 * register_event() callback will be used to add new userspace
	 * waiter for changes related to this event.  Use eventfd_signal()
	 * on eventfd to send notification to userspace.
	 */
165
	int (*register_event)(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
166
			      struct eventfd_ctx *eventfd, const char *args);
167 168 169 170 171
	/*
	 * unregister_event() callback will be called when userspace closes
	 * the eventfd or on cgroup removing.  This callback must be set,
	 * if you want provide notification functionality.
	 */
172
	void (*unregister_event)(struct mem_cgroup *memcg,
173
				 struct eventfd_ctx *eventfd);
174 175 176 177 178 179 180 181 182 183
	/*
	 * All fields below needed to unregister event when
	 * userspace closes eventfd.
	 */
	poll_table pt;
	wait_queue_head_t *wqh;
	wait_queue_t wait;
	struct work_struct remove;
};

184 185
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
186

187 188
/* Stuffs for move charges at task migration. */
/*
189
 * Types of charges to be moved.
190
 */
191 192 193
#define MOVE_ANON	0x1U
#define MOVE_FILE	0x2U
#define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
194

195 196
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
197
	spinlock_t	  lock; /* for from, to */
198 199
	struct mem_cgroup *from;
	struct mem_cgroup *to;
200
	unsigned long flags;
201
	unsigned long precharge;
202
	unsigned long moved_charge;
203
	unsigned long moved_swap;
204 205 206
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
207
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
208 209
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
210

211 212 213 214
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
215
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
216
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
217

218 219
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
220
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
221
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
222
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
223 224 225
	NR_CHARGE_TYPE,
};

226
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
227 228 229 230
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
231
	_KMEM,
G
Glauber Costa 已提交
232 233
};

234 235
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
236
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
237 238
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
239

240 241 242 243 244 245 246
/*
 * The memcg_create_mutex will be held whenever a new cgroup is created.
 * As a consequence, any change that needs to protect against new child cgroups
 * appearing has to hold it as well.
 */
static DEFINE_MUTEX(memcg_create_mutex);

247 248 249 250 251 252 253 254 255 256 257 258 259
/* Some nice accessors for the vmpressure. */
struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
{
	if (!memcg)
		memcg = root_mem_cgroup;
	return &memcg->vmpressure;
}

struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
{
	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}

260 261 262 263 264
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

265 266 267 268 269 270
/*
 * We restrict the id in the range of [1, 65535], so it can fit into
 * an unsigned short.
 */
#define MEM_CGROUP_ID_MAX	USHRT_MAX

L
Li Zefan 已提交
271 272
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
273
	return memcg->css.id;
L
Li Zefan 已提交
274 275
}

276 277 278 279 280 281
/*
 * A helper function to get mem_cgroup from ID. must be called under
 * rcu_read_lock().  The caller is responsible for calling
 * css_tryget_online() if the mem_cgroup is used for charging. (dropping
 * refcnt from swap can be called against removed memcg.)
 */
L
Li Zefan 已提交
282 283 284 285
static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	struct cgroup_subsys_state *css;

286
	css = css_from_id(id, &memory_cgrp_subsys);
L
Li Zefan 已提交
287 288 289
	return mem_cgroup_from_css(css);
}

G
Glauber Costa 已提交
290
/* Writing them here to avoid exposing memcg's inner layout */
M
Michal Hocko 已提交
291
#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
G
Glauber Costa 已提交
292 293 294

void sock_update_memcg(struct sock *sk)
{
295
	if (mem_cgroup_sockets_enabled) {
G
Glauber Costa 已提交
296
		struct mem_cgroup *memcg;
297
		struct cg_proto *cg_proto;
G
Glauber Costa 已提交
298 299 300

		BUG_ON(!sk->sk_prot->proto_cgroup);

301 302 303 304 305 306 307 308 309 310
		/* Socket cloning can throw us here with sk_cgrp already
		 * filled. It won't however, necessarily happen from
		 * process context. So the test for root memcg given
		 * the current task's memcg won't help us in this case.
		 *
		 * Respecting the original socket's memcg is a better
		 * decision in this case.
		 */
		if (sk->sk_cgrp) {
			BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
311
			css_get(&sk->sk_cgrp->memcg->css);
312 313 314
			return;
		}

G
Glauber Costa 已提交
315 316
		rcu_read_lock();
		memcg = mem_cgroup_from_task(current);
317
		cg_proto = sk->sk_prot->proto_cgroup(memcg);
318
		if (cg_proto && test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags) &&
319
		    css_tryget_online(&memcg->css)) {
320
			sk->sk_cgrp = cg_proto;
G
Glauber Costa 已提交
321 322 323 324 325 326 327 328
		}
		rcu_read_unlock();
	}
}
EXPORT_SYMBOL(sock_update_memcg);

void sock_release_memcg(struct sock *sk)
{
329
	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
G
Glauber Costa 已提交
330 331 332
		struct mem_cgroup *memcg;
		WARN_ON(!sk->sk_cgrp->memcg);
		memcg = sk->sk_cgrp->memcg;
333
		css_put(&sk->sk_cgrp->memcg->css);
G
Glauber Costa 已提交
334 335
	}
}
G
Glauber Costa 已提交
336 337 338 339 340 341

struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg || mem_cgroup_is_root(memcg))
		return NULL;

342
	return &memcg->tcp_mem;
G
Glauber Costa 已提交
343 344
}
EXPORT_SYMBOL(tcp_proto_cgroup);
G
Glauber Costa 已提交
345

346 347
#endif

348
#ifdef CONFIG_MEMCG_KMEM
349
/*
350
 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
L
Li Zefan 已提交
351 352 353 354 355
 * The main reason for not using cgroup id for this:
 *  this works better in sparse environments, where we have a lot of memcgs,
 *  but only a few kmem-limited. Or also, if we have, for instance, 200
 *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *  200 entry array for that.
356
 *
357 358
 * The current size of the caches array is stored in memcg_nr_cache_ids. It
 * will double each time we have to increase it.
359
 */
360 361
static DEFINE_IDA(memcg_cache_ida);
int memcg_nr_cache_ids;
362

363 364 365 366 367 368 369 370 371 372 373 374 375
/* Protects memcg_nr_cache_ids */
static DECLARE_RWSEM(memcg_cache_ids_sem);

void memcg_get_cache_ids(void)
{
	down_read(&memcg_cache_ids_sem);
}

void memcg_put_cache_ids(void)
{
	up_read(&memcg_cache_ids_sem);
}

376 377 378 379 380 381
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
L
Li Zefan 已提交
382
 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
383 384
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
L
Li Zefan 已提交
385
 * cgrp_id space is not getting any smaller, and we don't have to necessarily
386 387 388
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
L
Li Zefan 已提交
389
#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
390

391 392 393 394 395 396
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
397
struct static_key memcg_kmem_enabled_key;
398
EXPORT_SYMBOL(memcg_kmem_enabled_key);
399 400 401

#endif /* CONFIG_MEMCG_KMEM */

402
static struct mem_cgroup_per_zone *
403
mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone)
404
{
405 406 407
	int nid = zone_to_nid(zone);
	int zid = zone_idx(zone);

408
	return &memcg->nodeinfo[nid]->zoneinfo[zid];
409 410
}

411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
/**
 * mem_cgroup_css_from_page - css of the memcg associated with a page
 * @page: page of interest
 *
 * If memcg is bound to the default hierarchy, css of the memcg associated
 * with @page is returned.  The returned css remains associated with @page
 * until it is released.
 *
 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
 * is returned.
 *
 * XXX: The above description of behavior on the default hierarchy isn't
 * strictly true yet as replace_page_cache_page() can modify the
 * association before @page is released even on the default hierarchy;
 * however, the current and planned usages don't mix the the two functions
 * and replace_page_cache_page() will soon be updated to make the invariant
 * actually true.
 */
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
{
	struct mem_cgroup *memcg;

	rcu_read_lock();

	memcg = page->mem_cgroup;

	if (!memcg || !cgroup_on_dfl(memcg->css.cgroup))
		memcg = root_mem_cgroup;

	rcu_read_unlock();
	return &memcg->css;
}

444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/**
 * page_cgroup_ino - return inode number of the memcg a page is charged to
 * @page: the page
 *
 * Look up the closest online ancestor of the memory cgroup @page is charged to
 * and return its inode number or 0 if @page is not charged to any cgroup. It
 * is safe to call this function without holding a reference to @page.
 *
 * Note, this function is inherently racy, because there is nothing to prevent
 * the cgroup inode from getting torn down and potentially reallocated a moment
 * after page_cgroup_ino() returns, so it only should be used by callers that
 * do not care (such as procfs interfaces).
 */
ino_t page_cgroup_ino(struct page *page)
{
	struct mem_cgroup *memcg;
	unsigned long ino = 0;

	rcu_read_lock();
	memcg = READ_ONCE(page->mem_cgroup);
	while (memcg && !(memcg->css.flags & CSS_ONLINE))
		memcg = parent_mem_cgroup(memcg);
	if (memcg)
		ino = cgroup_ino(memcg->css.cgroup);
	rcu_read_unlock();
	return ino;
}

472
static struct mem_cgroup_per_zone *
473
mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page)
474
{
475 476
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
477

478
	return &memcg->nodeinfo[nid]->zoneinfo[zid];
479 480
}

481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

496 497
static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_zone *mz,
					 struct mem_cgroup_tree_per_zone *mctz,
498
					 unsigned long new_usage_in_excess)
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
}

528 529
static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
					 struct mem_cgroup_tree_per_zone *mctz)
530 531 532 533 534 535 536
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

537 538
static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
				       struct mem_cgroup_tree_per_zone *mctz)
539
{
540 541 542
	unsigned long flags;

	spin_lock_irqsave(&mctz->lock, flags);
543
	__mem_cgroup_remove_exceeded(mz, mctz);
544
	spin_unlock_irqrestore(&mctz->lock, flags);
545 546
}

547 548 549
static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
{
	unsigned long nr_pages = page_counter_read(&memcg->memory);
550
	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
551 552 553 554 555 556 557
	unsigned long excess = 0;

	if (nr_pages > soft_limit)
		excess = nr_pages - soft_limit;

	return excess;
}
558 559 560

static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
{
561
	unsigned long excess;
562 563 564
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

565
	mctz = soft_limit_tree_from_page(page);
566 567 568 569 570
	/*
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
571
		mz = mem_cgroup_page_zoneinfo(memcg, page);
572
		excess = soft_limit_excess(memcg);
573 574 575 576 577
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
		if (excess || mz->on_tree) {
578 579 580
			unsigned long flags;

			spin_lock_irqsave(&mctz->lock, flags);
581 582
			/* if on-tree, remove it */
			if (mz->on_tree)
583
				__mem_cgroup_remove_exceeded(mz, mctz);
584 585 586 587
			/*
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
			 */
588
			__mem_cgroup_insert_exceeded(mz, mctz, excess);
589
			spin_unlock_irqrestore(&mctz->lock, flags);
590 591 592 593 594 595 596
		}
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
{
	struct mem_cgroup_tree_per_zone *mctz;
597 598
	struct mem_cgroup_per_zone *mz;
	int nid, zid;
599

600 601 602 603
	for_each_node(nid) {
		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
			mctz = soft_limit_tree_node_zone(nid, zid);
604
			mem_cgroup_remove_exceeded(mz, mctz);
605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
		}
	}
}

static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
	struct mem_cgroup_per_zone *mz;

retry:
	mz = NULL;
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
627
	__mem_cgroup_remove_exceeded(mz, mctz);
628
	if (!soft_limit_excess(mz->memcg) ||
629
	    !css_tryget_online(&mz->memcg->css))
630 631 632 633 634 635 636 637 638 639
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

640
	spin_lock_irq(&mctz->lock);
641
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
642
	spin_unlock_irq(&mctz->lock);
643 644 645
	return mz;
}

646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
665
static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
666
				 enum mem_cgroup_stat_index idx)
667
{
668
	long val = 0;
669 670
	int cpu;

671
	for_each_possible_cpu(cpu)
672
		val += per_cpu(memcg->stat->count[idx], cpu);
673 674 675
	return val;
}

676
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
677 678 679 680 681
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

682
	for_each_possible_cpu(cpu)
683
		val += per_cpu(memcg->stat->events[idx], cpu);
684 685 686
	return val;
}

687
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
688
					 struct page *page,
689
					 int nr_pages)
690
{
691 692 693 694
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
695
	if (PageAnon(page))
696
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
697
				nr_pages);
698
	else
699
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
700
				nr_pages);
701

702 703 704 705
	if (PageTransHuge(page))
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
				nr_pages);

706 707
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
708
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
709
	else {
710
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
711 712
		nr_pages = -nr_pages; /* for event */
	}
713

714
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
715 716
}

717 718 719
static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
						  int nid,
						  unsigned int lru_mask)
720
{
721
	unsigned long nr = 0;
722 723
	int zid;

724
	VM_BUG_ON((unsigned)nid >= nr_node_ids);
725

726 727 728 729 730 731 732 733 734 735 736 737
	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		struct mem_cgroup_per_zone *mz;
		enum lru_list lru;

		for_each_lru(lru) {
			if (!(BIT(lru) & lru_mask))
				continue;
			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
			nr += mz->lru_size[lru];
		}
	}
	return nr;
738
}
739

740
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
741
			unsigned int lru_mask)
742
{
743
	unsigned long nr = 0;
744
	int nid;
745

746
	for_each_node_state(nid, N_MEMORY)
747 748
		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
	return nr;
749 750
}

751 752
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
753 754 755
{
	unsigned long val, next;

756
	val = __this_cpu_read(memcg->stat->nr_page_events);
757
	next = __this_cpu_read(memcg->stat->targets[target]);
758
	/* from time_after() in jiffies.h */
759 760 761 762 763
	if ((long)next - (long)val < 0) {
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
764 765 766
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
767 768 769 770 771 772 773 774
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
		__this_cpu_write(memcg->stat->targets[target], next);
		return true;
775
	}
776
	return false;
777 778 779 780 781 782
}

/*
 * Check events in order.
 *
 */
783
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
784 785
{
	/* threshold event is triggered in finer grain than soft limit */
786 787
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
788
		bool do_softlimit;
789
		bool do_numainfo __maybe_unused;
790

791 792
		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
793 794 795 796
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
797
		mem_cgroup_threshold(memcg);
798 799
		if (unlikely(do_softlimit))
			mem_cgroup_update_tree(memcg, page);
800
#if MAX_NUMNODES > 1
801
		if (unlikely(do_numainfo))
802
			atomic_inc(&memcg->numainfo_events);
803
#endif
804
	}
805 806
}

807
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
808
{
809 810 811 812 813 814 815 816
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

817
	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
818
}
M
Michal Hocko 已提交
819
EXPORT_SYMBOL(mem_cgroup_from_task);
820

821
static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
822
{
823
	struct mem_cgroup *memcg = NULL;
824

825 826
	rcu_read_lock();
	do {
827 828 829 830 831 832
		/*
		 * Page cache insertions can happen withou an
		 * actual mm context, e.g. during disk probing
		 * on boot, loopback IO, acct() writes etc.
		 */
		if (unlikely(!mm))
833
			memcg = root_mem_cgroup;
834 835 836 837 838
		else {
			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
			if (unlikely(!memcg))
				memcg = root_mem_cgroup;
		}
839
	} while (!css_tryget_online(&memcg->css));
840
	rcu_read_unlock();
841
	return memcg;
842 843
}

844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
 * Reclaimers can specify a zone and a priority level in @reclaim to
 * divide up the memcgs in the hierarchy among all concurrent
 * reclaimers operating on the same zone and priority.
 */
861
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
862
				   struct mem_cgroup *prev,
863
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
864
{
M
Michal Hocko 已提交
865
	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
866
	struct cgroup_subsys_state *css = NULL;
867
	struct mem_cgroup *memcg = NULL;
868
	struct mem_cgroup *pos = NULL;
869

870 871
	if (mem_cgroup_disabled())
		return NULL;
872

873 874
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
875

876
	if (prev && !reclaim)
877
		pos = prev;
K
KAMEZAWA Hiroyuki 已提交
878

879 880
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
881
			goto out;
882
		return root;
883
	}
K
KAMEZAWA Hiroyuki 已提交
884

885
	rcu_read_lock();
M
Michal Hocko 已提交
886

887 888 889 890 891 892 893 894 895 896
	if (reclaim) {
		struct mem_cgroup_per_zone *mz;

		mz = mem_cgroup_zone_zoneinfo(root, reclaim->zone);
		iter = &mz->iter[reclaim->priority];

		if (prev && reclaim->generation != iter->generation)
			goto out_unlock;

		do {
897
			pos = READ_ONCE(iter->position);
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
			/*
			 * A racing update may change the position and
			 * put the last reference, hence css_tryget(),
			 * or retry to see the updated position.
			 */
		} while (pos && !css_tryget(&pos->css));
	}

	if (pos)
		css = &pos->css;

	for (;;) {
		css = css_next_descendant_pre(css, &root->css);
		if (!css) {
			/*
			 * Reclaimers share the hierarchy walk, and a
			 * new one might jump in right at the end of
			 * the hierarchy - make sure they see at least
			 * one group and restart from the beginning.
			 */
			if (!prev)
				continue;
			break;
921
		}
K
KAMEZAWA Hiroyuki 已提交
922

923 924 925 926 927 928
		/*
		 * Verify the css and acquire a reference.  The root
		 * is provided by the caller, so we know it's alive
		 * and kicking, and don't take an extra reference.
		 */
		memcg = mem_cgroup_from_css(css);
K
KAMEZAWA Hiroyuki 已提交
929

930 931
		if (css == &root->css)
			break;
K
KAMEZAWA Hiroyuki 已提交
932

933
		if (css_tryget(css)) {
934 935 936 937 938 939 940
			/*
			 * Make sure the memcg is initialized:
			 * mem_cgroup_css_online() orders the the
			 * initialization against setting the flag.
			 */
			if (smp_load_acquire(&memcg->initialized))
				break;
941

942
			css_put(css);
943
		}
944

945
		memcg = NULL;
946
	}
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966

	if (reclaim) {
		if (cmpxchg(&iter->position, pos, memcg) == pos) {
			if (memcg)
				css_get(&memcg->css);
			if (pos)
				css_put(&pos->css);
		}

		/*
		 * pairs with css_tryget when dereferencing iter->position
		 * above.
		 */
		if (pos)
			css_put(&pos->css);

		if (!memcg)
			iter->generation++;
		else if (!prev)
			reclaim->generation = iter->generation;
967
	}
968

969 970
out_unlock:
	rcu_read_unlock();
971
out:
972 973 974
	if (prev && prev != root)
		css_put(&prev->css);

975
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
976
}
K
KAMEZAWA Hiroyuki 已提交
977

978 979 980 981 982 983 984
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
985 986 987 988 989 990
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
991

992 993 994 995 996 997
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
998
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
999
	     iter != NULL;				\
1000
	     iter = mem_cgroup_iter(root, iter, NULL))
1001

1002
#define for_each_mem_cgroup(iter)			\
1003
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
1004
	     iter != NULL;				\
1005
	     iter = mem_cgroup_iter(NULL, iter, NULL))
K
KAMEZAWA Hiroyuki 已提交
1006

1007 1008 1009
/**
 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
 * @zone: zone of the wanted lruvec
1010
 * @memcg: memcg of the wanted lruvec
1011 1012 1013 1014 1015 1016 1017 1018 1019
 *
 * Returns the lru list vector holding pages for the given @zone and
 * @mem.  This can be the global zone lruvec, if the memory controller
 * is disabled.
 */
struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
				      struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_zone *mz;
1020
	struct lruvec *lruvec;
1021

1022 1023 1024 1025
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1026

1027
	mz = mem_cgroup_zone_zoneinfo(memcg, zone);
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
1038 1039 1040
}

/**
1041
 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1042
 * @page: the page
1043
 * @zone: zone of the page
1044 1045 1046 1047
 *
 * This function is only safe when following the LRU page isolation
 * and putback protocol: the LRU lock must be held, and the page must
 * either be PageLRU() or the caller must have isolated/allocated it.
1048
 */
1049
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
K
KAMEZAWA Hiroyuki 已提交
1050 1051
{
	struct mem_cgroup_per_zone *mz;
1052
	struct mem_cgroup *memcg;
1053
	struct lruvec *lruvec;
1054

1055 1056 1057 1058
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1059

1060
	memcg = page->mem_cgroup;
1061
	/*
1062
	 * Swapcache readahead pages are added to the LRU - and
1063
	 * possibly migrated - before they are charged.
1064
	 */
1065 1066
	if (!memcg)
		memcg = root_mem_cgroup;
1067

1068
	mz = mem_cgroup_page_zoneinfo(memcg, page);
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1079
}
1080

1081
/**
1082 1083 1084 1085
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
 * @nr_pages: positive when adding or negative when removing
1086
 *
1087 1088
 * This function must be called when a page is added to or removed from an
 * lru list.
1089
 */
1090 1091
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
				int nr_pages)
1092 1093
{
	struct mem_cgroup_per_zone *mz;
1094
	unsigned long *lru_size;
1095 1096 1097 1098

	if (mem_cgroup_disabled())
		return;

1099 1100 1101 1102
	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	lru_size = mz->lru_size + lru;
	*lru_size += nr_pages;
	VM_BUG_ON((long)(*lru_size) < 0);
K
KAMEZAWA Hiroyuki 已提交
1103
}
1104

1105
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1106
{
1107
	struct mem_cgroup *task_memcg;
1108
	struct task_struct *p;
1109
	bool ret;
1110

1111
	p = find_lock_task_mm(task);
1112
	if (p) {
1113
		task_memcg = get_mem_cgroup_from_mm(p->mm);
1114 1115 1116 1117 1118 1119 1120
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
1121
		rcu_read_lock();
1122 1123
		task_memcg = mem_cgroup_from_task(task);
		css_get(&task_memcg->css);
1124
		rcu_read_unlock();
1125
	}
1126 1127
	ret = mem_cgroup_is_descendant(task_memcg, memcg);
	css_put(&task_memcg->css);
1128 1129 1130
	return ret;
}

1131
#define mem_cgroup_from_counter(counter, member)	\
1132 1133
	container_of(counter, struct mem_cgroup, member)

1134
/**
1135
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1136
 * @memcg: the memory cgroup
1137
 *
1138
 * Returns the maximum amount of memory @mem can be charged with, in
1139
 * pages.
1140
 */
1141
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1142
{
1143 1144 1145
	unsigned long margin = 0;
	unsigned long count;
	unsigned long limit;
1146

1147
	count = page_counter_read(&memcg->memory);
1148
	limit = READ_ONCE(memcg->memory.limit);
1149 1150 1151 1152 1153
	if (count < limit)
		margin = limit - count;

	if (do_swap_account) {
		count = page_counter_read(&memcg->memsw);
1154
		limit = READ_ONCE(memcg->memsw.limit);
1155 1156 1157 1158 1159
		if (count <= limit)
			margin = min(margin, limit - count);
	}

	return margin;
1160 1161
}

1162
/*
Q
Qiang Huang 已提交
1163
 * A routine for checking "mem" is under move_account() or not.
1164
 *
Q
Qiang Huang 已提交
1165 1166 1167
 * Checking a cgroup is mc.from or mc.to or under hierarchy of
 * moving cgroups. This is for waiting at high-memory pressure
 * caused by "move".
1168
 */
1169
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1170
{
1171 1172
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1173
	bool ret = false;
1174 1175 1176 1177 1178 1179 1180 1181 1182
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1183

1184 1185
	ret = mem_cgroup_is_descendant(from, memcg) ||
		mem_cgroup_is_descendant(to, memcg);
1186 1187
unlock:
	spin_unlock(&mc.lock);
1188 1189 1190
	return ret;
}

1191
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1192 1193
{
	if (mc.moving_task && current != mc.moving_task) {
1194
		if (mem_cgroup_under_move(memcg)) {
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1207
#define K(x) ((x) << (PAGE_SHIFT-10))
1208
/**
1209
 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1210 1211 1212 1213 1214 1215 1216 1217
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
T
Tejun Heo 已提交
1218
	/* oom_info_lock ensures that parallel ooms do not interleave */
1219
	static DEFINE_MUTEX(oom_info_lock);
1220 1221
	struct mem_cgroup *iter;
	unsigned int i;
1222

1223
	mutex_lock(&oom_info_lock);
1224 1225
	rcu_read_lock();

1226 1227 1228 1229 1230 1231 1232 1233
	if (p) {
		pr_info("Task in ");
		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
		pr_cont(" killed as a result of limit of ");
	} else {
		pr_info("Memory limit reached of cgroup ");
	}

T
Tejun Heo 已提交
1234
	pr_cont_cgroup_path(memcg->css.cgroup);
1235
	pr_cont("\n");
1236 1237 1238

	rcu_read_unlock();

1239 1240 1241 1242 1243 1244 1245 1246 1247
	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memory)),
		K((u64)memcg->memory.limit), memcg->memory.failcnt);
	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memsw)),
		K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->kmem)),
		K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1248 1249

	for_each_mem_cgroup_tree(iter, memcg) {
T
Tejun Heo 已提交
1250 1251
		pr_info("Memory cgroup stats for ");
		pr_cont_cgroup_path(iter->css.cgroup);
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
		pr_cont(":");

		for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
			if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
				continue;
			pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
				K(mem_cgroup_read_stat(iter, i)));
		}

		for (i = 0; i < NR_LRU_LISTS; i++)
			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));

		pr_cont("\n");
	}
1267
	mutex_unlock(&oom_info_lock);
1268 1269
}

1270 1271 1272 1273
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1274
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1275 1276
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1277 1278
	struct mem_cgroup *iter;

1279
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1280
		num++;
1281 1282 1283
	return num;
}

D
David Rientjes 已提交
1284 1285 1286
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1287
static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1288
{
1289
	unsigned long limit;
1290

1291
	limit = memcg->memory.limit;
1292
	if (mem_cgroup_swappiness(memcg)) {
1293
		unsigned long memsw_limit;
1294

1295 1296
		memsw_limit = memcg->memsw.limit;
		limit = min(limit + total_swap_pages, memsw_limit);
1297 1298
	}
	return limit;
D
David Rientjes 已提交
1299 1300
}

1301 1302
static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
				     int order)
1303
{
1304 1305 1306 1307 1308 1309
	struct oom_control oc = {
		.zonelist = NULL,
		.nodemask = NULL,
		.gfp_mask = gfp_mask,
		.order = order,
	};
1310 1311 1312 1313 1314 1315
	struct mem_cgroup *iter;
	unsigned long chosen_points = 0;
	unsigned long totalpages;
	unsigned int points = 0;
	struct task_struct *chosen = NULL;

1316 1317
	mutex_lock(&oom_lock);

1318
	/*
1319 1320 1321
	 * If current has a pending SIGKILL or is exiting, then automatically
	 * select it.  The goal is to allow it to allocate so that it may
	 * quickly exit and free its memory.
1322
	 */
1323
	if (fatal_signal_pending(current) || task_will_free_mem(current)) {
1324
		mark_oom_victim(current);
1325
		goto unlock;
1326 1327
	}

1328
	check_panic_on_oom(&oc, CONSTRAINT_MEMCG, memcg);
1329
	totalpages = mem_cgroup_get_limit(memcg) ? : 1;
1330
	for_each_mem_cgroup_tree(iter, memcg) {
1331
		struct css_task_iter it;
1332 1333
		struct task_struct *task;

1334 1335
		css_task_iter_start(&iter->css, &it);
		while ((task = css_task_iter_next(&it))) {
1336
			switch (oom_scan_process_thread(&oc, task, totalpages)) {
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
			case OOM_SCAN_SELECT:
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = ULONG_MAX;
				get_task_struct(chosen);
				/* fall through */
			case OOM_SCAN_CONTINUE:
				continue;
			case OOM_SCAN_ABORT:
1347
				css_task_iter_end(&it);
1348 1349 1350
				mem_cgroup_iter_break(memcg, iter);
				if (chosen)
					put_task_struct(chosen);
1351
				goto unlock;
1352 1353 1354 1355
			case OOM_SCAN_OK:
				break;
			};
			points = oom_badness(task, memcg, NULL, totalpages);
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
			if (!points || points < chosen_points)
				continue;
			/* Prefer thread group leaders for display purposes */
			if (points == chosen_points &&
			    thread_group_leader(chosen))
				continue;

			if (chosen)
				put_task_struct(chosen);
			chosen = task;
			chosen_points = points;
			get_task_struct(chosen);
1368
		}
1369
		css_task_iter_end(&it);
1370 1371
	}

1372 1373
	if (chosen) {
		points = chosen_points * 1000 / totalpages;
1374 1375
		oom_kill_process(&oc, chosen, points, totalpages, memcg,
				 "Memory cgroup out of memory");
1376 1377 1378
	}
unlock:
	mutex_unlock(&oom_lock);
1379 1380
}

1381 1382
#if MAX_NUMNODES > 1

1383 1384
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1385
 * @memcg: the target memcg
1386 1387 1388 1389 1390 1391 1392
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1393
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1394 1395
		int nid, bool noswap)
{
1396
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1397 1398 1399
		return true;
	if (noswap || !total_swap_pages)
		return false;
1400
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1401 1402 1403 1404
		return true;
	return false;

}
1405 1406 1407 1408 1409 1410 1411

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1412
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1413 1414
{
	int nid;
1415 1416 1417 1418
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1419
	if (!atomic_read(&memcg->numainfo_events))
1420
		return;
1421
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1422 1423 1424
		return;

	/* make a nodemask where this memcg uses memory from */
1425
	memcg->scan_nodes = node_states[N_MEMORY];
1426

1427
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1428

1429 1430
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1431
	}
1432

1433 1434
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1449
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1450 1451 1452
{
	int node;

1453 1454
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1455

1456
	node = next_node(node, memcg->scan_nodes);
1457
	if (node == MAX_NUMNODES)
1458
		node = first_node(memcg->scan_nodes);
1459 1460 1461 1462 1463 1464 1465 1466 1467
	/*
	 * We call this when we hit limit, not when pages are added to LRU.
	 * No LRU may hold pages because all pages are UNEVICTABLE or
	 * memcg is too small and all pages are not on LRU. In that case,
	 * we use curret node.
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1468
	memcg->last_scanned_node = node;
1469 1470 1471
	return node;
}
#else
1472
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1473 1474 1475 1476 1477
{
	return 0;
}
#endif

1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
				   struct zone *zone,
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
{
	struct mem_cgroup *victim = NULL;
	int total = 0;
	int loop = 0;
	unsigned long excess;
	unsigned long nr_scanned;
	struct mem_cgroup_reclaim_cookie reclaim = {
		.zone = zone,
		.priority = 0,
	};

1493
	excess = soft_limit_excess(root_memcg);
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521

	while (1) {
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
		if (!victim) {
			loop++;
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!total)
					break;
				/*
				 * We want to do more targeted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
					break;
			}
			continue;
		}
		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
						     zone, &nr_scanned);
		*total_scanned += nr_scanned;
1522
		if (!soft_limit_excess(root_memcg))
1523
			break;
1524
	}
1525 1526
	mem_cgroup_iter_break(root_memcg, victim);
	return total;
1527 1528
}

1529 1530 1531 1532 1533 1534
#ifdef CONFIG_LOCKDEP
static struct lockdep_map memcg_oom_lock_dep_map = {
	.name = "memcg_oom_lock",
};
#endif

1535 1536
static DEFINE_SPINLOCK(memcg_oom_lock);

K
KAMEZAWA Hiroyuki 已提交
1537 1538 1539 1540
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
 */
1541
static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1542
{
1543
	struct mem_cgroup *iter, *failed = NULL;
1544

1545 1546
	spin_lock(&memcg_oom_lock);

1547
	for_each_mem_cgroup_tree(iter, memcg) {
1548
		if (iter->oom_lock) {
1549 1550 1551 1552 1553
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1554 1555
			mem_cgroup_iter_break(memcg, iter);
			break;
1556 1557
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1558
	}
K
KAMEZAWA Hiroyuki 已提交
1559

1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	if (failed) {
		/*
		 * OK, we failed to lock the whole subtree so we have
		 * to clean up what we set up to the failing subtree
		 */
		for_each_mem_cgroup_tree(iter, memcg) {
			if (iter == failed) {
				mem_cgroup_iter_break(memcg, iter);
				break;
			}
			iter->oom_lock = false;
1571
		}
1572 1573
	} else
		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1574 1575 1576 1577

	spin_unlock(&memcg_oom_lock);

	return !failed;
1578
}
1579

1580
static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1581
{
K
KAMEZAWA Hiroyuki 已提交
1582 1583
	struct mem_cgroup *iter;

1584
	spin_lock(&memcg_oom_lock);
1585
	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1586
	for_each_mem_cgroup_tree(iter, memcg)
1587
		iter->oom_lock = false;
1588
	spin_unlock(&memcg_oom_lock);
1589 1590
}

1591
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1592 1593 1594
{
	struct mem_cgroup *iter;

1595
	spin_lock(&memcg_oom_lock);
1596
	for_each_mem_cgroup_tree(iter, memcg)
1597 1598
		iter->under_oom++;
	spin_unlock(&memcg_oom_lock);
1599 1600
}

1601
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1602 1603 1604
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1605 1606
	/*
	 * When a new child is created while the hierarchy is under oom,
1607
	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
K
KAMEZAWA Hiroyuki 已提交
1608
	 */
1609
	spin_lock(&memcg_oom_lock);
1610
	for_each_mem_cgroup_tree(iter, memcg)
1611 1612 1613
		if (iter->under_oom > 0)
			iter->under_oom--;
	spin_unlock(&memcg_oom_lock);
1614 1615
}

K
KAMEZAWA Hiroyuki 已提交
1616 1617
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1618
struct oom_wait_info {
1619
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
1620 1621 1622 1623 1624 1625
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
1626 1627
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1628 1629 1630
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1631
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
1632

1633 1634
	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
K
KAMEZAWA Hiroyuki 已提交
1635 1636 1637 1638
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1639
static void memcg_oom_recover(struct mem_cgroup *memcg)
1640
{
1641 1642 1643 1644 1645 1646 1647 1648 1649
	/*
	 * For the following lockless ->under_oom test, the only required
	 * guarantee is that it must see the state asserted by an OOM when
	 * this function is called as a result of userland actions
	 * triggered by the notification of the OOM.  This is trivially
	 * achieved by invoking mem_cgroup_mark_under_oom() before
	 * triggering notification.
	 */
	if (memcg && memcg->under_oom)
1650
		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1651 1652
}

1653
static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1654
{
1655 1656
	if (!current->memcg_oom.may_oom)
		return;
K
KAMEZAWA Hiroyuki 已提交
1657
	/*
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
	 * We are in the middle of the charge context here, so we
	 * don't want to block when potentially sitting on a callstack
	 * that holds all kinds of filesystem and mm locks.
	 *
	 * Also, the caller may handle a failed allocation gracefully
	 * (like optional page cache readahead) and so an OOM killer
	 * invocation might not even be necessary.
	 *
	 * That's why we don't do anything here except remember the
	 * OOM context and then deal with it at the end of the page
	 * fault when the stack is unwound, the locks are released,
	 * and when we know whether the fault was overall successful.
K
KAMEZAWA Hiroyuki 已提交
1670
	 */
1671 1672 1673 1674
	css_get(&memcg->css);
	current->memcg_oom.memcg = memcg;
	current->memcg_oom.gfp_mask = mask;
	current->memcg_oom.order = order;
1675 1676 1677 1678
}

/**
 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1679
 * @handle: actually kill/wait or just clean up the OOM state
1680
 *
1681 1682
 * This has to be called at the end of a page fault if the memcg OOM
 * handler was enabled.
1683
 *
1684
 * Memcg supports userspace OOM handling where failed allocations must
1685 1686 1687 1688
 * sleep on a waitqueue until the userspace task resolves the
 * situation.  Sleeping directly in the charge context with all kinds
 * of locks held is not a good idea, instead we remember an OOM state
 * in the task and mem_cgroup_oom_synchronize() has to be called at
1689
 * the end of the page fault to complete the OOM handling.
1690 1691
 *
 * Returns %true if an ongoing memcg OOM situation was detected and
1692
 * completed, %false otherwise.
1693
 */
1694
bool mem_cgroup_oom_synchronize(bool handle)
1695
{
1696
	struct mem_cgroup *memcg = current->memcg_oom.memcg;
1697
	struct oom_wait_info owait;
1698
	bool locked;
1699 1700 1701

	/* OOM is global, do not handle */
	if (!memcg)
1702
		return false;
1703

1704
	if (!handle || oom_killer_disabled)
1705
		goto cleanup;
1706 1707 1708 1709 1710 1711

	owait.memcg = memcg;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
K
KAMEZAWA Hiroyuki 已提交
1712

1713
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	if (locked && !memcg->oom_kill_disable) {
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
		mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
					 current->memcg_oom.order);
	} else {
1727
		schedule();
1728 1729 1730 1731 1732
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
	}

	if (locked) {
1733 1734 1735 1736 1737 1738 1739 1740
		mem_cgroup_oom_unlock(memcg);
		/*
		 * There is no guarantee that an OOM-lock contender
		 * sees the wakeups triggered by the OOM kill
		 * uncharges.  Wake any sleepers explicitely.
		 */
		memcg_oom_recover(memcg);
	}
1741 1742
cleanup:
	current->memcg_oom.memcg = NULL;
1743
	css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
1744
	return true;
1745 1746
}

1747 1748 1749
/**
 * mem_cgroup_begin_page_stat - begin a page state statistics transaction
 * @page: page that is going to change accounted state
1750
 *
1751 1752 1753
 * This function must mark the beginning of an accounted page state
 * change to prevent double accounting when the page is concurrently
 * being moved to another memcg:
1754
 *
1755
 *   memcg = mem_cgroup_begin_page_stat(page);
1756 1757
 *   if (TestClearPageState(page))
 *     mem_cgroup_update_page_stat(memcg, state, -1);
1758
 *   mem_cgroup_end_page_stat(memcg);
1759
 */
1760
struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)
1761 1762
{
	struct mem_cgroup *memcg;
1763
	unsigned long flags;
1764

1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776
	/*
	 * The RCU lock is held throughout the transaction.  The fast
	 * path can get away without acquiring the memcg->move_lock
	 * because page moving starts with an RCU grace period.
	 *
	 * The RCU lock also protects the memcg from being freed when
	 * the page state that is going to change is the only thing
	 * preventing the page from being uncharged.
	 * E.g. end-writeback clearing PageWriteback(), which allows
	 * migration to go ahead and uncharge the page before the
	 * account transaction might be complete.
	 */
1777 1778 1779 1780
	rcu_read_lock();

	if (mem_cgroup_disabled())
		return NULL;
1781
again:
1782
	memcg = page->mem_cgroup;
1783
	if (unlikely(!memcg))
1784 1785
		return NULL;

Q
Qiang Huang 已提交
1786
	if (atomic_read(&memcg->moving_account) <= 0)
1787
		return memcg;
1788

1789
	spin_lock_irqsave(&memcg->move_lock, flags);
1790
	if (memcg != page->mem_cgroup) {
1791
		spin_unlock_irqrestore(&memcg->move_lock, flags);
1792 1793
		goto again;
	}
1794 1795 1796 1797 1798 1799 1800 1801

	/*
	 * When charge migration first begins, we can have locked and
	 * unlocked page stat updates happening concurrently.  Track
	 * the task who has the lock for mem_cgroup_end_page_stat().
	 */
	memcg->move_lock_task = current;
	memcg->move_lock_flags = flags;
1802 1803

	return memcg;
1804
}
1805
EXPORT_SYMBOL(mem_cgroup_begin_page_stat);
1806

1807 1808 1809 1810
/**
 * mem_cgroup_end_page_stat - finish a page state statistics transaction
 * @memcg: the memcg that was accounted against
 */
1811
void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)
1812
{
1813 1814 1815 1816 1817 1818 1819 1820
	if (memcg && memcg->move_lock_task == current) {
		unsigned long flags = memcg->move_lock_flags;

		memcg->move_lock_task = NULL;
		memcg->move_lock_flags = 0;

		spin_unlock_irqrestore(&memcg->move_lock, flags);
	}
1821

1822
	rcu_read_unlock();
1823
}
1824
EXPORT_SYMBOL(mem_cgroup_end_page_stat);
1825

1826 1827 1828 1829
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
1830
#define CHARGE_BATCH	32U
1831 1832
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
1833
	unsigned int nr_pages;
1834
	struct work_struct work;
1835
	unsigned long flags;
1836
#define FLUSHING_CACHED_CHARGE	0
1837 1838
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1839
static DEFINE_MUTEX(percpu_charge_mutex);
1840

1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
1851
 */
1852
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1853 1854
{
	struct memcg_stock_pcp *stock;
1855
	bool ret = false;
1856

1857
	if (nr_pages > CHARGE_BATCH)
1858
		return ret;
1859

1860
	stock = &get_cpu_var(memcg_stock);
1861
	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1862
		stock->nr_pages -= nr_pages;
1863 1864
		ret = true;
	}
1865 1866 1867 1868 1869
	put_cpu_var(memcg_stock);
	return ret;
}

/*
1870
 * Returns stocks cached in percpu and reset cached information.
1871 1872 1873 1874 1875
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

1876
	if (stock->nr_pages) {
1877
		page_counter_uncharge(&old->memory, stock->nr_pages);
1878
		if (do_swap_account)
1879
			page_counter_uncharge(&old->memsw, stock->nr_pages);
1880
		css_put_many(&old->css, stock->nr_pages);
1881
		stock->nr_pages = 0;
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
1892
	struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock);
1893
	drain_stock(stock);
1894
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1895 1896 1897
}

/*
1898
 * Cache charges(val) to local per_cpu area.
1899
 * This will be consumed by consume_stock() function, later.
1900
 */
1901
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1902 1903 1904
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

1905
	if (stock->cached != memcg) { /* reset if necessary */
1906
		drain_stock(stock);
1907
		stock->cached = memcg;
1908
	}
1909
	stock->nr_pages += nr_pages;
1910 1911 1912 1913
	put_cpu_var(memcg_stock);
}

/*
1914
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
1915
 * of the hierarchy under it.
1916
 */
1917
static void drain_all_stock(struct mem_cgroup *root_memcg)
1918
{
1919
	int cpu, curcpu;
1920

1921 1922 1923
	/* If someone's already draining, avoid adding running more workers. */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
1924 1925
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
1926
	curcpu = get_cpu();
1927 1928
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1929
		struct mem_cgroup *memcg;
1930

1931 1932
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
1933
			continue;
1934
		if (!mem_cgroup_is_descendant(memcg, root_memcg))
1935
			continue;
1936 1937 1938 1939 1940 1941
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
1942
	}
1943
	put_cpu();
A
Andrew Morton 已提交
1944
	put_online_cpus();
1945
	mutex_unlock(&percpu_charge_mutex);
1946 1947
}

1948
static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
1949 1950 1951 1952 1953 1954
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;

1955
	if (action == CPU_ONLINE)
1956 1957
		return NOTIFY_OK;

1958
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
1959
		return NOTIFY_OK;
1960

1961 1962 1963 1964 1965
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

1966 1967
static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
		      unsigned int nr_pages)
1968
{
1969
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
1970
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1971
	struct mem_cgroup *mem_over_limit;
1972
	struct page_counter *counter;
1973
	unsigned long nr_reclaimed;
1974 1975
	bool may_swap = true;
	bool drained = false;
1976
	int ret = 0;
1977

1978 1979
	if (mem_cgroup_is_root(memcg))
		goto done;
1980
retry:
1981 1982
	if (consume_stock(memcg, nr_pages))
		goto done;
1983

1984
	if (!do_swap_account ||
1985 1986
	    !page_counter_try_charge(&memcg->memsw, batch, &counter)) {
		if (!page_counter_try_charge(&memcg->memory, batch, &counter))
1987
			goto done_restock;
1988
		if (do_swap_account)
1989 1990
			page_counter_uncharge(&memcg->memsw, batch);
		mem_over_limit = mem_cgroup_from_counter(counter, memory);
1991
	} else {
1992
		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1993
		may_swap = false;
1994
	}
1995

1996 1997 1998 1999
	if (batch > nr_pages) {
		batch = nr_pages;
		goto retry;
	}
2000

2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
	/*
	 * Unlike in global OOM situations, memcg is not in a physical
	 * memory shortage.  Allow dying and OOM-killed tasks to
	 * bypass the last charges so that they can exit quickly and
	 * free their memory.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE) ||
		     fatal_signal_pending(current) ||
		     current->flags & PF_EXITING))
		goto bypass;

	if (unlikely(task_in_memcg_oom(current)))
		goto nomem;

2015 2016
	if (!(gfp_mask & __GFP_WAIT))
		goto nomem;
2017

2018 2019
	mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);

2020 2021
	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
						    gfp_mask, may_swap);
2022

2023
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2024
		goto retry;
2025

2026
	if (!drained) {
2027
		drain_all_stock(mem_over_limit);
2028 2029 2030 2031
		drained = true;
		goto retry;
	}

2032 2033
	if (gfp_mask & __GFP_NORETRY)
		goto nomem;
2034 2035 2036 2037 2038 2039 2040 2041 2042
	/*
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
	 */
2043
	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2044 2045 2046 2047 2048 2049 2050 2051
		goto retry;
	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		goto retry;

2052 2053 2054
	if (nr_retries--)
		goto retry;

2055 2056 2057
	if (gfp_mask & __GFP_NOFAIL)
		goto bypass;

2058 2059 2060
	if (fatal_signal_pending(current))
		goto bypass;

2061 2062
	mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);

2063
	mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(nr_pages));
2064
nomem:
2065
	if (!(gfp_mask & __GFP_NOFAIL))
2066
		return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2067
bypass:
2068
	return -EINTR;
2069 2070

done_restock:
2071
	css_get_many(&memcg->css, batch);
2072 2073
	if (batch > nr_pages)
		refill_stock(memcg, batch - nr_pages);
2074 2075
	if (!(gfp_mask & __GFP_WAIT))
		goto done;
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
	/*
	 * If the hierarchy is above the normal consumption range,
	 * make the charging task trim their excess contribution.
	 */
	do {
		if (page_counter_read(&memcg->memory) <= memcg->high)
			continue;
		mem_cgroup_events(memcg, MEMCG_HIGH, 1);
		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
	} while ((memcg = parent_mem_cgroup(memcg)));
2086
done:
2087
	return ret;
2088
}
2089

2090
static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2091
{
2092 2093 2094
	if (mem_cgroup_is_root(memcg))
		return;

2095
	page_counter_uncharge(&memcg->memory, nr_pages);
2096
	if (do_swap_account)
2097
		page_counter_uncharge(&memcg->memsw, nr_pages);
2098

2099
	css_put_many(&memcg->css, nr_pages);
2100 2101
}

2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
static void lock_page_lru(struct page *page, int *isolated)
{
	struct zone *zone = page_zone(page);

	spin_lock_irq(&zone->lru_lock);
	if (PageLRU(page)) {
		struct lruvec *lruvec;

		lruvec = mem_cgroup_page_lruvec(page, zone);
		ClearPageLRU(page);
		del_page_from_lru_list(page, lruvec, page_lru(page));
		*isolated = 1;
	} else
		*isolated = 0;
}

static void unlock_page_lru(struct page *page, int isolated)
{
	struct zone *zone = page_zone(page);

	if (isolated) {
		struct lruvec *lruvec;

		lruvec = mem_cgroup_page_lruvec(page, zone);
		VM_BUG_ON_PAGE(PageLRU(page), page);
		SetPageLRU(page);
		add_page_to_lru_list(page, lruvec, page_lru(page));
	}
	spin_unlock_irq(&zone->lru_lock);
}

2133
static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2134
			  bool lrucare)
2135
{
2136
	int isolated;
2137

2138
	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2139 2140 2141 2142 2143

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
2144 2145
	if (lrucare)
		lock_page_lru(page, &isolated);
2146

2147 2148
	/*
	 * Nobody should be changing or seriously looking at
2149
	 * page->mem_cgroup at this point:
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
	 *
	 * - the page is uncharged
	 *
	 * - the page is off-LRU
	 *
	 * - an anonymous fault has exclusive page access, except for
	 *   a locked page table
	 *
	 * - a page cache insertion, a swapin fault, or a migration
	 *   have the page locked
	 */
2161
	page->mem_cgroup = memcg;
2162

2163 2164
	if (lrucare)
		unlock_page_lru(page, isolated);
2165
}
2166

2167
#ifdef CONFIG_MEMCG_KMEM
2168 2169
int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp,
		      unsigned long nr_pages)
2170
{
2171
	struct page_counter *counter;
2172 2173
	int ret = 0;

2174 2175
	ret = page_counter_try_charge(&memcg->kmem, nr_pages, &counter);
	if (ret < 0)
2176 2177
		return ret;

2178
	ret = try_charge(memcg, gfp, nr_pages);
2179 2180
	if (ret == -EINTR)  {
		/*
2181 2182 2183 2184 2185 2186
		 * try_charge() chose to bypass to root due to OOM kill or
		 * fatal signal.  Since our only options are to either fail
		 * the allocation or charge it to this cgroup, do it as a
		 * temporary condition. But we can't fail. From a kmem/slab
		 * perspective, the cache has already been selected, by
		 * mem_cgroup_kmem_get_cache(), so it is too late to change
2187 2188 2189
		 * our minds.
		 *
		 * This condition will only trigger if the task entered
2190 2191 2192
		 * memcg_charge_kmem in a sane state, but was OOM-killed
		 * during try_charge() above. Tasks that were already dying
		 * when the allocation triggers should have been already
2193 2194
		 * directed to the root cgroup in memcontrol.h
		 */
2195
		page_counter_charge(&memcg->memory, nr_pages);
2196
		if (do_swap_account)
2197
			page_counter_charge(&memcg->memsw, nr_pages);
2198
		css_get_many(&memcg->css, nr_pages);
2199 2200
		ret = 0;
	} else if (ret)
2201
		page_counter_uncharge(&memcg->kmem, nr_pages);
2202 2203 2204 2205

	return ret;
}

2206
void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages)
2207
{
2208
	page_counter_uncharge(&memcg->memory, nr_pages);
2209
	if (do_swap_account)
2210
		page_counter_uncharge(&memcg->memsw, nr_pages);
2211

2212
	page_counter_uncharge(&memcg->kmem, nr_pages);
2213

2214
	css_put_many(&memcg->css, nr_pages);
2215 2216
}

2217
static int memcg_alloc_cache_id(void)
2218
{
2219 2220 2221
	int id, size;
	int err;

2222
	id = ida_simple_get(&memcg_cache_ida,
2223 2224 2225
			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (id < 0)
		return id;
2226

2227
	if (id < memcg_nr_cache_ids)
2228 2229 2230 2231 2232 2233
		return id;

	/*
	 * There's no space for the new id in memcg_caches arrays,
	 * so we have to grow them.
	 */
2234
	down_write(&memcg_cache_ids_sem);
2235 2236

	size = 2 * (id + 1);
2237 2238 2239 2240 2241
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

2242
	err = memcg_update_all_caches(size);
2243 2244
	if (!err)
		err = memcg_update_all_list_lrus(size);
2245 2246 2247 2248 2249
	if (!err)
		memcg_nr_cache_ids = size;

	up_write(&memcg_cache_ids_sem);

2250
	if (err) {
2251
		ida_simple_remove(&memcg_cache_ida, id);
2252 2253 2254 2255 2256 2257 2258
		return err;
	}
	return id;
}

static void memcg_free_cache_id(int id)
{
2259
	ida_simple_remove(&memcg_cache_ida, id);
2260 2261
}

2262
struct memcg_kmem_cache_create_work {
2263 2264 2265 2266 2267
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

2268
static void memcg_kmem_cache_create_func(struct work_struct *w)
2269
{
2270 2271
	struct memcg_kmem_cache_create_work *cw =
		container_of(w, struct memcg_kmem_cache_create_work, work);
2272 2273
	struct mem_cgroup *memcg = cw->memcg;
	struct kmem_cache *cachep = cw->cachep;
2274

2275
	memcg_create_kmem_cache(memcg, cachep);
2276

2277
	css_put(&memcg->css);
2278 2279 2280 2281 2282 2283
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 */
2284 2285
static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					       struct kmem_cache *cachep)
2286
{
2287
	struct memcg_kmem_cache_create_work *cw;
2288

2289
	cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2290
	if (!cw)
2291
		return;
2292 2293

	css_get(&memcg->css);
2294 2295 2296

	cw->memcg = memcg;
	cw->cachep = cachep;
2297
	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2298 2299 2300 2301

	schedule_work(&cw->work);
}

2302 2303
static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					     struct kmem_cache *cachep)
2304 2305 2306 2307
{
	/*
	 * We need to stop accounting when we kmalloc, because if the
	 * corresponding kmalloc cache is not yet created, the first allocation
2308
	 * in __memcg_schedule_kmem_cache_create will recurse.
2309 2310 2311 2312 2313 2314 2315
	 *
	 * However, it is better to enclose the whole function. Depending on
	 * the debugging options enabled, INIT_WORK(), for instance, can
	 * trigger an allocation. This too, will make us recurse. Because at
	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
	 * the safest choice is to do it like this, wrapping the whole function.
	 */
2316
	current->memcg_kmem_skip_account = 1;
2317
	__memcg_schedule_kmem_cache_create(memcg, cachep);
2318
	current->memcg_kmem_skip_account = 0;
2319
}
2320

2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
/*
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
 * If the cache does not exist yet, if we are the first user of it,
 * we either create it immediately, if possible, or create it asynchronously
 * in a workqueue.
 * In the latter case, we will let the current allocation go through with
 * the original cache.
 *
 * Can't be called in interrupt context or from kernel threads.
 * This function needs to be called with rcu_read_lock() held.
 */
2334
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep)
2335 2336
{
	struct mem_cgroup *memcg;
2337
	struct kmem_cache *memcg_cachep;
2338
	int kmemcg_id;
2339

2340
	VM_BUG_ON(!is_root_cache(cachep));
2341

2342
	if (current->memcg_kmem_skip_account)
2343 2344
		return cachep;

2345
	memcg = get_mem_cgroup_from_mm(current->mm);
2346
	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2347
	if (kmemcg_id < 0)
2348
		goto out;
2349

2350
	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2351 2352
	if (likely(memcg_cachep))
		return memcg_cachep;
2353 2354 2355 2356 2357 2358 2359 2360 2361

	/*
	 * If we are in a safe context (can wait, and not in interrupt
	 * context), we could be be predictable and return right away.
	 * This would guarantee that the allocation being performed
	 * already belongs in the new cache.
	 *
	 * However, there are some clashes that can arrive from locking.
	 * For instance, because we acquire the slab_mutex while doing
2362 2363 2364
	 * memcg_create_kmem_cache, this means no further allocation
	 * could happen with the slab_mutex held. So it's better to
	 * defer everything.
2365
	 */
2366
	memcg_schedule_kmem_cache_create(memcg, cachep);
2367
out:
2368
	css_put(&memcg->css);
2369
	return cachep;
2370 2371
}

2372 2373 2374
void __memcg_kmem_put_cache(struct kmem_cache *cachep)
{
	if (!is_root_cache(cachep))
2375
		css_put(&cachep->memcg_params.memcg->css);
2376 2377
}

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398
/*
 * We need to verify if the allocation against current->mm->owner's memcg is
 * possible for the given order. But the page is not allocated yet, so we'll
 * need a further commit step to do the final arrangements.
 *
 * It is possible for the task to switch cgroups in this mean time, so at
 * commit time, we can't rely on task conversion any longer.  We'll then use
 * the handle argument to return to the caller which cgroup we should commit
 * against. We could also return the memcg directly and avoid the pointer
 * passing, but a boolean return value gives better semantics considering
 * the compiled-out case as well.
 *
 * Returning true means the allocation is possible.
 */
bool
__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
{
	struct mem_cgroup *memcg;
	int ret;

	*_memcg = NULL;
2399

2400
	memcg = get_mem_cgroup_from_mm(current->mm);
2401

2402
	if (!memcg_kmem_is_active(memcg)) {
2403 2404 2405 2406
		css_put(&memcg->css);
		return true;
	}

2407
	ret = memcg_charge_kmem(memcg, gfp, 1 << order);
2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
	if (!ret)
		*_memcg = memcg;

	css_put(&memcg->css);
	return (ret == 0);
}

void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
			      int order)
{
	VM_BUG_ON(mem_cgroup_is_root(memcg));

	/* The page allocation failed. Revert */
	if (!page) {
2422
		memcg_uncharge_kmem(memcg, 1 << order);
2423 2424
		return;
	}
2425
	page->mem_cgroup = memcg;
2426 2427 2428 2429
}

void __memcg_kmem_uncharge_pages(struct page *page, int order)
{
2430
	struct mem_cgroup *memcg = page->mem_cgroup;
2431 2432 2433 2434

	if (!memcg)
		return;

2435
	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2436

2437
	memcg_uncharge_kmem(memcg, 1 << order);
2438
	page->mem_cgroup = NULL;
2439
}
2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450

struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr)
{
	struct mem_cgroup *memcg = NULL;
	struct kmem_cache *cachep;
	struct page *page;

	page = virt_to_head_page(ptr);
	if (PageSlab(page)) {
		cachep = page->slab_cache;
		if (!is_root_cache(cachep))
2451
			memcg = cachep->memcg_params.memcg;
2452 2453 2454 2455 2456 2457
	} else
		/* page allocated by alloc_kmem_pages */
		memcg = page->mem_cgroup;

	return memcg;
}
2458 2459
#endif /* CONFIG_MEMCG_KMEM */

2460 2461 2462 2463
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

/*
 * Because tail pages are not marked as "used", set it. We're under
2464 2465 2466
 * zone->lru_lock, 'splitting on pmd' and compound_lock.
 * charge/uncharge will be never happen and move_account() is done under
 * compound_lock(), so we don't have to take care of races.
2467
 */
2468
void mem_cgroup_split_huge_fixup(struct page *head)
2469
{
2470
	int i;
2471

2472 2473
	if (mem_cgroup_disabled())
		return;
2474

2475
	for (i = 1; i < HPAGE_PMD_NR; i++)
2476
		head[i].mem_cgroup = head->mem_cgroup;
2477

2478
	__this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2479
		       HPAGE_PMD_NR);
2480
}
2481
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2482

A
Andrew Morton 已提交
2483
#ifdef CONFIG_MEMCG_SWAP
2484 2485
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
					 bool charge)
K
KAMEZAWA Hiroyuki 已提交
2486
{
2487 2488
	int val = (charge) ? 1 : -1;
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
K
KAMEZAWA Hiroyuki 已提交
2489
}
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
2502
 * The caller must have charged to @to, IOW, called page_counter_charge() about
2503 2504 2505
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
2506
				struct mem_cgroup *from, struct mem_cgroup *to)
2507 2508 2509
{
	unsigned short old_id, new_id;

L
Li Zefan 已提交
2510 2511
	old_id = mem_cgroup_id(from);
	new_id = mem_cgroup_id(to);
2512 2513 2514

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
2515
		mem_cgroup_swap_statistics(to, true);
2516 2517 2518 2519 2520 2521
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2522
				struct mem_cgroup *from, struct mem_cgroup *to)
2523 2524 2525
{
	return -EINVAL;
}
2526
#endif
K
KAMEZAWA Hiroyuki 已提交
2527

2528
static DEFINE_MUTEX(memcg_limit_mutex);
2529

2530
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2531
				   unsigned long limit)
2532
{
2533 2534 2535
	unsigned long curusage;
	unsigned long oldusage;
	bool enlarge = false;
2536
	int retry_count;
2537
	int ret;
2538 2539 2540 2541 2542 2543

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
2544 2545
	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
		      mem_cgroup_count_children(memcg);
2546

2547
	oldusage = page_counter_read(&memcg->memory);
2548

2549
	do {
2550 2551 2552 2553
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2554 2555 2556 2557

		mutex_lock(&memcg_limit_mutex);
		if (limit > memcg->memsw.limit) {
			mutex_unlock(&memcg_limit_mutex);
2558
			ret = -EINVAL;
2559 2560
			break;
		}
2561 2562 2563 2564
		if (limit > memcg->memory.limit)
			enlarge = true;
		ret = page_counter_limit(&memcg->memory, limit);
		mutex_unlock(&memcg_limit_mutex);
2565 2566 2567 2568

		if (!ret)
			break;

2569 2570
		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);

2571
		curusage = page_counter_read(&memcg->memory);
2572
		/* Usage is reduced ? */
A
Andrew Morton 已提交
2573
		if (curusage >= oldusage)
2574 2575 2576
			retry_count--;
		else
			oldusage = curusage;
2577 2578
	} while (retry_count);

2579 2580
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
2581

2582 2583 2584
	return ret;
}

L
Li Zefan 已提交
2585
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2586
					 unsigned long limit)
2587
{
2588 2589 2590
	unsigned long curusage;
	unsigned long oldusage;
	bool enlarge = false;
2591
	int retry_count;
2592
	int ret;
2593

2594
	/* see mem_cgroup_resize_res_limit */
2595 2596 2597 2598 2599 2600
	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
		      mem_cgroup_count_children(memcg);

	oldusage = page_counter_read(&memcg->memsw);

	do {
2601 2602 2603 2604
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2605 2606 2607 2608

		mutex_lock(&memcg_limit_mutex);
		if (limit < memcg->memory.limit) {
			mutex_unlock(&memcg_limit_mutex);
2609 2610 2611
			ret = -EINVAL;
			break;
		}
2612 2613 2614 2615
		if (limit > memcg->memsw.limit)
			enlarge = true;
		ret = page_counter_limit(&memcg->memsw, limit);
		mutex_unlock(&memcg_limit_mutex);
2616 2617 2618 2619

		if (!ret)
			break;

2620 2621
		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);

2622
		curusage = page_counter_read(&memcg->memsw);
2623
		/* Usage is reduced ? */
2624
		if (curusage >= oldusage)
2625
			retry_count--;
2626 2627
		else
			oldusage = curusage;
2628 2629
	} while (retry_count);

2630 2631
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
2632

2633 2634 2635
	return ret;
}

2636 2637 2638 2639 2640 2641 2642 2643 2644
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
2645
	unsigned long excess;
2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669
	unsigned long nr_scanned;

	if (order > 0)
		return 0;

	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		nr_scanned = 0;
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
						    gfp_mask, &nr_scanned);
		nr_reclaimed += reclaimed;
		*total_scanned += nr_scanned;
2670
		spin_lock_irq(&mctz->lock);
2671
		__mem_cgroup_remove_exceeded(mz, mctz);
2672 2673 2674 2675 2676 2677

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
2678 2679 2680
		if (!reclaimed)
			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);

2681
		excess = soft_limit_excess(mz->memcg);
2682 2683 2684 2685 2686 2687 2688 2689 2690
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
		/* If excess == 0, no tree ops */
2691
		__mem_cgroup_insert_exceeded(mz, mctz, excess);
2692
		spin_unlock_irq(&mctz->lock);
2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709
		css_put(&mz->memcg->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->memcg->css);
	return nr_reclaimed;
}

2710 2711 2712 2713 2714 2715
/*
 * Test whether @memcg has children, dead or alive.  Note that this
 * function doesn't care whether @memcg has use_hierarchy enabled and
 * returns %true if there are child csses according to the cgroup
 * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
 */
2716 2717
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
2718 2719
	bool ret;

2720
	/*
2721 2722 2723 2724
	 * The lock does not prevent addition or deletion of children, but
	 * it prevents a new child from being initialized based on this
	 * parent in css_online(), so it's enough to decide whether
	 * hierarchically inherited attributes can still be changed or not.
2725
	 */
2726 2727 2728 2729 2730 2731
	lockdep_assert_held(&memcg_create_mutex);

	rcu_read_lock();
	ret = css_next_child(NULL, &memcg->css);
	rcu_read_unlock();
	return ret;
2732 2733
}

2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
/*
 * Reclaims as many pages from the given memcg as possible and moves
 * the rest to the parent.
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;

2744 2745
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
2746
	/* try to free all pages in this cgroup */
2747
	while (nr_retries && page_counter_read(&memcg->memory)) {
2748
		int progress;
2749

2750 2751 2752
		if (signal_pending(current))
			return -EINTR;

2753 2754
		progress = try_to_free_mem_cgroup_pages(memcg, 1,
							GFP_KERNEL, true);
2755
		if (!progress) {
2756
			nr_retries--;
2757
			/* maybe some writeback is necessary */
2758
			congestion_wait(BLK_RW_ASYNC, HZ/10);
2759
		}
2760 2761

	}
2762 2763

	return 0;
2764 2765
}

2766 2767 2768
static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
2769
{
2770
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2771

2772 2773
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
2774
	return mem_cgroup_force_empty(memcg) ?: nbytes;
2775 2776
}

2777 2778
static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
				     struct cftype *cft)
2779
{
2780
	return mem_cgroup_from_css(css)->use_hierarchy;
2781 2782
}

2783 2784
static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, u64 val)
2785 2786
{
	int retval = 0;
2787
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
2788
	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2789

2790
	mutex_lock(&memcg_create_mutex);
2791 2792 2793 2794

	if (memcg->use_hierarchy == val)
		goto out;

2795
	/*
2796
	 * If parent's use_hierarchy is set, we can't make any modifications
2797 2798 2799 2800 2801 2802
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
2803
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2804
				(val == 1 || val == 0)) {
2805
		if (!memcg_has_children(memcg))
2806
			memcg->use_hierarchy = val;
2807 2808 2809 2810
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
2811 2812

out:
2813
	mutex_unlock(&memcg_create_mutex);
2814 2815 2816 2817

	return retval;
}

2818 2819
static unsigned long tree_stat(struct mem_cgroup *memcg,
			       enum mem_cgroup_stat_index idx)
2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
{
	struct mem_cgroup *iter;
	long val = 0;

	/* Per-cpu values can be negative, use a signed accumulator */
	for_each_mem_cgroup_tree(iter, memcg)
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
}

static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
{
	u64 val;

2837 2838 2839 2840 2841 2842
	if (mem_cgroup_is_root(memcg)) {
		val = tree_stat(memcg, MEM_CGROUP_STAT_CACHE);
		val += tree_stat(memcg, MEM_CGROUP_STAT_RSS);
		if (swap)
			val += tree_stat(memcg, MEM_CGROUP_STAT_SWAP);
	} else {
2843
		if (!swap)
2844
			val = page_counter_read(&memcg->memory);
2845
		else
2846
			val = page_counter_read(&memcg->memsw);
2847 2848 2849 2850
	}
	return val << PAGE_SHIFT;
}

2851 2852 2853 2854 2855 2856 2857
enum {
	RES_USAGE,
	RES_LIMIT,
	RES_MAX_USAGE,
	RES_FAILCNT,
	RES_SOFT_LIMIT,
};
2858

2859
static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2860
			       struct cftype *cft)
B
Balbir Singh 已提交
2861
{
2862
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2863
	struct page_counter *counter;
2864

2865
	switch (MEMFILE_TYPE(cft->private)) {
2866
	case _MEM:
2867 2868
		counter = &memcg->memory;
		break;
2869
	case _MEMSWAP:
2870 2871
		counter = &memcg->memsw;
		break;
2872
	case _KMEM:
2873
		counter = &memcg->kmem;
2874
		break;
2875 2876 2877
	default:
		BUG();
	}
2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896

	switch (MEMFILE_ATTR(cft->private)) {
	case RES_USAGE:
		if (counter == &memcg->memory)
			return mem_cgroup_usage(memcg, false);
		if (counter == &memcg->memsw)
			return mem_cgroup_usage(memcg, true);
		return (u64)page_counter_read(counter) * PAGE_SIZE;
	case RES_LIMIT:
		return (u64)counter->limit * PAGE_SIZE;
	case RES_MAX_USAGE:
		return (u64)counter->watermark * PAGE_SIZE;
	case RES_FAILCNT:
		return counter->failcnt;
	case RES_SOFT_LIMIT:
		return (u64)memcg->soft_limit * PAGE_SIZE;
	default:
		BUG();
	}
B
Balbir Singh 已提交
2897
}
2898 2899

#ifdef CONFIG_MEMCG_KMEM
2900 2901
static int memcg_activate_kmem(struct mem_cgroup *memcg,
			       unsigned long nr_pages)
2902 2903 2904 2905
{
	int err = 0;
	int memcg_id;

2906
	BUG_ON(memcg->kmemcg_id >= 0);
2907
	BUG_ON(memcg->kmem_acct_activated);
2908
	BUG_ON(memcg->kmem_acct_active);
2909

2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
	/*
	 * For simplicity, we won't allow this to be disabled.  It also can't
	 * be changed if the cgroup has children already, or if tasks had
	 * already joined.
	 *
	 * If tasks join before we set the limit, a person looking at
	 * kmem.usage_in_bytes will have no way to determine when it took
	 * place, which makes the value quite meaningless.
	 *
	 * After it first became limited, changes in the value of the limit are
	 * of course permitted.
	 */
2922
	mutex_lock(&memcg_create_mutex);
2923 2924
	if (cgroup_has_tasks(memcg->css.cgroup) ||
	    (memcg->use_hierarchy && memcg_has_children(memcg)))
2925 2926 2927 2928
		err = -EBUSY;
	mutex_unlock(&memcg_create_mutex);
	if (err)
		goto out;
2929

2930
	memcg_id = memcg_alloc_cache_id();
2931 2932 2933 2934 2935 2936
	if (memcg_id < 0) {
		err = memcg_id;
		goto out;
	}

	/*
V
Vladimir Davydov 已提交
2937 2938
	 * We couldn't have accounted to this cgroup, because it hasn't got
	 * activated yet, so this should succeed.
2939
	 */
2940
	err = page_counter_limit(&memcg->kmem, nr_pages);
2941 2942 2943 2944
	VM_BUG_ON(err);

	static_key_slow_inc(&memcg_kmem_enabled_key);
	/*
V
Vladimir Davydov 已提交
2945 2946
	 * A memory cgroup is considered kmem-active as soon as it gets
	 * kmemcg_id. Setting the id after enabling static branching will
2947 2948 2949
	 * guarantee no one starts accounting before all call sites are
	 * patched.
	 */
V
Vladimir Davydov 已提交
2950
	memcg->kmemcg_id = memcg_id;
2951
	memcg->kmem_acct_activated = true;
2952
	memcg->kmem_acct_active = true;
2953
out:
2954 2955 2956 2957
	return err;
}

static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2958
				   unsigned long limit)
2959 2960 2961
{
	int ret;

2962
	mutex_lock(&memcg_limit_mutex);
2963
	if (!memcg_kmem_is_active(memcg))
2964
		ret = memcg_activate_kmem(memcg, limit);
2965
	else
2966 2967
		ret = page_counter_limit(&memcg->kmem, limit);
	mutex_unlock(&memcg_limit_mutex);
2968 2969 2970
	return ret;
}

2971
static int memcg_propagate_kmem(struct mem_cgroup *memcg)
2972
{
2973
	int ret = 0;
2974
	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
2975

2976 2977
	if (!parent)
		return 0;
2978

2979
	mutex_lock(&memcg_limit_mutex);
2980
	/*
2981 2982
	 * If the parent cgroup is not kmem-active now, it cannot be activated
	 * after this point, because it has at least one child already.
2983
	 */
2984
	if (memcg_kmem_is_active(parent))
2985 2986
		ret = memcg_activate_kmem(memcg, PAGE_COUNTER_MAX);
	mutex_unlock(&memcg_limit_mutex);
2987
	return ret;
2988
}
2989 2990
#else
static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2991
				   unsigned long limit)
2992 2993 2994
{
	return -EINVAL;
}
2995
#endif /* CONFIG_MEMCG_KMEM */
2996

2997 2998 2999 3000
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3001 3002
static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
B
Balbir Singh 已提交
3003
{
3004
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3005
	unsigned long nr_pages;
3006 3007
	int ret;

3008
	buf = strstrip(buf);
3009
	ret = page_counter_memparse(buf, "-1", &nr_pages);
3010 3011
	if (ret)
		return ret;
3012

3013
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3014
	case RES_LIMIT:
3015 3016 3017 3018
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3019 3020 3021
		switch (MEMFILE_TYPE(of_cft(of)->private)) {
		case _MEM:
			ret = mem_cgroup_resize_limit(memcg, nr_pages);
3022
			break;
3023 3024
		case _MEMSWAP:
			ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3025
			break;
3026 3027 3028 3029
		case _KMEM:
			ret = memcg_update_kmem_limit(memcg, nr_pages);
			break;
		}
3030
		break;
3031 3032 3033
	case RES_SOFT_LIMIT:
		memcg->soft_limit = nr_pages;
		ret = 0;
3034 3035
		break;
	}
3036
	return ret ?: nbytes;
B
Balbir Singh 已提交
3037 3038
}

3039 3040
static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
				size_t nbytes, loff_t off)
3041
{
3042
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3043
	struct page_counter *counter;
3044

3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
	switch (MEMFILE_TYPE(of_cft(of)->private)) {
	case _MEM:
		counter = &memcg->memory;
		break;
	case _MEMSWAP:
		counter = &memcg->memsw;
		break;
	case _KMEM:
		counter = &memcg->kmem;
		break;
	default:
		BUG();
	}
3058

3059
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3060
	case RES_MAX_USAGE:
3061
		page_counter_reset_watermark(counter);
3062 3063
		break;
	case RES_FAILCNT:
3064
		counter->failcnt = 0;
3065
		break;
3066 3067
	default:
		BUG();
3068
	}
3069

3070
	return nbytes;
3071 3072
}

3073
static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3074 3075
					struct cftype *cft)
{
3076
	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3077 3078
}

3079
#ifdef CONFIG_MMU
3080
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3081 3082
					struct cftype *cft, u64 val)
{
3083
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3084

3085
	if (val & ~MOVE_MASK)
3086
		return -EINVAL;
3087

3088
	/*
3089 3090 3091 3092
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
3093
	 */
3094
	memcg->move_charge_at_immigrate = val;
3095 3096
	return 0;
}
3097
#else
3098
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3099 3100 3101 3102 3103
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
3104

3105
#ifdef CONFIG_NUMA
3106
static int memcg_numa_stat_show(struct seq_file *m, void *v)
3107
{
3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119
	struct numa_stat {
		const char *name;
		unsigned int lru_mask;
	};

	static const struct numa_stat stats[] = {
		{ "total", LRU_ALL },
		{ "file", LRU_ALL_FILE },
		{ "anon", LRU_ALL_ANON },
		{ "unevictable", BIT(LRU_UNEVICTABLE) },
	};
	const struct numa_stat *stat;
3120
	int nid;
3121
	unsigned long nr;
3122
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3123

3124 3125 3126 3127 3128 3129 3130 3131 3132
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
		seq_printf(m, "%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
							  stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3133 3134
	}

3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		struct mem_cgroup *iter;

		nr = 0;
		for_each_mem_cgroup_tree(iter, memcg)
			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = 0;
			for_each_mem_cgroup_tree(iter, memcg)
				nr += mem_cgroup_node_nr_lru_pages(
					iter, nid, stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3150 3151 3152 3153 3154 3155
	}

	return 0;
}
#endif /* CONFIG_NUMA */

3156
static int memcg_stat_show(struct seq_file *m, void *v)
3157
{
3158
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3159
	unsigned long memory, memsw;
3160 3161
	struct mem_cgroup *mi;
	unsigned int i;
3162

3163 3164 3165 3166
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
		     MEM_CGROUP_STAT_NSTATS);
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
		     MEM_CGROUP_EVENTS_NSTATS);
3167 3168
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);

3169
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3170
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
3171
			continue;
3172 3173
		seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3174
	}
L
Lee Schermerhorn 已提交
3175

3176 3177 3178 3179 3180 3181 3182 3183
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
			   mem_cgroup_read_events(memcg, i));

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
3184
	/* Hierarchical information */
3185 3186 3187 3188
	memory = memsw = PAGE_COUNTER_MAX;
	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
		memory = min(memory, mi->memory.limit);
		memsw = min(memsw, mi->memsw.limit);
3189
	}
3190 3191 3192 3193 3194
	seq_printf(m, "hierarchical_memory_limit %llu\n",
		   (u64)memory * PAGE_SIZE);
	if (do_swap_account)
		seq_printf(m, "hierarchical_memsw_limit %llu\n",
			   (u64)memsw * PAGE_SIZE);
K
KOSAKI Motohiro 已提交
3195

3196 3197 3198
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
		long long val = 0;

3199
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
3200
			continue;
3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220
		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
		seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
	}

	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_events(mi, i);
		seq_printf(m, "total_%s %llu\n",
			   mem_cgroup_events_names[i], val);
	}

	for (i = 0; i < NR_LRU_LISTS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3221
	}
K
KAMEZAWA Hiroyuki 已提交
3222

K
KOSAKI Motohiro 已提交
3223 3224 3225 3226
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
3227
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
3228 3229 3230 3231 3232
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3233
				mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
3234
				rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
3235

3236 3237 3238 3239
				recent_rotated[0] += rstat->recent_rotated[0];
				recent_rotated[1] += rstat->recent_rotated[1];
				recent_scanned[0] += rstat->recent_scanned[0];
				recent_scanned[1] += rstat->recent_scanned[1];
K
KOSAKI Motohiro 已提交
3240
			}
3241 3242 3243 3244
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
3245 3246 3247
	}
#endif

3248 3249 3250
	return 0;
}

3251 3252
static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
K
KOSAKI Motohiro 已提交
3253
{
3254
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3255

3256
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
3257 3258
}

3259 3260
static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
K
KOSAKI Motohiro 已提交
3261
{
3262
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3263

3264
	if (val > 100)
K
KOSAKI Motohiro 已提交
3265 3266
		return -EINVAL;

3267
	if (css->parent)
3268 3269 3270
		memcg->swappiness = val;
	else
		vm_swappiness = val;
3271

K
KOSAKI Motohiro 已提交
3272 3273 3274
	return 0;
}

3275 3276 3277
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
3278
	unsigned long usage;
3279 3280 3281 3282
	int i;

	rcu_read_lock();
	if (!swap)
3283
		t = rcu_dereference(memcg->thresholds.primary);
3284
	else
3285
		t = rcu_dereference(memcg->memsw_thresholds.primary);
3286 3287 3288 3289

	if (!t)
		goto unlock;

3290
	usage = mem_cgroup_usage(memcg, swap);
3291 3292

	/*
3293
	 * current_threshold points to threshold just below or equal to usage.
3294 3295 3296
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
3297
	i = t->current_threshold;
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
3321
	t->current_threshold = i - 1;
3322 3323 3324 3325 3326 3327
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
3328 3329 3330 3331 3332 3333 3334
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
3335 3336 3337 3338 3339 3340 3341
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

3342 3343 3344 3345 3346 3347 3348
	if (_a->threshold > _b->threshold)
		return 1;

	if (_a->threshold < _b->threshold)
		return -1;

	return 0;
3349 3350
}

3351
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3352 3353 3354
{
	struct mem_cgroup_eventfd_list *ev;

3355 3356
	spin_lock(&memcg_oom_lock);

3357
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
3358
		eventfd_signal(ev->eventfd, 1);
3359 3360

	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3361 3362 3363
	return 0;
}

3364
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3365
{
K
KAMEZAWA Hiroyuki 已提交
3366 3367
	struct mem_cgroup *iter;

3368
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
3369
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
3370 3371
}

3372
static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3373
	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3374
{
3375 3376
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3377 3378
	unsigned long threshold;
	unsigned long usage;
3379
	int i, size, ret;
3380

3381
	ret = page_counter_memparse(args, "-1", &threshold);
3382 3383 3384 3385
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
3386

3387
	if (type == _MEM) {
3388
		thresholds = &memcg->thresholds;
3389
		usage = mem_cgroup_usage(memcg, false);
3390
	} else if (type == _MEMSWAP) {
3391
		thresholds = &memcg->memsw_thresholds;
3392
		usage = mem_cgroup_usage(memcg, true);
3393
	} else
3394 3395 3396
		BUG();

	/* Check if a threshold crossed before adding a new one */
3397
	if (thresholds->primary)
3398 3399
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

3400
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3401 3402

	/* Allocate memory for new array of thresholds */
3403
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3404
			GFP_KERNEL);
3405
	if (!new) {
3406 3407 3408
		ret = -ENOMEM;
		goto unlock;
	}
3409
	new->size = size;
3410 3411

	/* Copy thresholds (if any) to new array */
3412 3413
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3414
				sizeof(struct mem_cgroup_threshold));
3415 3416
	}

3417
	/* Add new threshold */
3418 3419
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
3420 3421

	/* Sort thresholds. Registering of new threshold isn't time-critical */
3422
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3423 3424 3425
			compare_thresholds, NULL);

	/* Find current threshold */
3426
	new->current_threshold = -1;
3427
	for (i = 0; i < size; i++) {
3428
		if (new->entries[i].threshold <= usage) {
3429
			/*
3430 3431
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
3432 3433
			 * it here.
			 */
3434
			++new->current_threshold;
3435 3436
		} else
			break;
3437 3438
	}

3439 3440 3441 3442 3443
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
3444

3445
	/* To be sure that nobody uses thresholds */
3446 3447 3448 3449 3450 3451 3452 3453
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

3454
static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3455 3456
	struct eventfd_ctx *eventfd, const char *args)
{
3457
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
T
Tejun Heo 已提交
3458 3459
}

3460
static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3461 3462
	struct eventfd_ctx *eventfd, const char *args)
{
3463
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
T
Tejun Heo 已提交
3464 3465
}

3466
static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3467
	struct eventfd_ctx *eventfd, enum res_type type)
3468
{
3469 3470
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3471
	unsigned long usage;
3472
	int i, j, size;
3473 3474

	mutex_lock(&memcg->thresholds_lock);
3475 3476

	if (type == _MEM) {
3477
		thresholds = &memcg->thresholds;
3478
		usage = mem_cgroup_usage(memcg, false);
3479
	} else if (type == _MEMSWAP) {
3480
		thresholds = &memcg->memsw_thresholds;
3481
		usage = mem_cgroup_usage(memcg, true);
3482
	} else
3483 3484
		BUG();

3485 3486 3487
	if (!thresholds->primary)
		goto unlock;

3488 3489 3490 3491
	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
3492 3493 3494
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
3495 3496 3497
			size++;
	}

3498
	new = thresholds->spare;
3499

3500 3501
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
3502 3503
		kfree(new);
		new = NULL;
3504
		goto swap_buffers;
3505 3506
	}

3507
	new->size = size;
3508 3509

	/* Copy thresholds and find current threshold */
3510 3511 3512
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
3513 3514
			continue;

3515
		new->entries[j] = thresholds->primary->entries[i];
3516
		if (new->entries[j].threshold <= usage) {
3517
			/*
3518
			 * new->current_threshold will not be used
3519 3520 3521
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
3522
			++new->current_threshold;
3523 3524 3525 3526
		}
		j++;
	}

3527
swap_buffers:
3528 3529
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
3530 3531 3532 3533 3534 3535
	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}

3536
	rcu_assign_pointer(thresholds->primary, new);
3537

3538
	/* To be sure that nobody uses thresholds */
3539
	synchronize_rcu();
3540
unlock:
3541 3542
	mutex_unlock(&memcg->thresholds_lock);
}
3543

3544
static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3545 3546
	struct eventfd_ctx *eventfd)
{
3547
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
T
Tejun Heo 已提交
3548 3549
}

3550
static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3551 3552
	struct eventfd_ctx *eventfd)
{
3553
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
T
Tejun Heo 已提交
3554 3555
}

3556
static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3557
	struct eventfd_ctx *eventfd, const char *args)
K
KAMEZAWA Hiroyuki 已提交
3558 3559 3560 3561 3562 3563 3564
{
	struct mem_cgroup_eventfd_list *event;

	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

3565
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3566 3567 3568 3569 3570

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
3571
	if (memcg->under_oom)
K
KAMEZAWA Hiroyuki 已提交
3572
		eventfd_signal(eventfd, 1);
3573
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3574 3575 3576 3577

	return 0;
}

3578
static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3579
	struct eventfd_ctx *eventfd)
K
KAMEZAWA Hiroyuki 已提交
3580 3581 3582
{
	struct mem_cgroup_eventfd_list *ev, *tmp;

3583
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3584

3585
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
3586 3587 3588 3589 3590 3591
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

3592
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3593 3594
}

3595
static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3596
{
3597
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3598

3599
	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3600
	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3601 3602 3603
	return 0;
}

3604
static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3605 3606
	struct cftype *cft, u64 val)
{
3607
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3608 3609

	/* cannot set to root cgroup and only 0 and 1 are allowed */
3610
	if (!css->parent || !((val == 0) || (val == 1)))
3611 3612
		return -EINVAL;

3613
	memcg->oom_kill_disable = val;
3614
	if (!val)
3615
		memcg_oom_recover(memcg);
3616

3617 3618 3619
	return 0;
}

A
Andrew Morton 已提交
3620
#ifdef CONFIG_MEMCG_KMEM
3621
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
3622
{
3623 3624 3625 3626 3627
	int ret;

	ret = memcg_propagate_kmem(memcg);
	if (ret)
		return ret;
3628

3629
	return mem_cgroup_sockets_init(memcg, ss);
3630
}
3631

3632 3633
static void memcg_deactivate_kmem(struct mem_cgroup *memcg)
{
3634 3635 3636 3637
	struct cgroup_subsys_state *css;
	struct mem_cgroup *parent, *child;
	int kmemcg_id;

3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649
	if (!memcg->kmem_acct_active)
		return;

	/*
	 * Clear the 'active' flag before clearing memcg_caches arrays entries.
	 * Since we take the slab_mutex in memcg_deactivate_kmem_caches(), it
	 * guarantees no cache will be created for this cgroup after we are
	 * done (see memcg_create_kmem_cache()).
	 */
	memcg->kmem_acct_active = false;

	memcg_deactivate_kmem_caches(memcg);
3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675

	kmemcg_id = memcg->kmemcg_id;
	BUG_ON(kmemcg_id < 0);

	parent = parent_mem_cgroup(memcg);
	if (!parent)
		parent = root_mem_cgroup;

	/*
	 * Change kmemcg_id of this cgroup and all its descendants to the
	 * parent's id, and then move all entries from this cgroup's list_lrus
	 * to ones of the parent. After we have finished, all list_lrus
	 * corresponding to this cgroup are guaranteed to remain empty. The
	 * ordering is imposed by list_lru_node->lock taken by
	 * memcg_drain_all_list_lrus().
	 */
	css_for_each_descendant_pre(css, &memcg->css) {
		child = mem_cgroup_from_css(css);
		BUG_ON(child->kmemcg_id != kmemcg_id);
		child->kmemcg_id = parent->kmemcg_id;
		if (!memcg->use_hierarchy)
			break;
	}
	memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);

	memcg_free_cache_id(kmemcg_id);
3676 3677
}

3678
static void memcg_destroy_kmem(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
3679
{
3680 3681 3682 3683 3684
	if (memcg->kmem_acct_activated) {
		memcg_destroy_kmem_caches(memcg);
		static_key_slow_dec(&memcg_kmem_enabled_key);
		WARN_ON(page_counter_read(&memcg->kmem));
	}
3685
	mem_cgroup_sockets_destroy(memcg);
3686
}
3687
#else
3688
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
3689 3690 3691
{
	return 0;
}
G
Glauber Costa 已提交
3692

3693 3694 3695 3696
static void memcg_deactivate_kmem(struct mem_cgroup *memcg)
{
}

3697 3698 3699
static void memcg_destroy_kmem(struct mem_cgroup *memcg)
{
}
3700 3701
#endif

3702 3703 3704 3705 3706 3707 3708
#ifdef CONFIG_CGROUP_WRITEBACK

struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
{
	return &memcg->cgwb_list;
}

T
Tejun Heo 已提交
3709 3710 3711 3712 3713 3714 3715 3716 3717 3718
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return wb_domain_init(&memcg->cgwb_domain, gfp);
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
	wb_domain_exit(&memcg->cgwb_domain);
}

3719 3720 3721 3722 3723
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
	wb_domain_size_changed(&memcg->cgwb_domain);
}

T
Tejun Heo 已提交
3724 3725 3726 3727 3728 3729 3730 3731 3732 3733
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);

	if (!memcg->css.parent)
		return NULL;

	return &memcg->cgwb_domain;
}

3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776
/**
 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
 * @wb: bdi_writeback in question
 * @pavail: out parameter for number of available pages
 * @pdirty: out parameter for number of dirty pages
 * @pwriteback: out parameter for number of pages under writeback
 *
 * Determine the numbers of available, dirty, and writeback pages in @wb's
 * memcg.  Dirty and writeback are self-explanatory.  Available is a bit
 * more involved.
 *
 * A memcg's headroom is "min(max, high) - used".  The available memory is
 * calculated as the lowest headroom of itself and the ancestors plus the
 * number of pages already being used for file pages.  Note that this
 * doesn't consider the actual amount of available memory in the system.
 * The caller should further cap *@pavail accordingly.
 */
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pavail,
			 unsigned long *pdirty, unsigned long *pwriteback)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
	struct mem_cgroup *parent;
	unsigned long head_room = PAGE_COUNTER_MAX;
	unsigned long file_pages;

	*pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY);

	/* this should eventually include NR_UNSTABLE_NFS */
	*pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK);

	file_pages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
						    (1 << LRU_ACTIVE_FILE));
	while ((parent = parent_mem_cgroup(memcg))) {
		unsigned long ceiling = min(memcg->memory.limit, memcg->high);
		unsigned long used = page_counter_read(&memcg->memory);

		head_room = min(head_room, ceiling - min(ceiling, used));
		memcg = parent;
	}

	*pavail = file_pages + head_room;
}

T
Tejun Heo 已提交
3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787
#else	/* CONFIG_CGROUP_WRITEBACK */

static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return 0;
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
}

3788 3789 3790 3791
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
}

3792 3793
#endif	/* CONFIG_CGROUP_WRITEBACK */

3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806
/*
 * DO NOT USE IN NEW FILES.
 *
 * "cgroup.event_control" implementation.
 *
 * This is way over-engineered.  It tries to support fully configurable
 * events for each user.  Such level of flexibility is completely
 * unnecessary especially in the light of the planned unified hierarchy.
 *
 * Please deprecate this and replace with something simpler if at all
 * possible.
 */

3807 3808 3809 3810 3811
/*
 * Unregister event and free resources.
 *
 * Gets called from workqueue.
 */
3812
static void memcg_event_remove(struct work_struct *work)
3813
{
3814 3815
	struct mem_cgroup_event *event =
		container_of(work, struct mem_cgroup_event, remove);
3816
	struct mem_cgroup *memcg = event->memcg;
3817 3818 3819

	remove_wait_queue(event->wqh, &event->wait);

3820
	event->unregister_event(memcg, event->eventfd);
3821 3822 3823 3824 3825 3826

	/* Notify userspace the event is going away. */
	eventfd_signal(event->eventfd, 1);

	eventfd_ctx_put(event->eventfd);
	kfree(event);
3827
	css_put(&memcg->css);
3828 3829 3830 3831 3832 3833 3834
}

/*
 * Gets called on POLLHUP on eventfd when user closes it.
 *
 * Called with wqh->lock held and interrupts disabled.
 */
3835 3836
static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
			    int sync, void *key)
3837
{
3838 3839
	struct mem_cgroup_event *event =
		container_of(wait, struct mem_cgroup_event, wait);
3840
	struct mem_cgroup *memcg = event->memcg;
3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852
	unsigned long flags = (unsigned long)key;

	if (flags & POLLHUP) {
		/*
		 * If the event has been detached at cgroup removal, we
		 * can simply return knowing the other side will cleanup
		 * for us.
		 *
		 * We can't race against event freeing since the other
		 * side will require wqh->lock via remove_wait_queue(),
		 * which we hold.
		 */
3853
		spin_lock(&memcg->event_list_lock);
3854 3855 3856 3857 3858 3859 3860 3861
		if (!list_empty(&event->list)) {
			list_del_init(&event->list);
			/*
			 * We are in atomic context, but cgroup_event_remove()
			 * may sleep, so we have to call it in workqueue.
			 */
			schedule_work(&event->remove);
		}
3862
		spin_unlock(&memcg->event_list_lock);
3863 3864 3865 3866 3867
	}

	return 0;
}

3868
static void memcg_event_ptable_queue_proc(struct file *file,
3869 3870
		wait_queue_head_t *wqh, poll_table *pt)
{
3871 3872
	struct mem_cgroup_event *event =
		container_of(pt, struct mem_cgroup_event, pt);
3873 3874 3875 3876 3877 3878

	event->wqh = wqh;
	add_wait_queue(wqh, &event->wait);
}

/*
3879 3880
 * DO NOT USE IN NEW FILES.
 *
3881 3882 3883 3884 3885
 * Parse input and register new cgroup event handler.
 *
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
3886 3887
static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
					 char *buf, size_t nbytes, loff_t off)
3888
{
3889
	struct cgroup_subsys_state *css = of_css(of);
3890
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3891
	struct mem_cgroup_event *event;
3892 3893 3894 3895
	struct cgroup_subsys_state *cfile_css;
	unsigned int efd, cfd;
	struct fd efile;
	struct fd cfile;
3896
	const char *name;
3897 3898 3899
	char *endp;
	int ret;

3900 3901 3902
	buf = strstrip(buf);

	efd = simple_strtoul(buf, &endp, 10);
3903 3904
	if (*endp != ' ')
		return -EINVAL;
3905
	buf = endp + 1;
3906

3907
	cfd = simple_strtoul(buf, &endp, 10);
3908 3909
	if ((*endp != ' ') && (*endp != '\0'))
		return -EINVAL;
3910
	buf = endp + 1;
3911 3912 3913 3914 3915

	event = kzalloc(sizeof(*event), GFP_KERNEL);
	if (!event)
		return -ENOMEM;

3916
	event->memcg = memcg;
3917
	INIT_LIST_HEAD(&event->list);
3918 3919 3920
	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
	INIT_WORK(&event->remove, memcg_event_remove);
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945

	efile = fdget(efd);
	if (!efile.file) {
		ret = -EBADF;
		goto out_kfree;
	}

	event->eventfd = eventfd_ctx_fileget(efile.file);
	if (IS_ERR(event->eventfd)) {
		ret = PTR_ERR(event->eventfd);
		goto out_put_efile;
	}

	cfile = fdget(cfd);
	if (!cfile.file) {
		ret = -EBADF;
		goto out_put_eventfd;
	}

	/* the process need read permission on control file */
	/* AV: shouldn't we check that it's been opened for read instead? */
	ret = inode_permission(file_inode(cfile.file), MAY_READ);
	if (ret < 0)
		goto out_put_cfile;

3946 3947 3948 3949 3950
	/*
	 * Determine the event callbacks and set them in @event.  This used
	 * to be done via struct cftype but cgroup core no longer knows
	 * about these events.  The following is crude but the whole thing
	 * is for compatibility anyway.
3951 3952
	 *
	 * DO NOT ADD NEW FILES.
3953
	 */
A
Al Viro 已提交
3954
	name = cfile.file->f_path.dentry->d_name.name;
3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965

	if (!strcmp(name, "memory.usage_in_bytes")) {
		event->register_event = mem_cgroup_usage_register_event;
		event->unregister_event = mem_cgroup_usage_unregister_event;
	} else if (!strcmp(name, "memory.oom_control")) {
		event->register_event = mem_cgroup_oom_register_event;
		event->unregister_event = mem_cgroup_oom_unregister_event;
	} else if (!strcmp(name, "memory.pressure_level")) {
		event->register_event = vmpressure_register_event;
		event->unregister_event = vmpressure_unregister_event;
	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
T
Tejun Heo 已提交
3966 3967
		event->register_event = memsw_cgroup_usage_register_event;
		event->unregister_event = memsw_cgroup_usage_unregister_event;
3968 3969 3970 3971 3972
	} else {
		ret = -EINVAL;
		goto out_put_cfile;
	}

3973
	/*
3974 3975 3976
	 * Verify @cfile should belong to @css.  Also, remaining events are
	 * automatically removed on cgroup destruction but the removal is
	 * asynchronous, so take an extra ref on @css.
3977
	 */
A
Al Viro 已提交
3978
	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3979
					       &memory_cgrp_subsys);
3980
	ret = -EINVAL;
3981
	if (IS_ERR(cfile_css))
3982
		goto out_put_cfile;
3983 3984
	if (cfile_css != css) {
		css_put(cfile_css);
3985
		goto out_put_cfile;
3986
	}
3987

3988
	ret = event->register_event(memcg, event->eventfd, buf);
3989 3990 3991 3992 3993
	if (ret)
		goto out_put_css;

	efile.file->f_op->poll(efile.file, &event->pt);

3994 3995 3996
	spin_lock(&memcg->event_list_lock);
	list_add(&event->list, &memcg->event_list);
	spin_unlock(&memcg->event_list_lock);
3997 3998 3999 4000

	fdput(cfile);
	fdput(efile);

4001
	return nbytes;
4002 4003

out_put_css:
4004
	css_put(css);
4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016
out_put_cfile:
	fdput(cfile);
out_put_eventfd:
	eventfd_ctx_put(event->eventfd);
out_put_efile:
	fdput(efile);
out_kfree:
	kfree(event);

	return ret;
}

4017
static struct cftype mem_cgroup_legacy_files[] = {
B
Balbir Singh 已提交
4018
	{
4019
		.name = "usage_in_bytes",
4020
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4021
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4022
	},
4023 4024
	{
		.name = "max_usage_in_bytes",
4025
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4026
		.write = mem_cgroup_reset,
4027
		.read_u64 = mem_cgroup_read_u64,
4028
	},
B
Balbir Singh 已提交
4029
	{
4030
		.name = "limit_in_bytes",
4031
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4032
		.write = mem_cgroup_write,
4033
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4034
	},
4035 4036 4037
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4038
		.write = mem_cgroup_write,
4039
		.read_u64 = mem_cgroup_read_u64,
4040
	},
B
Balbir Singh 已提交
4041 4042
	{
		.name = "failcnt",
4043
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4044
		.write = mem_cgroup_reset,
4045
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4046
	},
4047 4048
	{
		.name = "stat",
4049
		.seq_show = memcg_stat_show,
4050
	},
4051 4052
	{
		.name = "force_empty",
4053
		.write = mem_cgroup_force_empty_write,
4054
	},
4055 4056 4057 4058 4059
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
4060
	{
4061
		.name = "cgroup.event_control",		/* XXX: for compat */
4062
		.write = memcg_write_event_control,
4063 4064 4065
		.flags = CFTYPE_NO_PREFIX,
		.mode = S_IWUGO,
	},
K
KOSAKI Motohiro 已提交
4066 4067 4068 4069 4070
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4071 4072 4073 4074 4075
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4076 4077
	{
		.name = "oom_control",
4078
		.seq_show = mem_cgroup_oom_control_read,
4079
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4080 4081
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
4082 4083 4084
	{
		.name = "pressure_level",
	},
4085 4086 4087
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
4088
		.seq_show = memcg_numa_stat_show,
4089 4090
	},
#endif
4091 4092 4093 4094
#ifdef CONFIG_MEMCG_KMEM
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4095
		.write = mem_cgroup_write,
4096
		.read_u64 = mem_cgroup_read_u64,
4097 4098 4099 4100
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4101
		.read_u64 = mem_cgroup_read_u64,
4102 4103 4104 4105
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4106
		.write = mem_cgroup_reset,
4107
		.read_u64 = mem_cgroup_read_u64,
4108 4109 4110 4111
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4112
		.write = mem_cgroup_reset,
4113
		.read_u64 = mem_cgroup_read_u64,
4114
	},
4115 4116 4117
#ifdef CONFIG_SLABINFO
	{
		.name = "kmem.slabinfo",
4118 4119 4120 4121
		.seq_start = slab_start,
		.seq_next = slab_next,
		.seq_stop = slab_stop,
		.seq_show = memcg_slab_show,
4122 4123
	},
#endif
4124
#endif
4125
	{ },	/* terminate */
4126
};
4127

4128
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4129 4130
{
	struct mem_cgroup_per_node *pn;
4131
	struct mem_cgroup_per_zone *mz;
4132
	int zone, tmp = node;
4133 4134 4135 4136 4137 4138 4139 4140
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4141 4142
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4143
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4144 4145
	if (!pn)
		return 1;
4146 4147 4148

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
4149
		lruvec_init(&mz->lruvec);
4150 4151
		mz->usage_in_excess = 0;
		mz->on_tree = false;
4152
		mz->memcg = memcg;
4153
	}
4154
	memcg->nodeinfo[node] = pn;
4155 4156 4157
	return 0;
}

4158
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4159
{
4160
	kfree(memcg->nodeinfo[node]);
4161 4162
}

4163 4164
static struct mem_cgroup *mem_cgroup_alloc(void)
{
4165
	struct mem_cgroup *memcg;
4166
	size_t size;
4167

4168 4169
	size = sizeof(struct mem_cgroup);
	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4170

4171
	memcg = kzalloc(size, GFP_KERNEL);
4172
	if (!memcg)
4173 4174
		return NULL;

4175 4176
	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat)
4177
		goto out_free;
T
Tejun Heo 已提交
4178 4179 4180 4181

	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
		goto out_free_stat;

4182 4183
	spin_lock_init(&memcg->pcp_counter_lock);
	return memcg;
4184

T
Tejun Heo 已提交
4185 4186
out_free_stat:
	free_percpu(memcg->stat);
4187
out_free:
4188
	kfree(memcg);
4189
	return NULL;
4190 4191
}

4192
/*
4193 4194 4195 4196 4197 4198 4199 4200
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
4201
 */
4202 4203

static void __mem_cgroup_free(struct mem_cgroup *memcg)
4204
{
4205
	int node;
4206

4207
	mem_cgroup_remove_from_trees(memcg);
4208 4209 4210 4211 4212

	for_each_node(node)
		free_mem_cgroup_per_zone_info(memcg, node);

	free_percpu(memcg->stat);
T
Tejun Heo 已提交
4213
	memcg_wb_domain_exit(memcg);
4214
	kfree(memcg);
4215
}
4216

4217 4218 4219
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
G
Glauber Costa 已提交
4220
struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
4221
{
4222
	if (!memcg->memory.parent)
4223
		return NULL;
4224
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
4225
}
G
Glauber Costa 已提交
4226
EXPORT_SYMBOL(parent_mem_cgroup);
4227

L
Li Zefan 已提交
4228
static struct cgroup_subsys_state * __ref
4229
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
B
Balbir Singh 已提交
4230
{
4231
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
4232
	long error = -ENOMEM;
4233
	int node;
B
Balbir Singh 已提交
4234

4235 4236
	memcg = mem_cgroup_alloc();
	if (!memcg)
K
KAMEZAWA Hiroyuki 已提交
4237
		return ERR_PTR(error);
4238

B
Bob Liu 已提交
4239
	for_each_node(node)
4240
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
4241
			goto free_out;
4242

4243
	/* root ? */
4244
	if (parent_css == NULL) {
4245
		root_mem_cgroup = memcg;
T
Tejun Heo 已提交
4246
		mem_cgroup_root_css = &memcg->css;
4247
		page_counter_init(&memcg->memory, NULL);
4248
		memcg->high = PAGE_COUNTER_MAX;
4249
		memcg->soft_limit = PAGE_COUNTER_MAX;
4250 4251
		page_counter_init(&memcg->memsw, NULL);
		page_counter_init(&memcg->kmem, NULL);
4252
	}
4253

4254 4255 4256 4257 4258
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	memcg->move_charge_at_immigrate = 0;
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);
4259
	vmpressure_init(&memcg->vmpressure);
4260 4261
	INIT_LIST_HEAD(&memcg->event_list);
	spin_lock_init(&memcg->event_list_lock);
V
Vladimir Davydov 已提交
4262 4263 4264
#ifdef CONFIG_MEMCG_KMEM
	memcg->kmemcg_id = -1;
#endif
4265 4266 4267
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&memcg->cgwb_list);
#endif
4268 4269 4270 4271 4272 4273 4274 4275
	return &memcg->css;

free_out:
	__mem_cgroup_free(memcg);
	return ERR_PTR(error);
}

static int
4276
mem_cgroup_css_online(struct cgroup_subsys_state *css)
4277
{
4278
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
4279
	struct mem_cgroup *parent = mem_cgroup_from_css(css->parent);
4280
	int ret;
4281

4282
	if (css->id > MEM_CGROUP_ID_MAX)
4283 4284
		return -ENOSPC;

T
Tejun Heo 已提交
4285
	if (!parent)
4286 4287
		return 0;

4288
	mutex_lock(&memcg_create_mutex);
4289 4290 4291 4292 4293 4294

	memcg->use_hierarchy = parent->use_hierarchy;
	memcg->oom_kill_disable = parent->oom_kill_disable;
	memcg->swappiness = mem_cgroup_swappiness(parent);

	if (parent->use_hierarchy) {
4295
		page_counter_init(&memcg->memory, &parent->memory);
4296
		memcg->high = PAGE_COUNTER_MAX;
4297
		memcg->soft_limit = PAGE_COUNTER_MAX;
4298 4299
		page_counter_init(&memcg->memsw, &parent->memsw);
		page_counter_init(&memcg->kmem, &parent->kmem);
4300

4301
		/*
4302 4303
		 * No need to take a reference to the parent because cgroup
		 * core guarantees its existence.
4304
		 */
4305
	} else {
4306
		page_counter_init(&memcg->memory, NULL);
4307
		memcg->high = PAGE_COUNTER_MAX;
4308
		memcg->soft_limit = PAGE_COUNTER_MAX;
4309 4310
		page_counter_init(&memcg->memsw, NULL);
		page_counter_init(&memcg->kmem, NULL);
4311 4312 4313 4314 4315
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
4316
		if (parent != root_mem_cgroup)
4317
			memory_cgrp_subsys.broken_hierarchy = true;
4318
	}
4319
	mutex_unlock(&memcg_create_mutex);
4320

4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332
	ret = memcg_init_kmem(memcg, &memory_cgrp_subsys);
	if (ret)
		return ret;

	/*
	 * Make sure the memcg is initialized: mem_cgroup_iter()
	 * orders reading memcg->initialized against its callers
	 * reading the memcg members.
	 */
	smp_store_release(&memcg->initialized, 1);

	return 0;
B
Balbir Singh 已提交
4333 4334
}

4335
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4336
{
4337
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4338
	struct mem_cgroup_event *event, *tmp;
4339 4340 4341 4342 4343 4344

	/*
	 * Unregister events and notify userspace.
	 * Notify userspace about cgroup removing only after rmdir of cgroup
	 * directory to avoid race between userspace and kernelspace.
	 */
4345 4346
	spin_lock(&memcg->event_list_lock);
	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4347 4348 4349
		list_del_init(&event->list);
		schedule_work(&event->remove);
	}
4350
	spin_unlock(&memcg->event_list_lock);
4351

4352
	vmpressure_cleanup(&memcg->vmpressure);
4353 4354

	memcg_deactivate_kmem(memcg);
4355 4356

	wb_memcg_offline(memcg);
4357 4358
}

4359
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
B
Balbir Singh 已提交
4360
{
4361
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4362

4363
	memcg_destroy_kmem(memcg);
4364
	__mem_cgroup_free(memcg);
B
Balbir Singh 已提交
4365 4366
}

4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383
/**
 * mem_cgroup_css_reset - reset the states of a mem_cgroup
 * @css: the target css
 *
 * Reset the states of the mem_cgroup associated with @css.  This is
 * invoked when the userland requests disabling on the default hierarchy
 * but the memcg is pinned through dependency.  The memcg should stop
 * applying policies and should revert to the vanilla state as it may be
 * made visible again.
 *
 * The current implementation only resets the essential configurations.
 * This needs to be expanded to cover all the visible parts.
 */
static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

4384 4385 4386
	mem_cgroup_resize_limit(memcg, PAGE_COUNTER_MAX);
	mem_cgroup_resize_memsw_limit(memcg, PAGE_COUNTER_MAX);
	memcg_update_kmem_limit(memcg, PAGE_COUNTER_MAX);
4387 4388
	memcg->low = 0;
	memcg->high = PAGE_COUNTER_MAX;
4389
	memcg->soft_limit = PAGE_COUNTER_MAX;
4390
	memcg_wb_domain_size_changed(memcg);
4391 4392
}

4393
#ifdef CONFIG_MMU
4394
/* Handlers for move charge at task migration. */
4395
static int mem_cgroup_do_precharge(unsigned long count)
4396
{
4397
	int ret;
4398 4399

	/* Try a single bulk charge without reclaim first */
4400
	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_WAIT, count);
4401
	if (!ret) {
4402 4403 4404
		mc.precharge += count;
		return ret;
	}
4405
	if (ret == -EINTR) {
4406
		cancel_charge(root_mem_cgroup, count);
4407 4408
		return ret;
	}
4409 4410

	/* Try charges one by one with reclaim */
4411
	while (count--) {
4412
		ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1);
4413 4414 4415
		/*
		 * In case of failure, any residual charges against
		 * mc.to will be dropped by mem_cgroup_clear_mc()
4416 4417
		 * later on.  However, cancel any charges that are
		 * bypassed to root right away or they'll be lost.
4418
		 */
4419
		if (ret == -EINTR)
4420
			cancel_charge(root_mem_cgroup, 1);
4421 4422
		if (ret)
			return ret;
4423
		mc.precharge++;
4424
		cond_resched();
4425
	}
4426
	return 0;
4427 4428 4429
}

/**
4430
 * get_mctgt_type - get target type of moving charge
4431 4432 4433
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
4434
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
4435 4436 4437 4438 4439 4440
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
4441 4442 4443
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
4444 4445 4446 4447 4448
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
4449
	swp_entry_t	ent;
4450 4451 4452
};

enum mc_target_type {
4453
	MC_TARGET_NONE = 0,
4454
	MC_TARGET_PAGE,
4455
	MC_TARGET_SWAP,
4456 4457
};

D
Daisuke Nishimura 已提交
4458 4459
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
4460
{
D
Daisuke Nishimura 已提交
4461
	struct page *page = vm_normal_page(vma, addr, ptent);
4462

D
Daisuke Nishimura 已提交
4463 4464 4465
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
4466
		if (!(mc.flags & MOVE_ANON))
D
Daisuke Nishimura 已提交
4467
			return NULL;
4468 4469 4470 4471
	} else {
		if (!(mc.flags & MOVE_FILE))
			return NULL;
	}
D
Daisuke Nishimura 已提交
4472 4473 4474 4475 4476 4477
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

4478
#ifdef CONFIG_SWAP
D
Daisuke Nishimura 已提交
4479 4480 4481 4482 4483 4484
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

4485
	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
D
Daisuke Nishimura 已提交
4486
		return NULL;
4487 4488 4489 4490
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
4491
	page = find_get_page(swap_address_space(ent), ent.val);
D
Daisuke Nishimura 已提交
4492 4493 4494 4495 4496
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}
4497 4498 4499 4500 4501 4502 4503
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
4504

4505 4506 4507 4508 4509 4510 4511 4512 4513
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
4514
	if (!(mc.flags & MOVE_FILE))
4515 4516 4517
		return NULL;

	mapping = vma->vm_file->f_mapping;
4518
	pgoff = linear_page_index(vma, addr);
4519 4520

	/* page is moved even if it's not RSS of this task(page-faulted). */
4521 4522
#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534
	if (shmem_mapping(mapping)) {
		page = find_get_entry(mapping, pgoff);
		if (radix_tree_exceptional_entry(page)) {
			swp_entry_t swp = radix_to_swp_entry(page);
			if (do_swap_account)
				*entry = swp;
			page = find_get_page(swap_address_space(swp), swp.val);
		}
	} else
		page = find_get_page(mapping, pgoff);
#else
	page = find_get_page(mapping, pgoff);
4535
#endif
4536 4537 4538
	return page;
}

4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559
/**
 * mem_cgroup_move_account - move account of the page
 * @page: the page
 * @nr_pages: number of regular pages (>1 for huge pages)
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
 * The caller must confirm following.
 * - page is not on LRU (isolate_page() is useful.)
 * - compound_lock is held when nr_pages > 1
 *
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
 */
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct mem_cgroup *from,
				   struct mem_cgroup *to)
{
	unsigned long flags;
	int ret;
4560
	bool anon;
4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585

	VM_BUG_ON(from == to);
	VM_BUG_ON_PAGE(PageLRU(page), page);
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
	if (nr_pages > 1 && !PageTransHuge(page))
		goto out;

	/*
	 * Prevent mem_cgroup_migrate() from looking at page->mem_cgroup
	 * of its source page while we change it: page migration takes
	 * both pages off the LRU, but page cache replacement doesn't.
	 */
	if (!trylock_page(page))
		goto out;

	ret = -EINVAL;
	if (page->mem_cgroup != from)
		goto out_unlock;

4586 4587
	anon = PageAnon(page);

4588 4589
	spin_lock_irqsave(&from->move_lock, flags);

4590
	if (!anon && page_mapped(page)) {
4591 4592 4593 4594 4595 4596
		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
			       nr_pages);
		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
			       nr_pages);
	}

4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612
	/*
	 * move_lock grabbed above and caller set from->moving_account, so
	 * mem_cgroup_update_page_stat() will serialize updates to PageDirty.
	 * So mapping should be stable for dirty pages.
	 */
	if (!anon && PageDirty(page)) {
		struct address_space *mapping = page_mapping(page);

		if (mapping_cap_account_dirty(mapping)) {
			__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY],
				       nr_pages);
			__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY],
				       nr_pages);
		}
	}

4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643
	if (PageWriteback(page)) {
		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
			       nr_pages);
		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
			       nr_pages);
	}

	/*
	 * It is safe to change page->mem_cgroup here because the page
	 * is referenced, charged, and isolated - we can't race with
	 * uncharging, charging, migration, or LRU putback.
	 */

	/* caller should have done css_get */
	page->mem_cgroup = to;
	spin_unlock_irqrestore(&from->move_lock, flags);

	ret = 0;

	local_irq_disable();
	mem_cgroup_charge_statistics(to, page, nr_pages);
	memcg_check_events(to, page);
	mem_cgroup_charge_statistics(from, page, -nr_pages);
	memcg_check_events(from, page);
	local_irq_enable();
out_unlock:
	unlock_page(page);
out:
	return ret;
}

4644
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
4645 4646 4647
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
4648
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
4649 4650 4651 4652 4653 4654
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4655
	else if (pte_none(ptent))
4656
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
4657 4658

	if (!page && !ent.val)
4659
		return ret;
4660 4661
	if (page) {
		/*
4662
		 * Do only loose check w/o serialization.
4663
		 * mem_cgroup_move_account() checks the page is valid or
4664
		 * not under LRU exclusion.
4665
		 */
4666
		if (page->mem_cgroup == mc.from) {
4667 4668 4669 4670 4671 4672 4673
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
4674 4675
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
L
Li Zefan 已提交
4676
	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4677 4678 4679
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
4680 4681 4682 4683
	}
	return ret;
}

4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
 * We don't consider swapping or file mapped pages because THP does not
 * support them for now.
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	enum mc_target_type ret = MC_TARGET_NONE;

	page = pmd_page(pmd);
4697
	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4698
	if (!(mc.flags & MOVE_ANON))
4699
		return ret;
4700
	if (page->mem_cgroup == mc.from) {
4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

4717 4718 4719 4720
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
4721
	struct vm_area_struct *vma = walk->vma;
4722 4723 4724
	pte_t *pte;
	spinlock_t *ptl;

4725
	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
4726 4727
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
4728
		spin_unlock(ptl);
4729
		return 0;
4730
	}
4731

4732 4733
	if (pmd_trans_unstable(pmd))
		return 0;
4734 4735
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
4736
		if (get_mctgt_type(vma, addr, *pte, NULL))
4737 4738 4739 4740
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

4741 4742 4743
	return 0;
}

4744 4745 4746 4747
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;

4748 4749 4750 4751
	struct mm_walk mem_cgroup_count_precharge_walk = {
		.pmd_entry = mem_cgroup_count_precharge_pte_range,
		.mm = mm,
	};
4752
	down_read(&mm->mmap_sem);
4753
	walk_page_range(0, ~0UL, &mem_cgroup_count_precharge_walk);
4754
	up_read(&mm->mmap_sem);
4755 4756 4757 4758 4759 4760 4761 4762 4763

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
4764 4765 4766 4767 4768
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
4769 4770
}

4771 4772
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
4773
{
4774 4775 4776
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

4777
	/* we must uncharge all the leftover precharges from mc.to */
4778
	if (mc.precharge) {
4779
		cancel_charge(mc.to, mc.precharge);
4780 4781 4782 4783 4784 4785 4786
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
4787
		cancel_charge(mc.from, mc.moved_charge);
4788
		mc.moved_charge = 0;
4789
	}
4790 4791 4792
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
4793
		if (!mem_cgroup_is_root(mc.from))
4794
			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4795

4796
		/*
4797 4798
		 * we charged both to->memory and to->memsw, so we
		 * should uncharge to->memory.
4799
		 */
4800
		if (!mem_cgroup_is_root(mc.to))
4801 4802
			page_counter_uncharge(&mc.to->memory, mc.moved_swap);

4803
		css_put_many(&mc.from->css, mc.moved_swap);
4804

L
Li Zefan 已提交
4805
		/* we've already done css_get(mc.to) */
4806 4807
		mc.moved_swap = 0;
	}
4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
4821
	spin_lock(&mc.lock);
4822 4823
	mc.from = NULL;
	mc.to = NULL;
4824
	spin_unlock(&mc.lock);
4825 4826
}

4827
static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
4828
				 struct cgroup_taskset *tset)
4829
{
4830
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4831 4832 4833
	struct mem_cgroup *from;
	struct task_struct *p;
	struct mm_struct *mm;
4834
	unsigned long move_flags;
4835
	int ret = 0;
4836

4837 4838 4839 4840 4841
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
4842
	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871
	if (!move_flags)
		return 0;

	p = cgroup_taskset_first(tset);
	from = mem_cgroup_from_task(p);

	VM_BUG_ON(from == memcg);

	mm = get_task_mm(p);
	if (!mm)
		return 0;
	/* We move charges only when we move a owner of the mm */
	if (mm->owner == p) {
		VM_BUG_ON(mc.from);
		VM_BUG_ON(mc.to);
		VM_BUG_ON(mc.precharge);
		VM_BUG_ON(mc.moved_charge);
		VM_BUG_ON(mc.moved_swap);

		spin_lock(&mc.lock);
		mc.from = from;
		mc.to = memcg;
		mc.flags = move_flags;
		spin_unlock(&mc.lock);
		/* We set mc.moving_task later */

		ret = mem_cgroup_precharge_mc(mm);
		if (ret)
			mem_cgroup_clear_mc();
4872
	}
4873
	mmput(mm);
4874 4875 4876
	return ret;
}

4877
static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
4878
				     struct cgroup_taskset *tset)
4879
{
4880 4881
	if (mc.to)
		mem_cgroup_clear_mc();
4882 4883
}

4884 4885 4886
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
4887
{
4888
	int ret = 0;
4889
	struct vm_area_struct *vma = walk->vma;
4890 4891
	pte_t *pte;
	spinlock_t *ptl;
4892 4893 4894
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
4895

4896 4897 4898 4899 4900 4901 4902 4903 4904 4905
	/*
	 * We don't take compound_lock() here but no race with splitting thp
	 * happens because:
	 *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
	 *    under splitting, which means there's no concurrent thp split,
	 *  - if another thread runs into split_huge_page() just after we
	 *    entered this if-block, the thread must wait for page table lock
	 *    to be unlocked in __split_huge_page_splitting(), where the main
	 *    part of thp split is not executed yet.
	 */
4906
	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
4907
		if (mc.precharge < HPAGE_PMD_NR) {
4908
			spin_unlock(ptl);
4909 4910 4911 4912 4913 4914 4915
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
				if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
4916
							     mc.from, mc.to)) {
4917 4918 4919 4920 4921 4922 4923
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
		}
4924
		spin_unlock(ptl);
4925
		return 0;
4926 4927
	}

4928 4929
	if (pmd_trans_unstable(pmd))
		return 0;
4930 4931 4932 4933
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
4934
		swp_entry_t ent;
4935 4936 4937 4938

		if (!mc.precharge)
			break;

4939
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
4940 4941 4942 4943
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
4944
			if (!mem_cgroup_move_account(page, 1, mc.from, mc.to)) {
4945
				mc.precharge--;
4946 4947
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
4948 4949
			}
			putback_lru_page(page);
4950
put:			/* get_mctgt_type() gets the page */
4951 4952
			put_page(page);
			break;
4953 4954
		case MC_TARGET_SWAP:
			ent = target.ent;
4955
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4956
				mc.precharge--;
4957 4958 4959
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
4960
			break;
4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
4975
		ret = mem_cgroup_do_precharge(1);
4976 4977 4978 4979 4980 4981 4982 4983 4984
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
4985 4986 4987 4988
	struct mm_walk mem_cgroup_move_charge_walk = {
		.pmd_entry = mem_cgroup_move_charge_pte_range,
		.mm = mm,
	};
4989 4990

	lru_add_drain_all();
4991 4992 4993 4994 4995 4996 4997
	/*
	 * Signal mem_cgroup_begin_page_stat() to take the memcg's
	 * move_lock while we're moving its pages to another memcg.
	 * Then wait for already started RCU-only updates to finish.
	 */
	atomic_inc(&mc.from->moving_account);
	synchronize_rcu();
4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5011 5012 5013 5014 5015
	/*
	 * When we have consumed all precharges and failed in doing
	 * additional charge, the page walk just aborts.
	 */
	walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk);
5016
	up_read(&mm->mmap_sem);
5017
	atomic_dec(&mc.from->moving_account);
5018 5019
}

5020
static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
5021
				 struct cgroup_taskset *tset)
B
Balbir Singh 已提交
5022
{
5023
	struct task_struct *p = cgroup_taskset_first(tset);
5024
	struct mm_struct *mm = get_task_mm(p);
5025 5026

	if (mm) {
5027 5028
		if (mc.to)
			mem_cgroup_move_charge(mm);
5029 5030
		mmput(mm);
	}
5031 5032
	if (mc.to)
		mem_cgroup_clear_mc();
B
Balbir Singh 已提交
5033
}
5034
#else	/* !CONFIG_MMU */
5035
static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
5036
				 struct cgroup_taskset *tset)
5037 5038 5039
{
	return 0;
}
5040
static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
5041
				     struct cgroup_taskset *tset)
5042 5043
{
}
5044
static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
5045
				 struct cgroup_taskset *tset)
5046 5047 5048
{
}
#endif
B
Balbir Singh 已提交
5049

5050 5051
/*
 * Cgroup retains root cgroups across [un]mount cycles making it necessary
5052 5053
 * to verify whether we're attached to the default hierarchy on each mount
 * attempt.
5054
 */
5055
static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5056 5057
{
	/*
5058
	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5059 5060 5061
	 * guarantees that @root doesn't have any children, so turning it
	 * on for the root memcg is enough.
	 */
5062
	if (cgroup_on_dfl(root_css->cgroup))
5063 5064 5065
		root_mem_cgroup->use_hierarchy = true;
	else
		root_mem_cgroup->use_hierarchy = false;
5066 5067
}

5068 5069 5070 5071 5072 5073 5074 5075 5076
static u64 memory_current_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
{
	return mem_cgroup_usage(mem_cgroup_from_css(css), false);
}

static int memory_low_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5077
	unsigned long low = READ_ONCE(memcg->low);
5078 5079

	if (low == PAGE_COUNTER_MAX)
5080
		seq_puts(m, "max\n");
5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094
	else
		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);

	return 0;
}

static ssize_t memory_low_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long low;
	int err;

	buf = strstrip(buf);
5095
	err = page_counter_memparse(buf, "max", &low);
5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106
	if (err)
		return err;

	memcg->low = low;

	return nbytes;
}

static int memory_high_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5107
	unsigned long high = READ_ONCE(memcg->high);
5108 5109

	if (high == PAGE_COUNTER_MAX)
5110
		seq_puts(m, "max\n");
5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124
	else
		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);

	return 0;
}

static ssize_t memory_high_write(struct kernfs_open_file *of,
				 char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long high;
	int err;

	buf = strstrip(buf);
5125
	err = page_counter_memparse(buf, "max", &high);
5126 5127 5128 5129 5130
	if (err)
		return err;

	memcg->high = high;

5131
	memcg_wb_domain_size_changed(memcg);
5132 5133 5134 5135 5136 5137
	return nbytes;
}

static int memory_max_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5138
	unsigned long max = READ_ONCE(memcg->memory.limit);
5139 5140

	if (max == PAGE_COUNTER_MAX)
5141
		seq_puts(m, "max\n");
5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155
	else
		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);

	return 0;
}

static ssize_t memory_max_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long max;
	int err;

	buf = strstrip(buf);
5156
	err = page_counter_memparse(buf, "max", &max);
5157 5158 5159 5160 5161 5162 5163
	if (err)
		return err;

	err = mem_cgroup_resize_limit(memcg, max);
	if (err)
		return err;

5164
	memcg_wb_domain_size_changed(memcg);
5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210
	return nbytes;
}

static int memory_events_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));

	seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
	seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
	seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
	seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));

	return 0;
}

static struct cftype memory_files[] = {
	{
		.name = "current",
		.read_u64 = memory_current_read,
	},
	{
		.name = "low",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_low_show,
		.write = memory_low_write,
	},
	{
		.name = "high",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_high_show,
		.write = memory_high_write,
	},
	{
		.name = "max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_max_show,
		.write = memory_max_write,
	},
	{
		.name = "events",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_events_show,
	},
	{ }	/* terminate */
};

5211
struct cgroup_subsys memory_cgrp_subsys = {
5212
	.css_alloc = mem_cgroup_css_alloc,
5213
	.css_online = mem_cgroup_css_online,
5214 5215
	.css_offline = mem_cgroup_css_offline,
	.css_free = mem_cgroup_css_free,
5216
	.css_reset = mem_cgroup_css_reset,
5217 5218
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
5219
	.attach = mem_cgroup_move_task,
5220
	.bind = mem_cgroup_bind,
5221 5222
	.dfl_cftypes = memory_files,
	.legacy_cftypes = mem_cgroup_legacy_files,
5223
	.early_init = 0,
B
Balbir Singh 已提交
5224
};
5225

5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247
/**
 * mem_cgroup_low - check if memory consumption is below the normal range
 * @root: the highest ancestor to consider
 * @memcg: the memory cgroup to check
 *
 * Returns %true if memory consumption of @memcg, and that of all
 * configurable ancestors up to @root, is below the normal range.
 */
bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return false;

	/*
	 * The toplevel group doesn't have a configurable range, so
	 * it's never low when looked at directly, and it is not
	 * considered an ancestor when assessing the hierarchy.
	 */

	if (memcg == root_mem_cgroup)
		return false;

M
Michal Hocko 已提交
5248
	if (page_counter_read(&memcg->memory) >= memcg->low)
5249 5250 5251 5252 5253 5254 5255 5256
		return false;

	while (memcg != root) {
		memcg = parent_mem_cgroup(memcg);

		if (memcg == root_mem_cgroup)
			break;

M
Michal Hocko 已提交
5257
		if (page_counter_read(&memcg->memory) >= memcg->low)
5258 5259 5260 5261 5262
			return false;
	}
	return true;
}

5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297
/**
 * mem_cgroup_try_charge - try charging a page
 * @page: page to charge
 * @mm: mm context of the victim
 * @gfp_mask: reclaim mode
 * @memcgp: charged memcg return
 *
 * Try to charge @page to the memcg that @mm belongs to, reclaiming
 * pages according to @gfp_mask if necessary.
 *
 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
 * Otherwise, an error code is returned.
 *
 * After page->mapping has been set up, the caller must finalize the
 * charge with mem_cgroup_commit_charge().  Or abort the transaction
 * with mem_cgroup_cancel_charge() in case page instantiation fails.
 */
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp)
{
	struct mem_cgroup *memcg = NULL;
	unsigned int nr_pages = 1;
	int ret = 0;

	if (mem_cgroup_disabled())
		goto out;

	if (PageSwapCache(page)) {
		/*
		 * Every swap fault against a single page tries to charge the
		 * page, bail as early as possible.  shmem_unuse() encounters
		 * already charged pages, too.  The USED bit is protected by
		 * the page lock, which serializes swap cache removal, which
		 * in turn serializes uncharging.
		 */
5298
		VM_BUG_ON_PAGE(!PageLocked(page), page);
5299
		if (page->mem_cgroup)
5300
			goto out;
5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311

		if (do_swap_account) {
			swp_entry_t ent = { .val = page_private(page), };
			unsigned short id = lookup_swap_cgroup_id(ent);

			rcu_read_lock();
			memcg = mem_cgroup_from_id(id);
			if (memcg && !css_tryget_online(&memcg->css))
				memcg = NULL;
			rcu_read_unlock();
		}
5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368
	}

	if (PageTransHuge(page)) {
		nr_pages <<= compound_order(page);
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
	}

	if (!memcg)
		memcg = get_mem_cgroup_from_mm(mm);

	ret = try_charge(memcg, gfp_mask, nr_pages);

	css_put(&memcg->css);

	if (ret == -EINTR) {
		memcg = root_mem_cgroup;
		ret = 0;
	}
out:
	*memcgp = memcg;
	return ret;
}

/**
 * mem_cgroup_commit_charge - commit a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 * @lrucare: page might be on LRU already
 *
 * Finalize a charge transaction started by mem_cgroup_try_charge(),
 * after page->mapping has been set up.  This must happen atomically
 * as part of the page instantiation, i.e. under the page table lock
 * for anonymous pages, under the page lock for page and swap cache.
 *
 * In addition, the page must not be on the LRU during the commit, to
 * prevent racing with task migration.  If it might be, use @lrucare.
 *
 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
 */
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
			      bool lrucare)
{
	unsigned int nr_pages = 1;

	VM_BUG_ON_PAGE(!page->mapping, page);
	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

5369 5370
	commit_charge(page, memcg, lrucare);

5371 5372 5373 5374 5375
	if (PageTransHuge(page)) {
		nr_pages <<= compound_order(page);
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
	}

5376 5377 5378 5379
	local_irq_disable();
	mem_cgroup_charge_statistics(memcg, page, nr_pages);
	memcg_check_events(memcg, page);
	local_irq_enable();
5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420

	if (do_swap_account && PageSwapCache(page)) {
		swp_entry_t entry = { .val = page_private(page) };
		/*
		 * The swap entry might not get freed for a long time,
		 * let's not wait for it.  The page already received a
		 * memory+swap charge, drop the swap entry duplicate.
		 */
		mem_cgroup_uncharge_swap(entry);
	}
}

/**
 * mem_cgroup_cancel_charge - cancel a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 *
 * Cancel a charge transaction started by mem_cgroup_try_charge().
 */
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)
{
	unsigned int nr_pages = 1;

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

	if (PageTransHuge(page)) {
		nr_pages <<= compound_order(page);
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
	}

	cancel_charge(memcg, nr_pages);
}

5421 5422 5423 5424
static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
			   unsigned long nr_anon, unsigned long nr_file,
			   unsigned long nr_huge, struct page *dummy_page)
{
5425
	unsigned long nr_pages = nr_anon + nr_file;
5426 5427
	unsigned long flags;

5428
	if (!mem_cgroup_is_root(memcg)) {
5429 5430 5431
		page_counter_uncharge(&memcg->memory, nr_pages);
		if (do_swap_account)
			page_counter_uncharge(&memcg->memsw, nr_pages);
5432 5433
		memcg_oom_recover(memcg);
	}
5434 5435 5436 5437 5438 5439

	local_irq_save(flags);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
	__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5440
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5441 5442
	memcg_check_events(memcg, dummy_page);
	local_irq_restore(flags);
5443 5444

	if (!mem_cgroup_is_root(memcg))
5445
		css_put_many(&memcg->css, nr_pages);
5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467
}

static void uncharge_list(struct list_head *page_list)
{
	struct mem_cgroup *memcg = NULL;
	unsigned long nr_anon = 0;
	unsigned long nr_file = 0;
	unsigned long nr_huge = 0;
	unsigned long pgpgout = 0;
	struct list_head *next;
	struct page *page;

	next = page_list->next;
	do {
		unsigned int nr_pages = 1;

		page = list_entry(next, struct page, lru);
		next = page->lru.next;

		VM_BUG_ON_PAGE(PageLRU(page), page);
		VM_BUG_ON_PAGE(page_count(page), page);

5468
		if (!page->mem_cgroup)
5469 5470 5471 5472
			continue;

		/*
		 * Nobody should be changing or seriously looking at
5473
		 * page->mem_cgroup at this point, we have fully
5474
		 * exclusive access to the page.
5475 5476
		 */

5477
		if (memcg != page->mem_cgroup) {
5478
			if (memcg) {
5479 5480 5481
				uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
					       nr_huge, page);
				pgpgout = nr_anon = nr_file = nr_huge = 0;
5482
			}
5483
			memcg = page->mem_cgroup;
5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496
		}

		if (PageTransHuge(page)) {
			nr_pages <<= compound_order(page);
			VM_BUG_ON_PAGE(!PageTransHuge(page), page);
			nr_huge += nr_pages;
		}

		if (PageAnon(page))
			nr_anon += nr_pages;
		else
			nr_file += nr_pages;

5497
		page->mem_cgroup = NULL;
5498 5499 5500 5501 5502

		pgpgout++;
	} while (next != page_list);

	if (memcg)
5503 5504
		uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
			       nr_huge, page);
5505 5506
}

5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518
/**
 * mem_cgroup_uncharge - uncharge a page
 * @page: page to uncharge
 *
 * Uncharge a page previously charged with mem_cgroup_try_charge() and
 * mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge(struct page *page)
{
	if (mem_cgroup_disabled())
		return;

5519
	/* Don't touch page->lru of any random page, pre-check: */
5520
	if (!page->mem_cgroup)
5521 5522
		return;

5523 5524 5525
	INIT_LIST_HEAD(&page->lru);
	uncharge_list(&page->lru);
}
5526

5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537
/**
 * mem_cgroup_uncharge_list - uncharge a list of page
 * @page_list: list of pages to uncharge
 *
 * Uncharge a list of pages previously charged with
 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge_list(struct list_head *page_list)
{
	if (mem_cgroup_disabled())
		return;
5538

5539 5540
	if (!list_empty(page_list))
		uncharge_list(page_list);
5541 5542 5543 5544 5545 5546
}

/**
 * mem_cgroup_migrate - migrate a charge to another page
 * @oldpage: currently charged page
 * @newpage: page to transfer the charge to
5547
 * @lrucare: either or both pages might be on the LRU already
5548 5549 5550 5551 5552 5553 5554 5555
 *
 * Migrate the charge from @oldpage to @newpage.
 *
 * Both pages must be locked, @newpage->mapping must be set up.
 */
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage,
			bool lrucare)
{
5556
	struct mem_cgroup *memcg;
5557 5558 5559 5560 5561 5562 5563
	int isolated;

	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
	VM_BUG_ON_PAGE(!lrucare && PageLRU(oldpage), oldpage);
	VM_BUG_ON_PAGE(!lrucare && PageLRU(newpage), newpage);
	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5564 5565
	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
		       newpage);
5566 5567 5568 5569 5570

	if (mem_cgroup_disabled())
		return;

	/* Page cache replacement: new page already charged? */
5571
	if (newpage->mem_cgroup)
5572 5573
		return;

5574 5575 5576 5577 5578 5579
	/*
	 * Swapcache readahead pages can get migrated before being
	 * charged, and migration from compaction can happen to an
	 * uncharged page when the PFN walker finds a page that
	 * reclaim just put back on the LRU but has not released yet.
	 */
5580
	memcg = oldpage->mem_cgroup;
5581
	if (!memcg)
5582 5583 5584 5585 5586
		return;

	if (lrucare)
		lock_page_lru(oldpage, &isolated);

5587
	oldpage->mem_cgroup = NULL;
5588 5589 5590 5591

	if (lrucare)
		unlock_page_lru(oldpage, isolated);

5592
	commit_charge(newpage, memcg, lrucare);
5593 5594
}

5595
/*
5596 5597 5598 5599 5600 5601
 * subsys_initcall() for memory controller.
 *
 * Some parts like hotcpu_notifier() have to be initialized from this context
 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
 * everything that doesn't depend on a specific mem_cgroup structure should
 * be initialized from here.
5602 5603 5604
 */
static int __init mem_cgroup_init(void)
{
5605 5606
	int cpu, node;

5607
	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629

	for_each_possible_cpu(cpu)
		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
			  drain_local_stock);

	for_each_node(node) {
		struct mem_cgroup_tree_per_node *rtpn;
		int zone;

		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
				    node_online(node) ? node : NUMA_NO_NODE);

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			struct mem_cgroup_tree_per_zone *rtpz;

			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
		soft_limit_tree.rb_tree_per_node[node] = rtpn;
	}

5630 5631 5632
	return 0;
}
subsys_initcall(mem_cgroup_init);
5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667

#ifdef CONFIG_MEMCG_SWAP
/**
 * mem_cgroup_swapout - transfer a memsw charge to swap
 * @page: page whose memsw charge to transfer
 * @entry: swap entry to move the charge to
 *
 * Transfer the memsw charge of @page to @entry.
 */
void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
{
	struct mem_cgroup *memcg;
	unsigned short oldid;

	VM_BUG_ON_PAGE(PageLRU(page), page);
	VM_BUG_ON_PAGE(page_count(page), page);

	if (!do_swap_account)
		return;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return;

	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
	VM_BUG_ON_PAGE(oldid, page);
	mem_cgroup_swap_statistics(memcg, true);

	page->mem_cgroup = NULL;

	if (!mem_cgroup_is_root(memcg))
		page_counter_uncharge(&memcg->memory, 1);

5668 5669 5670 5671 5672 5673 5674
	/*
	 * Interrupts should be disabled here because the caller holds the
	 * mapping->tree_lock lock which is taken with interrupts-off. It is
	 * important here to have the interrupts disabled because it is the
	 * only synchronisation we have for udpating the per-CPU variables.
	 */
	VM_BUG_ON(!irqs_disabled());
5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694
	mem_cgroup_charge_statistics(memcg, page, -1);
	memcg_check_events(memcg, page);
}

/**
 * mem_cgroup_uncharge_swap - uncharge a swap entry
 * @entry: swap entry to uncharge
 *
 * Drop the memsw charge associated with @entry.
 */
void mem_cgroup_uncharge_swap(swp_entry_t entry)
{
	struct mem_cgroup *memcg;
	unsigned short id;

	if (!do_swap_account)
		return;

	id = swap_cgroup_record(entry, 0);
	rcu_read_lock();
5695
	memcg = mem_cgroup_from_id(id);
5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760
	if (memcg) {
		if (!mem_cgroup_is_root(memcg))
			page_counter_uncharge(&memcg->memsw, 1);
		mem_cgroup_swap_statistics(memcg, false);
		css_put(&memcg->css);
	}
	rcu_read_unlock();
}

/* for remember boot option*/
#ifdef CONFIG_MEMCG_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata;
#endif

static int __init enable_swap_account(char *s)
{
	if (!strcmp(s, "1"))
		really_do_swap_account = 1;
	else if (!strcmp(s, "0"))
		really_do_swap_account = 0;
	return 1;
}
__setup("swapaccount=", enable_swap_account);

static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{ },	/* terminate */
};

static int __init mem_cgroup_swap_init(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
						  memsw_cgroup_files));
	}
	return 0;
}
subsys_initcall(mem_cgroup_swap_init);

#endif /* CONFIG_MEMCG_SWAP */