memcontrol.c 172.1 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
13 14 15 16
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
17 18 19 20 21 22
 * Native page reclaim
 * Charge lifetime sanitation
 * Lockless page tracking & accounting
 * Unified hierarchy configuration model
 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
 *
B
Balbir Singh 已提交
23 24 25 26 27 28 29 30 31 32 33
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

34
#include <linux/page_counter.h>
B
Balbir Singh 已提交
35 36
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
37
#include <linux/mm.h>
38
#include <linux/sched/mm.h>
39
#include <linux/shmem_fs.h>
40
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
41
#include <linux/pagemap.h>
42
#include <linux/smp.h>
43
#include <linux/page-flags.h>
44
#include <linux/backing-dev.h>
45 46
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
47
#include <linux/limits.h>
48
#include <linux/export.h>
49
#include <linux/mutex.h>
50
#include <linux/rbtree.h>
51
#include <linux/slab.h>
52
#include <linux/swap.h>
53
#include <linux/swapops.h>
54
#include <linux/spinlock.h>
55
#include <linux/eventfd.h>
56
#include <linux/poll.h>
57
#include <linux/sort.h>
58
#include <linux/fs.h>
59
#include <linux/seq_file.h>
60
#include <linux/vmpressure.h>
61
#include <linux/mm_inline.h>
62
#include <linux/swap_cgroup.h>
63
#include <linux/cpu.h>
64
#include <linux/oom.h>
65
#include <linux/lockdep.h>
66
#include <linux/file.h>
67
#include <linux/tracehook.h>
K
KAMEZAWA Hiroyuki 已提交
68
#include "internal.h"
G
Glauber Costa 已提交
69
#include <net/sock.h>
M
Michal Hocko 已提交
70
#include <net/ip.h>
71
#include "slab.h"
B
Balbir Singh 已提交
72

73
#include <linux/uaccess.h>
74

75 76
#include <trace/events/vmscan.h>

77 78
struct cgroup_subsys memory_cgrp_subsys __read_mostly;
EXPORT_SYMBOL(memory_cgrp_subsys);
79

80 81
struct mem_cgroup *root_mem_cgroup __read_mostly;

82
#define MEM_CGROUP_RECLAIM_RETRIES	5
B
Balbir Singh 已提交
83

84 85 86
/* Socket memory accounting disabled? */
static bool cgroup_memory_nosocket;

87 88 89
/* Kernel memory accounting disabled? */
static bool cgroup_memory_nokmem;

90
/* Whether the swap controller is active */
A
Andrew Morton 已提交
91
#ifdef CONFIG_MEMCG_SWAP
92 93
int do_swap_account __read_mostly;
#else
94
#define do_swap_account		0
95 96
#endif

97 98 99 100 101 102
/* Whether legacy memory+swap accounting is active */
static bool do_memsw_account(void)
{
	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
}

103
static const char *const mem_cgroup_lru_names[] = {
104 105 106 107 108 109 110
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

111 112 113
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
114

115 116 117 118 119
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

120
struct mem_cgroup_tree_per_node {
121
	struct rb_root rb_root;
122
	struct rb_node *rb_rightmost;
123 124 125 126 127 128 129 130 131
	spinlock_t lock;
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

K
KAMEZAWA Hiroyuki 已提交
132 133 134 135 136
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
137

138 139 140
/*
 * cgroup_event represents events which userspace want to receive.
 */
141
struct mem_cgroup_event {
142
	/*
143
	 * memcg which the event belongs to.
144
	 */
145
	struct mem_cgroup *memcg;
146 147 148 149 150 151 152 153
	/*
	 * eventfd to signal userspace about the event.
	 */
	struct eventfd_ctx *eventfd;
	/*
	 * Each of these stored in a list by the cgroup.
	 */
	struct list_head list;
154 155 156 157 158
	/*
	 * register_event() callback will be used to add new userspace
	 * waiter for changes related to this event.  Use eventfd_signal()
	 * on eventfd to send notification to userspace.
	 */
159
	int (*register_event)(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
160
			      struct eventfd_ctx *eventfd, const char *args);
161 162 163 164 165
	/*
	 * unregister_event() callback will be called when userspace closes
	 * the eventfd or on cgroup removing.  This callback must be set,
	 * if you want provide notification functionality.
	 */
166
	void (*unregister_event)(struct mem_cgroup *memcg,
167
				 struct eventfd_ctx *eventfd);
168 169 170 171 172 173
	/*
	 * All fields below needed to unregister event when
	 * userspace closes eventfd.
	 */
	poll_table pt;
	wait_queue_head_t *wqh;
174
	wait_queue_entry_t wait;
175 176 177
	struct work_struct remove;
};

178 179
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
180

181 182
/* Stuffs for move charges at task migration. */
/*
183
 * Types of charges to be moved.
184
 */
185 186 187
#define MOVE_ANON	0x1U
#define MOVE_FILE	0x2U
#define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
188

189 190
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
191
	spinlock_t	  lock; /* for from, to */
192
	struct mm_struct  *mm;
193 194
	struct mem_cgroup *from;
	struct mem_cgroup *to;
195
	unsigned long flags;
196
	unsigned long precharge;
197
	unsigned long moved_charge;
198
	unsigned long moved_swap;
199 200 201
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
202
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
203 204
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
205

206 207 208 209
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
210
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
211
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
212

213 214
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
215
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
216
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
217
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
218 219 220
	NR_CHARGE_TYPE,
};

221
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
222 223 224 225
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
226
	_KMEM,
V
Vladimir Davydov 已提交
227
	_TCP,
G
Glauber Costa 已提交
228 229
};

230 231
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
232
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
233 234
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
235

236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
	     iter != NULL;				\
	     iter = mem_cgroup_iter(root, iter, NULL))

#define for_each_mem_cgroup(iter)			\
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
	     iter != NULL;				\
	     iter = mem_cgroup_iter(NULL, iter, NULL))

251 252 253 254 255 256
static inline bool should_force_charge(void)
{
	return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
		(current->flags & PF_EXITING);
}

257 258 259 260 261 262 263 264 265 266 267 268 269
/* Some nice accessors for the vmpressure. */
struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
{
	if (!memcg)
		memcg = root_mem_cgroup;
	return &memcg->vmpressure;
}

struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
{
	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}

270
#ifdef CONFIG_MEMCG_KMEM
271
/*
272
 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
L
Li Zefan 已提交
273 274 275 276 277
 * The main reason for not using cgroup id for this:
 *  this works better in sparse environments, where we have a lot of memcgs,
 *  but only a few kmem-limited. Or also, if we have, for instance, 200
 *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *  200 entry array for that.
278
 *
279 280
 * The current size of the caches array is stored in memcg_nr_cache_ids. It
 * will double each time we have to increase it.
281
 */
282 283
static DEFINE_IDA(memcg_cache_ida);
int memcg_nr_cache_ids;
284

285 286 287 288 289 290 291 292 293 294 295 296 297
/* Protects memcg_nr_cache_ids */
static DECLARE_RWSEM(memcg_cache_ids_sem);

void memcg_get_cache_ids(void)
{
	down_read(&memcg_cache_ids_sem);
}

void memcg_put_cache_ids(void)
{
	up_read(&memcg_cache_ids_sem);
}

298 299 300 301 302 303
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
L
Li Zefan 已提交
304
 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
305 306
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
L
Li Zefan 已提交
307
 * cgrp_id space is not getting any smaller, and we don't have to necessarily
308 309 310
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
L
Li Zefan 已提交
311
#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
312

313 314 315 316 317 318
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
319
DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
320
EXPORT_SYMBOL(memcg_kmem_enabled_key);
321

322 323
struct workqueue_struct *memcg_kmem_cache_wq;

324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
static int memcg_shrinker_map_size;
static DEFINE_MUTEX(memcg_shrinker_map_mutex);

static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
{
	kvfree(container_of(head, struct memcg_shrinker_map, rcu));
}

static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
					 int size, int old_size)
{
	struct memcg_shrinker_map *new, *old;
	int nid;

	lockdep_assert_held(&memcg_shrinker_map_mutex);

	for_each_node(nid) {
		old = rcu_dereference_protected(
			mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
		/* Not yet online memcg */
		if (!old)
			return 0;

		new = kvmalloc(sizeof(*new) + size, GFP_KERNEL);
		if (!new)
			return -ENOMEM;

		/* Set all old bits, clear all new bits */
		memset(new->map, (int)0xff, old_size);
		memset((void *)new->map + old_size, 0, size - old_size);

		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
	}

	return 0;
}

static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_node *pn;
	struct memcg_shrinker_map *map;
	int nid;

	if (mem_cgroup_is_root(memcg))
		return;

	for_each_node(nid) {
		pn = mem_cgroup_nodeinfo(memcg, nid);
		map = rcu_dereference_protected(pn->shrinker_map, true);
		if (map)
			kvfree(map);
		rcu_assign_pointer(pn->shrinker_map, NULL);
	}
}

static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
{
	struct memcg_shrinker_map *map;
	int nid, size, ret = 0;

	if (mem_cgroup_is_root(memcg))
		return 0;

	mutex_lock(&memcg_shrinker_map_mutex);
	size = memcg_shrinker_map_size;
	for_each_node(nid) {
		map = kvzalloc(sizeof(*map) + size, GFP_KERNEL);
		if (!map) {
			memcg_free_shrinker_maps(memcg);
			ret = -ENOMEM;
			break;
		}
		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
	}
	mutex_unlock(&memcg_shrinker_map_mutex);

	return ret;
}

int memcg_expand_shrinker_maps(int new_id)
{
	int size, old_size, ret = 0;
	struct mem_cgroup *memcg;

	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
	old_size = memcg_shrinker_map_size;
	if (size <= old_size)
		return 0;

	mutex_lock(&memcg_shrinker_map_mutex);
	if (!root_mem_cgroup)
		goto unlock;

	for_each_mem_cgroup(memcg) {
		if (mem_cgroup_is_root(memcg))
			continue;
		ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
		if (ret)
			goto unlock;
	}
unlock:
	if (!ret)
		memcg_shrinker_map_size = size;
	mutex_unlock(&memcg_shrinker_map_mutex);
	return ret;
}
431 432 433 434 435 436 437 438

void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
{
	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
		struct memcg_shrinker_map *map;

		rcu_read_lock();
		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
439 440
		/* Pairs with smp mb in shrink_slab() */
		smp_mb__before_atomic();
441 442 443 444 445
		set_bit(shrinker_id, map->map);
		rcu_read_unlock();
	}
}

446 447 448 449 450 451
#else /* CONFIG_MEMCG_KMEM */
static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
{
	return 0;
}
static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { }
452
#endif /* CONFIG_MEMCG_KMEM */
453

454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
/**
 * mem_cgroup_css_from_page - css of the memcg associated with a page
 * @page: page of interest
 *
 * If memcg is bound to the default hierarchy, css of the memcg associated
 * with @page is returned.  The returned css remains associated with @page
 * until it is released.
 *
 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
 * is returned.
 */
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
{
	struct mem_cgroup *memcg;

	memcg = page->mem_cgroup;

471
	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
472 473 474 475 476
		memcg = root_mem_cgroup;

	return &memcg->css;
}

477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
/**
 * page_cgroup_ino - return inode number of the memcg a page is charged to
 * @page: the page
 *
 * Look up the closest online ancestor of the memory cgroup @page is charged to
 * and return its inode number or 0 if @page is not charged to any cgroup. It
 * is safe to call this function without holding a reference to @page.
 *
 * Note, this function is inherently racy, because there is nothing to prevent
 * the cgroup inode from getting torn down and potentially reallocated a moment
 * after page_cgroup_ino() returns, so it only should be used by callers that
 * do not care (such as procfs interfaces).
 */
ino_t page_cgroup_ino(struct page *page)
{
	struct mem_cgroup *memcg;
	unsigned long ino = 0;

	rcu_read_lock();
	memcg = READ_ONCE(page->mem_cgroup);
	while (memcg && !(memcg->css.flags & CSS_ONLINE))
		memcg = parent_mem_cgroup(memcg);
	if (memcg)
		ino = cgroup_ino(memcg->css.cgroup);
	rcu_read_unlock();
	return ino;
}

505 506
static struct mem_cgroup_per_node *
mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
507
{
508
	int nid = page_to_nid(page);
509

510
	return memcg->nodeinfo[nid];
511 512
}

513 514
static struct mem_cgroup_tree_per_node *
soft_limit_tree_node(int nid)
515
{
516
	return soft_limit_tree.rb_tree_per_node[nid];
517 518
}

519
static struct mem_cgroup_tree_per_node *
520 521 522 523
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);

524
	return soft_limit_tree.rb_tree_per_node[nid];
525 526
}

527 528
static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
					 struct mem_cgroup_tree_per_node *mctz,
529
					 unsigned long new_usage_in_excess)
530 531 532
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
533
	struct mem_cgroup_per_node *mz_node;
534
	bool rightmost = true;
535 536 537 538 539 540 541 542 543

	if (mz->on_tree)
		return;

	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
	while (*p) {
		parent = *p;
544
		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
545
					tree_node);
546
		if (mz->usage_in_excess < mz_node->usage_in_excess) {
547
			p = &(*p)->rb_left;
548 549 550
			rightmost = false;
		}

551 552 553 554 555 556 557
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
558 559 560 561

	if (rightmost)
		mctz->rb_rightmost = &mz->tree_node;

562 563 564 565 566
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
}

567 568
static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
					 struct mem_cgroup_tree_per_node *mctz)
569 570 571
{
	if (!mz->on_tree)
		return;
572 573 574 575

	if (&mz->tree_node == mctz->rb_rightmost)
		mctz->rb_rightmost = rb_prev(&mz->tree_node);

576 577 578 579
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

580 581
static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
				       struct mem_cgroup_tree_per_node *mctz)
582
{
583 584 585
	unsigned long flags;

	spin_lock_irqsave(&mctz->lock, flags);
586
	__mem_cgroup_remove_exceeded(mz, mctz);
587
	spin_unlock_irqrestore(&mctz->lock, flags);
588 589
}

590 591 592
static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
{
	unsigned long nr_pages = page_counter_read(&memcg->memory);
593
	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
594 595 596 597 598 599 600
	unsigned long excess = 0;

	if (nr_pages > soft_limit)
		excess = nr_pages - soft_limit;

	return excess;
}
601 602 603

static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
{
604
	unsigned long excess;
605 606
	struct mem_cgroup_per_node *mz;
	struct mem_cgroup_tree_per_node *mctz;
607

608
	mctz = soft_limit_tree_from_page(page);
609 610
	if (!mctz)
		return;
611 612 613 614 615
	/*
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
616
		mz = mem_cgroup_page_nodeinfo(memcg, page);
617
		excess = soft_limit_excess(memcg);
618 619 620 621 622
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
		if (excess || mz->on_tree) {
623 624 625
			unsigned long flags;

			spin_lock_irqsave(&mctz->lock, flags);
626 627
			/* if on-tree, remove it */
			if (mz->on_tree)
628
				__mem_cgroup_remove_exceeded(mz, mctz);
629 630 631 632
			/*
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
			 */
633
			__mem_cgroup_insert_exceeded(mz, mctz, excess);
634
			spin_unlock_irqrestore(&mctz->lock, flags);
635 636 637 638 639 640
		}
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
{
641 642 643
	struct mem_cgroup_tree_per_node *mctz;
	struct mem_cgroup_per_node *mz;
	int nid;
644

645
	for_each_node(nid) {
646 647
		mz = mem_cgroup_nodeinfo(memcg, nid);
		mctz = soft_limit_tree_node(nid);
648 649
		if (mctz)
			mem_cgroup_remove_exceeded(mz, mctz);
650 651 652
	}
}

653 654
static struct mem_cgroup_per_node *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
655
{
656
	struct mem_cgroup_per_node *mz;
657 658 659

retry:
	mz = NULL;
660
	if (!mctz->rb_rightmost)
661 662
		goto done;		/* Nothing to reclaim from */

663 664
	mz = rb_entry(mctz->rb_rightmost,
		      struct mem_cgroup_per_node, tree_node);
665 666 667 668 669
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
670
	__mem_cgroup_remove_exceeded(mz, mctz);
671
	if (!soft_limit_excess(mz->memcg) ||
672
	    !css_tryget_online(&mz->memcg->css))
673 674 675 676 677
		goto retry;
done:
	return mz;
}

678 679
static struct mem_cgroup_per_node *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
680
{
681
	struct mem_cgroup_per_node *mz;
682

683
	spin_lock_irq(&mctz->lock);
684
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
685
	spin_unlock_irq(&mctz->lock);
686 687 688
	return mz;
}

689
static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
690
				      int event)
691
{
692
	return atomic_long_read(&memcg->events[event]);
693 694
}

695
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
696
					 struct page *page,
697
					 bool compound, int nr_pages)
698
{
699 700 701 702
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
703
	if (PageAnon(page))
704
		__mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
705
	else {
706
		__mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
707
		if (PageSwapBacked(page))
708
			__mod_memcg_state(memcg, NR_SHMEM, nr_pages);
709
	}
710

711 712
	if (compound) {
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
713
		__mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
714
	}
715

716 717
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
718
		__count_memcg_events(memcg, PGPGIN, 1);
719
	else {
720
		__count_memcg_events(memcg, PGPGOUT, 1);
721 722
		nr_pages = -nr_pages; /* for event */
	}
723

724
	__this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages);
725 726
}

727 728
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
					   int nid, unsigned int lru_mask)
729
{
730
	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
731
	unsigned long nr = 0;
732
	enum lru_list lru;
733

734
	VM_BUG_ON((unsigned)nid >= nr_node_ids);
735

736 737 738
	for_each_lru(lru) {
		if (!(BIT(lru) & lru_mask))
			continue;
739
		nr += mem_cgroup_get_lru_size(lruvec, lru);
740 741
	}
	return nr;
742
}
743

744
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
745
			unsigned int lru_mask)
746
{
747
	unsigned long nr = 0;
748
	int nid;
749

750
	for_each_node_state(nid, N_MEMORY)
751 752
		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
	return nr;
753 754
}

755 756
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
757 758 759
{
	unsigned long val, next;

760 761
	val = __this_cpu_read(memcg->stat_cpu->nr_page_events);
	next = __this_cpu_read(memcg->stat_cpu->targets[target]);
762
	/* from time_after() in jiffies.h */
763
	if ((long)(next - val) < 0) {
764 765 766 767
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
768 769 770
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
771 772 773 774 775 776
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
777
		__this_cpu_write(memcg->stat_cpu->targets[target], next);
778
		return true;
779
	}
780
	return false;
781 782 783 784 785 786
}

/*
 * Check events in order.
 *
 */
787
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
788 789
{
	/* threshold event is triggered in finer grain than soft limit */
790 791
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
792
		bool do_softlimit;
793
		bool do_numainfo __maybe_unused;
794

795 796
		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
797 798 799 800
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
801
		mem_cgroup_threshold(memcg);
802 803
		if (unlikely(do_softlimit))
			mem_cgroup_update_tree(memcg, page);
804
#if MAX_NUMNODES > 1
805
		if (unlikely(do_numainfo))
806
			atomic_inc(&memcg->numainfo_events);
807
#endif
808
	}
809 810
}

811
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
812
{
813 814 815 816 817 818 819 820
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

821
	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
822
}
M
Michal Hocko 已提交
823
EXPORT_SYMBOL(mem_cgroup_from_task);
824

825 826 827 828 829 830 831 832 833
/**
 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
 * @mm: mm from which memcg should be extracted. It can be NULL.
 *
 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
 * returned.
 */
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
834
{
835 836 837 838
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return NULL;
839

840 841
	rcu_read_lock();
	do {
842 843 844 845 846 847
		/*
		 * Page cache insertions can happen withou an
		 * actual mm context, e.g. during disk probing
		 * on boot, loopback IO, acct() writes etc.
		 */
		if (unlikely(!mm))
848
			memcg = root_mem_cgroup;
849 850 851 852 853
		else {
			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
			if (unlikely(!memcg))
				memcg = root_mem_cgroup;
		}
854
	} while (!css_tryget(&memcg->css));
855
	rcu_read_unlock();
856
	return memcg;
857
}
858 859
EXPORT_SYMBOL(get_mem_cgroup_from_mm);

860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
/**
 * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
 * @page: page from which memcg should be extracted.
 *
 * Obtain a reference on page->memcg and returns it if successful. Otherwise
 * root_mem_cgroup is returned.
 */
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
	struct mem_cgroup *memcg = page->mem_cgroup;

	if (mem_cgroup_disabled())
		return NULL;

	rcu_read_lock();
	if (!memcg || !css_tryget_online(&memcg->css))
		memcg = root_mem_cgroup;
	rcu_read_unlock();
	return memcg;
}
EXPORT_SYMBOL(get_mem_cgroup_from_page);

882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
/**
 * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
 */
static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
{
	if (unlikely(current->active_memcg)) {
		struct mem_cgroup *memcg = root_mem_cgroup;

		rcu_read_lock();
		if (css_tryget_online(&current->active_memcg->css))
			memcg = current->active_memcg;
		rcu_read_unlock();
		return memcg;
	}
	return get_mem_cgroup_from_mm(current->mm);
}
898

899 900 901 902 903 904 905 906 907 908 909 910 911
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
912
 * Reclaimers can specify a node and a priority level in @reclaim to
913
 * divide up the memcgs in the hierarchy among all concurrent
914
 * reclaimers operating on the same node and priority.
915
 */
916
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
917
				   struct mem_cgroup *prev,
918
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
919
{
M
Michal Hocko 已提交
920
	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
921
	struct cgroup_subsys_state *css = NULL;
922
	struct mem_cgroup *memcg = NULL;
923
	struct mem_cgroup *pos = NULL;
924

925 926
	if (mem_cgroup_disabled())
		return NULL;
927

928 929
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
930

931
	if (prev && !reclaim)
932
		pos = prev;
K
KAMEZAWA Hiroyuki 已提交
933

934 935
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
936
			goto out;
937
		return root;
938
	}
K
KAMEZAWA Hiroyuki 已提交
939

940
	rcu_read_lock();
M
Michal Hocko 已提交
941

942
	if (reclaim) {
943
		struct mem_cgroup_per_node *mz;
944

945
		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
946 947 948 949 950
		iter = &mz->iter[reclaim->priority];

		if (prev && reclaim->generation != iter->generation)
			goto out_unlock;

951
		while (1) {
952
			pos = READ_ONCE(iter->position);
953 954
			if (!pos || css_tryget(&pos->css))
				break;
955
			/*
956 957 958 959 960 961
			 * css reference reached zero, so iter->position will
			 * be cleared by ->css_released. However, we should not
			 * rely on this happening soon, because ->css_released
			 * is called from a work queue, and by busy-waiting we
			 * might block it. So we clear iter->position right
			 * away.
962
			 */
963 964
			(void)cmpxchg(&iter->position, pos, NULL);
		}
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981
	}

	if (pos)
		css = &pos->css;

	for (;;) {
		css = css_next_descendant_pre(css, &root->css);
		if (!css) {
			/*
			 * Reclaimers share the hierarchy walk, and a
			 * new one might jump in right at the end of
			 * the hierarchy - make sure they see at least
			 * one group and restart from the beginning.
			 */
			if (!prev)
				continue;
			break;
982
		}
K
KAMEZAWA Hiroyuki 已提交
983

984 985 986 987 988 989
		/*
		 * Verify the css and acquire a reference.  The root
		 * is provided by the caller, so we know it's alive
		 * and kicking, and don't take an extra reference.
		 */
		memcg = mem_cgroup_from_css(css);
K
KAMEZAWA Hiroyuki 已提交
990

991 992
		if (css == &root->css)
			break;
K
KAMEZAWA Hiroyuki 已提交
993

994 995
		if (css_tryget(css))
			break;
996

997
		memcg = NULL;
998
	}
999 1000 1001

	if (reclaim) {
		/*
1002 1003 1004
		 * The position could have already been updated by a competing
		 * thread, so check that the value hasn't changed since we read
		 * it to avoid reclaiming from the same cgroup twice.
1005
		 */
1006 1007
		(void)cmpxchg(&iter->position, pos, memcg);

1008 1009 1010 1011 1012 1013 1014
		if (pos)
			css_put(&pos->css);

		if (!memcg)
			iter->generation++;
		else if (!prev)
			reclaim->generation = iter->generation;
1015
	}
1016

1017 1018
out_unlock:
	rcu_read_unlock();
1019
out:
1020 1021 1022
	if (prev && prev != root)
		css_put(&prev->css);

1023
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
1024
}
K
KAMEZAWA Hiroyuki 已提交
1025

1026 1027 1028 1029 1030 1031 1032
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
1033 1034 1035 1036 1037 1038
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
1039

1040 1041
static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
					struct mem_cgroup *dead_memcg)
1042 1043
{
	struct mem_cgroup_reclaim_iter *iter;
1044 1045
	struct mem_cgroup_per_node *mz;
	int nid;
1046 1047
	int i;

1048 1049 1050 1051 1052 1053
	for_each_node(nid) {
		mz = mem_cgroup_nodeinfo(from, nid);
		for (i = 0; i <= DEF_PRIORITY; i++) {
			iter = &mz->iter[i];
			cmpxchg(&iter->position,
				dead_memcg, NULL);
1054 1055 1056 1057
		}
	}
}

1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
{
	struct mem_cgroup *memcg = dead_memcg;
	struct mem_cgroup *last;

	do {
		__invalidate_reclaim_iterators(memcg, dead_memcg);
		last = memcg;
	} while ((memcg = parent_mem_cgroup(memcg)));

	/*
	 * When cgruop1 non-hierarchy mode is used,
	 * parent_mem_cgroup() does not walk all the way up to the
	 * cgroup root (root_mem_cgroup). So we have to handle
	 * dead_memcg from cgroup root separately.
	 */
	if (last != root_mem_cgroup)
		__invalidate_reclaim_iterators(root_mem_cgroup,
						dead_memcg);
}

1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
/**
 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
 * @memcg: hierarchy root
 * @fn: function to call for each task
 * @arg: argument passed to @fn
 *
 * This function iterates over tasks attached to @memcg or to any of its
 * descendants and calls @fn for each task. If @fn returns a non-zero
 * value, the function breaks the iteration loop and returns the value.
 * Otherwise, it will iterate over all tasks and return 0.
 *
 * This function must not be called for the root memory cgroup.
 */
int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
			  int (*fn)(struct task_struct *, void *), void *arg)
{
	struct mem_cgroup *iter;
	int ret = 0;

	BUG_ON(memcg == root_mem_cgroup);

	for_each_mem_cgroup_tree(iter, memcg) {
		struct css_task_iter it;
		struct task_struct *task;

1104
		css_task_iter_start(&iter->css, 0, &it);
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
		while (!ret && (task = css_task_iter_next(&it)))
			ret = fn(task, arg);
		css_task_iter_end(&it);
		if (ret) {
			mem_cgroup_iter_break(memcg, iter);
			break;
		}
	}
	return ret;
}

1116
/**
1117
 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1118
 * @page: the page
1119
 * @pgdat: pgdat of the page
1120 1121 1122 1123
 *
 * This function is only safe when following the LRU page isolation
 * and putback protocol: the LRU lock must be held, and the page must
 * either be PageLRU() or the caller must have isolated/allocated it.
1124
 */
M
Mel Gorman 已提交
1125
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
K
KAMEZAWA Hiroyuki 已提交
1126
{
1127
	struct mem_cgroup_per_node *mz;
1128
	struct mem_cgroup *memcg;
1129
	struct lruvec *lruvec;
1130

1131
	if (mem_cgroup_disabled()) {
M
Mel Gorman 已提交
1132
		lruvec = &pgdat->lruvec;
1133 1134
		goto out;
	}
1135

1136
	memcg = page->mem_cgroup;
1137
	/*
1138
	 * Swapcache readahead pages are added to the LRU - and
1139
	 * possibly migrated - before they are charged.
1140
	 */
1141 1142
	if (!memcg)
		memcg = root_mem_cgroup;
1143

1144
	mz = mem_cgroup_page_nodeinfo(memcg, page);
1145 1146 1147 1148 1149 1150 1151
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
M
Mel Gorman 已提交
1152 1153
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
1154
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1155
}
1156

1157
/**
1158 1159 1160
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
1161
 * @zid: zone id of the accounted pages
1162
 * @nr_pages: positive when adding or negative when removing
1163
 *
1164 1165 1166
 * This function must be called under lru_lock, just before a page is added
 * to or just after a page is removed from an lru list (that ordering being
 * so as to allow it to check that lru_size 0 is consistent with list_empty).
1167
 */
1168
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1169
				int zid, int nr_pages)
1170
{
1171
	struct mem_cgroup_per_node *mz;
1172
	unsigned long *lru_size;
1173
	long size;
1174 1175 1176 1177

	if (mem_cgroup_disabled())
		return;

1178
	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1179
	lru_size = &mz->lru_zone_size[zid][lru];
1180 1181 1182 1183 1184

	if (nr_pages < 0)
		*lru_size += nr_pages;

	size = *lru_size;
1185 1186 1187
	if (WARN_ONCE(size < 0,
		"%s(%p, %d, %d): lru_size %ld\n",
		__func__, lruvec, lru, nr_pages, size)) {
1188 1189 1190 1191 1192 1193
		VM_BUG_ON(1);
		*lru_size = 0;
	}

	if (nr_pages > 0)
		*lru_size += nr_pages;
K
KAMEZAWA Hiroyuki 已提交
1194
}
1195

1196
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1197
{
1198
	struct mem_cgroup *task_memcg;
1199
	struct task_struct *p;
1200
	bool ret;
1201

1202
	p = find_lock_task_mm(task);
1203
	if (p) {
1204
		task_memcg = get_mem_cgroup_from_mm(p->mm);
1205 1206 1207 1208 1209 1210 1211
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
1212
		rcu_read_lock();
1213 1214
		task_memcg = mem_cgroup_from_task(task);
		css_get(&task_memcg->css);
1215
		rcu_read_unlock();
1216
	}
1217 1218
	ret = mem_cgroup_is_descendant(task_memcg, memcg);
	css_put(&task_memcg->css);
1219 1220 1221
	return ret;
}

1222
/**
1223
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1224
 * @memcg: the memory cgroup
1225
 *
1226
 * Returns the maximum amount of memory @mem can be charged with, in
1227
 * pages.
1228
 */
1229
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1230
{
1231 1232 1233
	unsigned long margin = 0;
	unsigned long count;
	unsigned long limit;
1234

1235
	count = page_counter_read(&memcg->memory);
1236
	limit = READ_ONCE(memcg->memory.max);
1237 1238 1239
	if (count < limit)
		margin = limit - count;

1240
	if (do_memsw_account()) {
1241
		count = page_counter_read(&memcg->memsw);
1242
		limit = READ_ONCE(memcg->memsw.max);
1243 1244
		if (count <= limit)
			margin = min(margin, limit - count);
1245 1246
		else
			margin = 0;
1247 1248 1249
	}

	return margin;
1250 1251
}

1252
/*
Q
Qiang Huang 已提交
1253
 * A routine for checking "mem" is under move_account() or not.
1254
 *
Q
Qiang Huang 已提交
1255 1256 1257
 * Checking a cgroup is mc.from or mc.to or under hierarchy of
 * moving cgroups. This is for waiting at high-memory pressure
 * caused by "move".
1258
 */
1259
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1260
{
1261 1262
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1263
	bool ret = false;
1264 1265 1266 1267 1268 1269 1270 1271 1272
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1273

1274 1275
	ret = mem_cgroup_is_descendant(from, memcg) ||
		mem_cgroup_is_descendant(to, memcg);
1276 1277
unlock:
	spin_unlock(&mc.lock);
1278 1279 1280
	return ret;
}

1281
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1282 1283
{
	if (mc.moving_task && current != mc.moving_task) {
1284
		if (mem_cgroup_under_move(memcg)) {
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1297
static const unsigned int memcg1_stats[] = {
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
	MEMCG_CACHE,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	NR_SHMEM,
	NR_FILE_MAPPED,
	NR_FILE_DIRTY,
	NR_WRITEBACK,
	MEMCG_SWAP,
};

static const char *const memcg1_stat_names[] = {
	"cache",
	"rss",
	"rss_huge",
	"shmem",
	"mapped_file",
	"dirty",
	"writeback",
	"swap",
};

1319
#define K(x) ((x) << (PAGE_SHIFT-10))
1320
/**
1321
 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1322 1323 1324 1325 1326 1327 1328 1329
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
1330 1331
	struct mem_cgroup *iter;
	unsigned int i;
1332 1333 1334

	rcu_read_lock();

1335 1336 1337 1338 1339 1340 1341 1342
	if (p) {
		pr_info("Task in ");
		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
		pr_cont(" killed as a result of limit of ");
	} else {
		pr_info("Memory limit reached of cgroup ");
	}

T
Tejun Heo 已提交
1343
	pr_cont_cgroup_path(memcg->css.cgroup);
1344
	pr_cont("\n");
1345 1346 1347

	rcu_read_unlock();

1348 1349
	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memory)),
1350
		K((u64)memcg->memory.max), memcg->memory.failcnt);
1351 1352
	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memsw)),
1353
		K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1354 1355
	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->kmem)),
1356
		K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1357 1358

	for_each_mem_cgroup_tree(iter, memcg) {
T
Tejun Heo 已提交
1359 1360
		pr_info("Memory cgroup stats for ");
		pr_cont_cgroup_path(iter->css.cgroup);
1361 1362
		pr_cont(":");

1363 1364
		for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
			if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1365
				continue;
1366
			pr_cont(" %s:%luKB", memcg1_stat_names[i],
1367
				K(memcg_page_state(iter, memcg1_stats[i])));
1368 1369 1370 1371 1372 1373 1374 1375
		}

		for (i = 0; i < NR_LRU_LISTS; i++)
			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));

		pr_cont("\n");
	}
1376 1377
}

D
David Rientjes 已提交
1378 1379 1380
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1381
unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1382
{
1383
	unsigned long max;
1384

1385
	max = memcg->memory.max;
1386
	if (mem_cgroup_swappiness(memcg)) {
1387 1388
		unsigned long memsw_max;
		unsigned long swap_max;
1389

1390 1391 1392 1393
		memsw_max = memcg->memsw.max;
		swap_max = memcg->swap.max;
		swap_max = min(swap_max, (unsigned long)total_swap_pages);
		max = min(max + swap_max, memsw_max);
1394
	}
1395
	return max;
D
David Rientjes 已提交
1396 1397
}

1398
static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1399
				     int order)
1400
{
1401 1402 1403
	struct oom_control oc = {
		.zonelist = NULL,
		.nodemask = NULL,
1404
		.memcg = memcg,
1405 1406 1407
		.gfp_mask = gfp_mask,
		.order = order,
	};
1408
	bool ret;
1409

1410 1411 1412 1413 1414 1415 1416
	if (mutex_lock_killable(&oom_lock))
		return true;
	/*
	 * A few threads which were not waiting at mutex_lock_killable() can
	 * fail to bail out. Therefore, check again after holding oom_lock.
	 */
	ret = should_force_charge() || out_of_memory(&oc);
1417
	mutex_unlock(&oom_lock);
1418
	return ret;
1419 1420
}

1421 1422
#if MAX_NUMNODES > 1

1423 1424
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1425
 * @memcg: the target memcg
1426 1427 1428 1429 1430 1431 1432
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1433
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1434 1435
		int nid, bool noswap)
{
1436
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1437 1438 1439
		return true;
	if (noswap || !total_swap_pages)
		return false;
1440
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1441 1442 1443 1444
		return true;
	return false;

}
1445 1446 1447 1448 1449 1450 1451

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1452
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1453 1454
{
	int nid;
1455 1456 1457 1458
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1459
	if (!atomic_read(&memcg->numainfo_events))
1460
		return;
1461
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1462 1463 1464
		return;

	/* make a nodemask where this memcg uses memory from */
1465
	memcg->scan_nodes = node_states[N_MEMORY];
1466

1467
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1468

1469 1470
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1471
	}
1472

1473 1474
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1489
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1490 1491 1492
{
	int node;

1493 1494
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1495

1496
	node = next_node_in(node, memcg->scan_nodes);
1497
	/*
1498 1499 1500
	 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
	 * last time it really checked all the LRUs due to rate limiting.
	 * Fallback to the current node in that case for simplicity.
1501 1502 1503 1504
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1505
	memcg->last_scanned_node = node;
1506 1507 1508
	return node;
}
#else
1509
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1510 1511 1512 1513 1514
{
	return 0;
}
#endif

1515
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1516
				   pg_data_t *pgdat,
1517 1518 1519 1520 1521 1522 1523 1524 1525
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
{
	struct mem_cgroup *victim = NULL;
	int total = 0;
	int loop = 0;
	unsigned long excess;
	unsigned long nr_scanned;
	struct mem_cgroup_reclaim_cookie reclaim = {
1526
		.pgdat = pgdat,
1527 1528 1529
		.priority = 0,
	};

1530
	excess = soft_limit_excess(root_memcg);
1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555

	while (1) {
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
		if (!victim) {
			loop++;
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!total)
					break;
				/*
				 * We want to do more targeted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
					break;
			}
			continue;
		}
1556
		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1557
					pgdat, &nr_scanned);
1558
		*total_scanned += nr_scanned;
1559
		if (!soft_limit_excess(root_memcg))
1560
			break;
1561
	}
1562 1563
	mem_cgroup_iter_break(root_memcg, victim);
	return total;
1564 1565
}

1566 1567 1568 1569 1570 1571
#ifdef CONFIG_LOCKDEP
static struct lockdep_map memcg_oom_lock_dep_map = {
	.name = "memcg_oom_lock",
};
#endif

1572 1573
static DEFINE_SPINLOCK(memcg_oom_lock);

K
KAMEZAWA Hiroyuki 已提交
1574 1575 1576 1577
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
 */
1578
static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1579
{
1580
	struct mem_cgroup *iter, *failed = NULL;
1581

1582 1583
	spin_lock(&memcg_oom_lock);

1584
	for_each_mem_cgroup_tree(iter, memcg) {
1585
		if (iter->oom_lock) {
1586 1587 1588 1589 1590
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1591 1592
			mem_cgroup_iter_break(memcg, iter);
			break;
1593 1594
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1595
	}
K
KAMEZAWA Hiroyuki 已提交
1596

1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
	if (failed) {
		/*
		 * OK, we failed to lock the whole subtree so we have
		 * to clean up what we set up to the failing subtree
		 */
		for_each_mem_cgroup_tree(iter, memcg) {
			if (iter == failed) {
				mem_cgroup_iter_break(memcg, iter);
				break;
			}
			iter->oom_lock = false;
1608
		}
1609 1610
	} else
		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1611 1612 1613 1614

	spin_unlock(&memcg_oom_lock);

	return !failed;
1615
}
1616

1617
static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1618
{
K
KAMEZAWA Hiroyuki 已提交
1619 1620
	struct mem_cgroup *iter;

1621
	spin_lock(&memcg_oom_lock);
1622
	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1623
	for_each_mem_cgroup_tree(iter, memcg)
1624
		iter->oom_lock = false;
1625
	spin_unlock(&memcg_oom_lock);
1626 1627
}

1628
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1629 1630 1631
{
	struct mem_cgroup *iter;

1632
	spin_lock(&memcg_oom_lock);
1633
	for_each_mem_cgroup_tree(iter, memcg)
1634 1635
		iter->under_oom++;
	spin_unlock(&memcg_oom_lock);
1636 1637
}

1638
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1639 1640 1641
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1642 1643
	/*
	 * When a new child is created while the hierarchy is under oom,
1644
	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
K
KAMEZAWA Hiroyuki 已提交
1645
	 */
1646
	spin_lock(&memcg_oom_lock);
1647
	for_each_mem_cgroup_tree(iter, memcg)
1648 1649 1650
		if (iter->under_oom > 0)
			iter->under_oom--;
	spin_unlock(&memcg_oom_lock);
1651 1652
}

K
KAMEZAWA Hiroyuki 已提交
1653 1654
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1655
struct oom_wait_info {
1656
	struct mem_cgroup *memcg;
1657
	wait_queue_entry_t	wait;
K
KAMEZAWA Hiroyuki 已提交
1658 1659
};

1660
static int memcg_oom_wake_function(wait_queue_entry_t *wait,
K
KAMEZAWA Hiroyuki 已提交
1661 1662
	unsigned mode, int sync, void *arg)
{
1663 1664
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1665 1666 1667
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1668
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
1669

1670 1671
	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
K
KAMEZAWA Hiroyuki 已提交
1672 1673 1674 1675
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1676
static void memcg_oom_recover(struct mem_cgroup *memcg)
1677
{
1678 1679 1680 1681 1682 1683 1684 1685 1686
	/*
	 * For the following lockless ->under_oom test, the only required
	 * guarantee is that it must see the state asserted by an OOM when
	 * this function is called as a result of userland actions
	 * triggered by the notification of the OOM.  This is trivially
	 * achieved by invoking mem_cgroup_mark_under_oom() before
	 * triggering notification.
	 */
	if (memcg && memcg->under_oom)
1687
		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1688 1689
}

1690 1691 1692 1693 1694 1695 1696 1697
enum oom_status {
	OOM_SUCCESS,
	OOM_FAILED,
	OOM_ASYNC,
	OOM_SKIPPED
};

static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1698
{
1699 1700 1701
	enum oom_status ret;
	bool locked;

1702 1703 1704
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		return OOM_SKIPPED;

K
KAMEZAWA Hiroyuki 已提交
1705
	/*
1706 1707 1708 1709
	 * We are in the middle of the charge context here, so we
	 * don't want to block when potentially sitting on a callstack
	 * that holds all kinds of filesystem and mm locks.
	 *
1710 1711 1712 1713
	 * cgroup1 allows disabling the OOM killer and waiting for outside
	 * handling until the charge can succeed; remember the context and put
	 * the task to sleep at the end of the page fault when all locks are
	 * released.
1714
	 *
1715 1716 1717 1718 1719 1720 1721
	 * On the other hand, in-kernel OOM killer allows for an async victim
	 * memory reclaim (oom_reaper) and that means that we are not solely
	 * relying on the oom victim to make a forward progress and we can
	 * invoke the oom killer here.
	 *
	 * Please note that mem_cgroup_out_of_memory might fail to find a
	 * victim and then we have to bail out from the charge path.
K
KAMEZAWA Hiroyuki 已提交
1722
	 */
1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
	if (memcg->oom_kill_disable) {
		if (!current->in_user_fault)
			return OOM_SKIPPED;
		css_get(&memcg->css);
		current->memcg_in_oom = memcg;
		current->memcg_oom_gfp_mask = mask;
		current->memcg_oom_order = order;

		return OOM_ASYNC;
	}

1734 1735 1736 1737 1738 1739 1740 1741
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	mem_cgroup_unmark_under_oom(memcg);
1742
	if (mem_cgroup_out_of_memory(memcg, mask, order))
1743 1744 1745 1746 1747 1748
		ret = OOM_SUCCESS;
	else
		ret = OOM_FAILED;

	if (locked)
		mem_cgroup_oom_unlock(memcg);
1749

1750
	return ret;
1751 1752 1753 1754
}

/**
 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1755
 * @handle: actually kill/wait or just clean up the OOM state
1756
 *
1757 1758
 * This has to be called at the end of a page fault if the memcg OOM
 * handler was enabled.
1759
 *
1760
 * Memcg supports userspace OOM handling where failed allocations must
1761 1762 1763 1764
 * sleep on a waitqueue until the userspace task resolves the
 * situation.  Sleeping directly in the charge context with all kinds
 * of locks held is not a good idea, instead we remember an OOM state
 * in the task and mem_cgroup_oom_synchronize() has to be called at
1765
 * the end of the page fault to complete the OOM handling.
1766 1767
 *
 * Returns %true if an ongoing memcg OOM situation was detected and
1768
 * completed, %false otherwise.
1769
 */
1770
bool mem_cgroup_oom_synchronize(bool handle)
1771
{
T
Tejun Heo 已提交
1772
	struct mem_cgroup *memcg = current->memcg_in_oom;
1773
	struct oom_wait_info owait;
1774
	bool locked;
1775 1776 1777

	/* OOM is global, do not handle */
	if (!memcg)
1778
		return false;
1779

1780
	if (!handle)
1781
		goto cleanup;
1782 1783 1784 1785 1786

	owait.memcg = memcg;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
1787
	INIT_LIST_HEAD(&owait.wait.entry);
K
KAMEZAWA Hiroyuki 已提交
1788

1789
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	if (locked && !memcg->oom_kill_disable) {
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
T
Tejun Heo 已提交
1800 1801
		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
					 current->memcg_oom_order);
1802
	} else {
1803
		schedule();
1804 1805 1806 1807 1808
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
	}

	if (locked) {
1809 1810 1811 1812 1813 1814 1815 1816
		mem_cgroup_oom_unlock(memcg);
		/*
		 * There is no guarantee that an OOM-lock contender
		 * sees the wakeups triggered by the OOM kill
		 * uncharges.  Wake any sleepers explicitely.
		 */
		memcg_oom_recover(memcg);
	}
1817
cleanup:
T
Tejun Heo 已提交
1818
	current->memcg_in_oom = NULL;
1819
	css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
1820
	return true;
1821 1822
}

1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
/**
 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
 * @victim: task to be killed by the OOM killer
 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
 *
 * Returns a pointer to a memory cgroup, which has to be cleaned up
 * by killing all belonging OOM-killable tasks.
 *
 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
 */
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
					    struct mem_cgroup *oom_domain)
{
	struct mem_cgroup *oom_group = NULL;
	struct mem_cgroup *memcg;

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return NULL;

	if (!oom_domain)
		oom_domain = root_mem_cgroup;

	rcu_read_lock();

	memcg = mem_cgroup_from_task(victim);
	if (memcg == root_mem_cgroup)
		goto out;

	/*
	 * Traverse the memory cgroup hierarchy from the victim task's
	 * cgroup up to the OOMing cgroup (or root) to find the
	 * highest-level memory cgroup with oom.group set.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		if (memcg->oom_group)
			oom_group = memcg;

		if (memcg == oom_domain)
			break;
	}

	if (oom_group)
		css_get(&oom_group->css);
out:
	rcu_read_unlock();

	return oom_group;
}

void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
{
	pr_info("Tasks in ");
	pr_cont_cgroup_path(memcg->css.cgroup);
	pr_cont(" are going to be killed due to memory.oom.group set\n");
}

1879
/**
1880 1881
 * lock_page_memcg - lock a page->mem_cgroup binding
 * @page: the page
1882
 *
1883
 * This function protects unlocked LRU pages from being moved to
1884 1885 1886 1887 1888
 * another cgroup.
 *
 * It ensures lifetime of the returned memcg. Caller is responsible
 * for the lifetime of the page; __unlock_page_memcg() is available
 * when @page might get freed inside the locked section.
1889
 */
1890
struct mem_cgroup *lock_page_memcg(struct page *page)
1891 1892
{
	struct mem_cgroup *memcg;
1893
	unsigned long flags;
1894

1895 1896 1897 1898
	/*
	 * The RCU lock is held throughout the transaction.  The fast
	 * path can get away without acquiring the memcg->move_lock
	 * because page moving starts with an RCU grace period.
1899 1900 1901 1902 1903 1904 1905
	 *
	 * The RCU lock also protects the memcg from being freed when
	 * the page state that is going to change is the only thing
	 * preventing the page itself from being freed. E.g. writeback
	 * doesn't hold a page reference and relies on PG_writeback to
	 * keep off truncation, migration and so forth.
         */
1906 1907 1908
	rcu_read_lock();

	if (mem_cgroup_disabled())
1909
		return NULL;
1910
again:
1911
	memcg = page->mem_cgroup;
1912
	if (unlikely(!memcg))
1913
		return NULL;
1914

Q
Qiang Huang 已提交
1915
	if (atomic_read(&memcg->moving_account) <= 0)
1916
		return memcg;
1917

1918
	spin_lock_irqsave(&memcg->move_lock, flags);
1919
	if (memcg != page->mem_cgroup) {
1920
		spin_unlock_irqrestore(&memcg->move_lock, flags);
1921 1922
		goto again;
	}
1923 1924 1925 1926

	/*
	 * When charge migration first begins, we can have locked and
	 * unlocked page stat updates happening concurrently.  Track
1927
	 * the task who has the lock for unlock_page_memcg().
1928 1929 1930
	 */
	memcg->move_lock_task = current;
	memcg->move_lock_flags = flags;
1931

1932
	return memcg;
1933
}
1934
EXPORT_SYMBOL(lock_page_memcg);
1935

1936
/**
1937 1938 1939 1940
 * __unlock_page_memcg - unlock and unpin a memcg
 * @memcg: the memcg
 *
 * Unlock and unpin a memcg returned by lock_page_memcg().
1941
 */
1942
void __unlock_page_memcg(struct mem_cgroup *memcg)
1943
{
1944 1945 1946 1947 1948 1949 1950 1951
	if (memcg && memcg->move_lock_task == current) {
		unsigned long flags = memcg->move_lock_flags;

		memcg->move_lock_task = NULL;
		memcg->move_lock_flags = 0;

		spin_unlock_irqrestore(&memcg->move_lock, flags);
	}
1952

1953
	rcu_read_unlock();
1954
}
1955 1956 1957 1958 1959 1960 1961 1962 1963

/**
 * unlock_page_memcg - unlock a page->mem_cgroup binding
 * @page: the page
 */
void unlock_page_memcg(struct page *page)
{
	__unlock_page_memcg(page->mem_cgroup);
}
1964
EXPORT_SYMBOL(unlock_page_memcg);
1965

1966 1967
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
1968
	unsigned int nr_pages;
1969
	struct work_struct work;
1970
	unsigned long flags;
1971
#define FLUSHING_CACHED_CHARGE	0
1972 1973
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1974
static DEFINE_MUTEX(percpu_charge_mutex);
1975

1976 1977 1978 1979 1980 1981 1982 1983 1984 1985
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
1986
 */
1987
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1988 1989
{
	struct memcg_stock_pcp *stock;
1990
	unsigned long flags;
1991
	bool ret = false;
1992

1993
	if (nr_pages > MEMCG_CHARGE_BATCH)
1994
		return ret;
1995

1996 1997 1998
	local_irq_save(flags);

	stock = this_cpu_ptr(&memcg_stock);
1999
	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2000
		stock->nr_pages -= nr_pages;
2001 2002
		ret = true;
	}
2003 2004 2005

	local_irq_restore(flags);

2006 2007 2008 2009
	return ret;
}

/*
2010
 * Returns stocks cached in percpu and reset cached information.
2011 2012 2013 2014 2015
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2016
	if (stock->nr_pages) {
2017
		page_counter_uncharge(&old->memory, stock->nr_pages);
2018
		if (do_memsw_account())
2019
			page_counter_uncharge(&old->memsw, stock->nr_pages);
2020
		css_put_many(&old->css, stock->nr_pages);
2021
		stock->nr_pages = 0;
2022 2023 2024 2025 2026 2027
	}
	stock->cached = NULL;
}

static void drain_local_stock(struct work_struct *dummy)
{
2028 2029 2030
	struct memcg_stock_pcp *stock;
	unsigned long flags;

2031 2032 2033 2034
	/*
	 * The only protection from memory hotplug vs. drain_stock races is
	 * that we always operate on local CPU stock here with IRQ disabled
	 */
2035 2036 2037
	local_irq_save(flags);

	stock = this_cpu_ptr(&memcg_stock);
2038
	drain_stock(stock);
2039
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2040 2041

	local_irq_restore(flags);
2042 2043 2044
}

/*
2045
 * Cache charges(val) to local per_cpu area.
2046
 * This will be consumed by consume_stock() function, later.
2047
 */
2048
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2049
{
2050 2051 2052 2053
	struct memcg_stock_pcp *stock;
	unsigned long flags;

	local_irq_save(flags);
2054

2055
	stock = this_cpu_ptr(&memcg_stock);
2056
	if (stock->cached != memcg) { /* reset if necessary */
2057
		drain_stock(stock);
2058
		stock->cached = memcg;
2059
	}
2060
	stock->nr_pages += nr_pages;
2061

2062
	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2063 2064
		drain_stock(stock);

2065
	local_irq_restore(flags);
2066 2067 2068
}

/*
2069
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2070
 * of the hierarchy under it.
2071
 */
2072
static void drain_all_stock(struct mem_cgroup *root_memcg)
2073
{
2074
	int cpu, curcpu;
2075

2076 2077 2078
	/* If someone's already draining, avoid adding running more workers. */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2079 2080 2081 2082 2083 2084
	/*
	 * Notify other cpus that system-wide "drain" is running
	 * We do not care about races with the cpu hotplug because cpu down
	 * as well as workers from this path always operate on the local
	 * per-cpu data. CPU up doesn't touch memcg_stock at all.
	 */
2085
	curcpu = get_cpu();
2086 2087
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2088
		struct mem_cgroup *memcg;
2089

2090
		memcg = stock->cached;
2091
		if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
2092
			continue;
2093 2094
		if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
			css_put(&memcg->css);
2095
			continue;
2096
		}
2097 2098 2099 2100 2101 2102
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2103
		css_put(&memcg->css);
2104
	}
2105
	put_cpu();
2106
	mutex_unlock(&percpu_charge_mutex);
2107 2108
}

2109
static int memcg_hotplug_cpu_dead(unsigned int cpu)
2110 2111
{
	struct memcg_stock_pcp *stock;
2112
	struct mem_cgroup *memcg;
2113 2114 2115

	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140

	for_each_mem_cgroup(memcg) {
		int i;

		for (i = 0; i < MEMCG_NR_STAT; i++) {
			int nid;
			long x;

			x = this_cpu_xchg(memcg->stat_cpu->count[i], 0);
			if (x)
				atomic_long_add(x, &memcg->stat[i]);

			if (i >= NR_VM_NODE_STAT_ITEMS)
				continue;

			for_each_node(nid) {
				struct mem_cgroup_per_node *pn;

				pn = mem_cgroup_nodeinfo(memcg, nid);
				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
				if (x)
					atomic_long_add(x, &pn->lruvec_stat[i]);
			}
		}

2141
		for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2142 2143 2144 2145 2146 2147 2148 2149
			long x;

			x = this_cpu_xchg(memcg->stat_cpu->events[i], 0);
			if (x)
				atomic_long_add(x, &memcg->events[i]);
		}
	}

2150
	return 0;
2151 2152
}

2153 2154 2155 2156 2157 2158 2159
static void reclaim_high(struct mem_cgroup *memcg,
			 unsigned int nr_pages,
			 gfp_t gfp_mask)
{
	do {
		if (page_counter_read(&memcg->memory) <= memcg->high)
			continue;
2160
		memcg_memory_event(memcg, MEMCG_HIGH);
2161 2162 2163 2164 2165 2166 2167 2168 2169
		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
	} while ((memcg = parent_mem_cgroup(memcg)));
}

static void high_work_func(struct work_struct *work)
{
	struct mem_cgroup *memcg;

	memcg = container_of(work, struct mem_cgroup, high_work);
2170
	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2171 2172
}

2173 2174 2175 2176 2177 2178 2179
/*
 * Scheduled by try_charge() to be executed from the userland return path
 * and reclaims memory over the high limit.
 */
void mem_cgroup_handle_over_high(void)
{
	unsigned int nr_pages = current->memcg_nr_pages_over_high;
2180
	struct mem_cgroup *memcg;
2181 2182 2183 2184

	if (likely(!nr_pages))
		return;

2185 2186
	memcg = get_mem_cgroup_from_mm(current->mm);
	reclaim_high(memcg, nr_pages, GFP_KERNEL);
2187 2188 2189 2190
	css_put(&memcg->css);
	current->memcg_nr_pages_over_high = 0;
}

2191 2192
static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
		      unsigned int nr_pages)
2193
{
2194
	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2195
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2196
	struct mem_cgroup *mem_over_limit;
2197
	struct page_counter *counter;
2198
	unsigned long nr_reclaimed;
2199 2200
	bool may_swap = true;
	bool drained = false;
2201 2202
	bool oomed = false;
	enum oom_status oom_status;
2203

2204
	if (mem_cgroup_is_root(memcg))
2205
		return 0;
2206
retry:
2207
	if (consume_stock(memcg, nr_pages))
2208
		return 0;
2209

2210
	if (!do_memsw_account() ||
2211 2212
	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
		if (page_counter_try_charge(&memcg->memory, batch, &counter))
2213
			goto done_restock;
2214
		if (do_memsw_account())
2215 2216
			page_counter_uncharge(&memcg->memsw, batch);
		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2217
	} else {
2218
		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2219
		may_swap = false;
2220
	}
2221

2222 2223 2224 2225
	if (batch > nr_pages) {
		batch = nr_pages;
		goto retry;
	}
2226

2227 2228 2229 2230 2231 2232 2233 2234 2235
	/*
	 * Memcg doesn't have a dedicated reserve for atomic
	 * allocations. But like the global atomic pool, we need to
	 * put the burden of reclaim on regular allocation requests
	 * and let these go through as privileged allocations.
	 */
	if (gfp_mask & __GFP_ATOMIC)
		goto force;

2236 2237 2238 2239 2240 2241
	/*
	 * Unlike in global OOM situations, memcg is not in a physical
	 * memory shortage.  Allow dying and OOM-killed tasks to
	 * bypass the last charges so that they can exit quickly and
	 * free their memory.
	 */
2242
	if (unlikely(should_force_charge()))
2243
		goto force;
2244

2245 2246 2247 2248 2249 2250 2251 2252 2253
	/*
	 * Prevent unbounded recursion when reclaim operations need to
	 * allocate memory. This might exceed the limits temporarily,
	 * but we prefer facilitating memory reclaim and getting back
	 * under the limit over triggering OOM kills in these cases.
	 */
	if (unlikely(current->flags & PF_MEMALLOC))
		goto force;

2254 2255 2256
	if (unlikely(task_in_memcg_oom(current)))
		goto nomem;

2257
	if (!gfpflags_allow_blocking(gfp_mask))
2258
		goto nomem;
2259

2260
	memcg_memory_event(mem_over_limit, MEMCG_MAX);
2261

2262 2263
	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
						    gfp_mask, may_swap);
2264

2265
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2266
		goto retry;
2267

2268
	if (!drained) {
2269
		drain_all_stock(mem_over_limit);
2270 2271 2272 2273
		drained = true;
		goto retry;
	}

2274 2275
	if (gfp_mask & __GFP_NORETRY)
		goto nomem;
2276 2277 2278 2279 2280 2281 2282 2283 2284
	/*
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
	 */
2285
	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2286 2287 2288 2289 2290 2291 2292 2293
		goto retry;
	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		goto retry;

2294 2295 2296
	if (nr_retries--)
		goto retry;

2297 2298 2299
	if (gfp_mask & __GFP_RETRY_MAYFAIL && oomed)
		goto nomem;

2300
	if (gfp_mask & __GFP_NOFAIL)
2301
		goto force;
2302

2303
	if (fatal_signal_pending(current))
2304
		goto force;
2305

2306
	memcg_memory_event(mem_over_limit, MEMCG_OOM);
2307

2308 2309 2310 2311 2312 2313
	/*
	 * keep retrying as long as the memcg oom killer is able to make
	 * a forward progress or bypass the charge if the oom killer
	 * couldn't make any progress.
	 */
	oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2314
		       get_order(nr_pages * PAGE_SIZE));
2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
	switch (oom_status) {
	case OOM_SUCCESS:
		nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
		oomed = true;
		goto retry;
	case OOM_FAILED:
		goto force;
	default:
		goto nomem;
	}
2325
nomem:
2326
	if (!(gfp_mask & __GFP_NOFAIL))
2327
		return -ENOMEM;
2328 2329 2330 2331 2332 2333 2334
force:
	/*
	 * The allocation either can't fail or will lead to more memory
	 * being freed very soon.  Allow memory usage go over the limit
	 * temporarily by force charging it.
	 */
	page_counter_charge(&memcg->memory, nr_pages);
2335
	if (do_memsw_account())
2336 2337 2338 2339
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);

	return 0;
2340 2341

done_restock:
2342
	css_get_many(&memcg->css, batch);
2343 2344
	if (batch > nr_pages)
		refill_stock(memcg, batch - nr_pages);
2345

2346
	/*
2347 2348
	 * If the hierarchy is above the normal consumption range, schedule
	 * reclaim on returning to userland.  We can perform reclaim here
2349
	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2350 2351 2352 2353
	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
	 * not recorded as it most likely matches current's and won't
	 * change in the meantime.  As high limit is checked again before
	 * reclaim, the cost of mismatch is negligible.
2354 2355
	 */
	do {
2356
		if (page_counter_read(&memcg->memory) > memcg->high) {
2357 2358 2359 2360 2361
			/* Don't bother a random interrupted task */
			if (in_interrupt()) {
				schedule_work(&memcg->high_work);
				break;
			}
V
Vladimir Davydov 已提交
2362
			current->memcg_nr_pages_over_high += batch;
2363 2364 2365
			set_notify_resume(current);
			break;
		}
2366
	} while ((memcg = parent_mem_cgroup(memcg)));
2367 2368

	return 0;
2369
}
2370

2371
static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2372
{
2373 2374 2375
	if (mem_cgroup_is_root(memcg))
		return;

2376
	page_counter_uncharge(&memcg->memory, nr_pages);
2377
	if (do_memsw_account())
2378
		page_counter_uncharge(&memcg->memsw, nr_pages);
2379

2380
	css_put_many(&memcg->css, nr_pages);
2381 2382
}

2383 2384 2385 2386
static void lock_page_lru(struct page *page, int *isolated)
{
	struct zone *zone = page_zone(page);

2387
	spin_lock_irq(zone_lru_lock(zone));
2388 2389 2390
	if (PageLRU(page)) {
		struct lruvec *lruvec;

M
Mel Gorman 已提交
2391
		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
		ClearPageLRU(page);
		del_page_from_lru_list(page, lruvec, page_lru(page));
		*isolated = 1;
	} else
		*isolated = 0;
}

static void unlock_page_lru(struct page *page, int isolated)
{
	struct zone *zone = page_zone(page);

	if (isolated) {
		struct lruvec *lruvec;

M
Mel Gorman 已提交
2406
		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2407 2408 2409 2410
		VM_BUG_ON_PAGE(PageLRU(page), page);
		SetPageLRU(page);
		add_page_to_lru_list(page, lruvec, page_lru(page));
	}
2411
	spin_unlock_irq(zone_lru_lock(zone));
2412 2413
}

2414
static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2415
			  bool lrucare)
2416
{
2417
	int isolated;
2418

2419
	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2420 2421 2422 2423 2424

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
2425 2426
	if (lrucare)
		lock_page_lru(page, &isolated);
2427

2428 2429
	/*
	 * Nobody should be changing or seriously looking at
2430
	 * page->mem_cgroup at this point:
2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441
	 *
	 * - the page is uncharged
	 *
	 * - the page is off-LRU
	 *
	 * - an anonymous fault has exclusive page access, except for
	 *   a locked page table
	 *
	 * - a page cache insertion, a swapin fault, or a migration
	 *   have the page locked
	 */
2442
	page->mem_cgroup = memcg;
2443

2444 2445
	if (lrucare)
		unlock_page_lru(page, isolated);
2446
}
2447

2448
#ifdef CONFIG_MEMCG_KMEM
2449
static int memcg_alloc_cache_id(void)
2450
{
2451 2452 2453
	int id, size;
	int err;

2454
	id = ida_simple_get(&memcg_cache_ida,
2455 2456 2457
			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (id < 0)
		return id;
2458

2459
	if (id < memcg_nr_cache_ids)
2460 2461 2462 2463 2464 2465
		return id;

	/*
	 * There's no space for the new id in memcg_caches arrays,
	 * so we have to grow them.
	 */
2466
	down_write(&memcg_cache_ids_sem);
2467 2468

	size = 2 * (id + 1);
2469 2470 2471 2472 2473
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

2474
	err = memcg_update_all_caches(size);
2475 2476
	if (!err)
		err = memcg_update_all_list_lrus(size);
2477 2478 2479 2480 2481
	if (!err)
		memcg_nr_cache_ids = size;

	up_write(&memcg_cache_ids_sem);

2482
	if (err) {
2483
		ida_simple_remove(&memcg_cache_ida, id);
2484 2485 2486 2487 2488 2489 2490
		return err;
	}
	return id;
}

static void memcg_free_cache_id(int id)
{
2491
	ida_simple_remove(&memcg_cache_ida, id);
2492 2493
}

2494
struct memcg_kmem_cache_create_work {
2495 2496 2497 2498 2499
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

2500
static void memcg_kmem_cache_create_func(struct work_struct *w)
2501
{
2502 2503
	struct memcg_kmem_cache_create_work *cw =
		container_of(w, struct memcg_kmem_cache_create_work, work);
2504 2505
	struct mem_cgroup *memcg = cw->memcg;
	struct kmem_cache *cachep = cw->cachep;
2506

2507
	memcg_create_kmem_cache(memcg, cachep);
2508

2509
	css_put(&memcg->css);
2510 2511 2512 2513 2514 2515
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 */
2516 2517
static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					       struct kmem_cache *cachep)
2518
{
2519
	struct memcg_kmem_cache_create_work *cw;
2520

2521
	cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2522
	if (!cw)
2523
		return;
2524 2525

	css_get(&memcg->css);
2526 2527 2528

	cw->memcg = memcg;
	cw->cachep = cachep;
2529
	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2530

2531
	queue_work(memcg_kmem_cache_wq, &cw->work);
2532 2533
}

2534 2535
static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					     struct kmem_cache *cachep)
2536 2537 2538 2539
{
	/*
	 * We need to stop accounting when we kmalloc, because if the
	 * corresponding kmalloc cache is not yet created, the first allocation
2540
	 * in __memcg_schedule_kmem_cache_create will recurse.
2541 2542 2543 2544 2545 2546 2547
	 *
	 * However, it is better to enclose the whole function. Depending on
	 * the debugging options enabled, INIT_WORK(), for instance, can
	 * trigger an allocation. This too, will make us recurse. Because at
	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
	 * the safest choice is to do it like this, wrapping the whole function.
	 */
2548
	current->memcg_kmem_skip_account = 1;
2549
	__memcg_schedule_kmem_cache_create(memcg, cachep);
2550
	current->memcg_kmem_skip_account = 0;
2551
}
2552

2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563
static inline bool memcg_kmem_bypass(void)
{
	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
		return true;
	return false;
}

/**
 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
 * @cachep: the original global kmem cache
 *
2564 2565 2566
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
2567 2568 2569
 * If the cache does not exist yet, if we are the first user of it, we
 * create it asynchronously in a workqueue and let the current allocation
 * go through with the original cache.
2570
 *
2571 2572 2573 2574
 * This function takes a reference to the cache it returns to assure it
 * won't get destroyed while we are working with it. Once the caller is
 * done with it, memcg_kmem_put_cache() must be called to release the
 * reference.
2575
 */
2576
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2577 2578
{
	struct mem_cgroup *memcg;
2579
	struct kmem_cache *memcg_cachep;
2580
	int kmemcg_id;
2581

2582
	VM_BUG_ON(!is_root_cache(cachep));
2583

2584
	if (memcg_kmem_bypass())
V
Vladimir Davydov 已提交
2585 2586
		return cachep;

2587
	if (current->memcg_kmem_skip_account)
2588 2589
		return cachep;

2590
	memcg = get_mem_cgroup_from_current();
2591
	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2592
	if (kmemcg_id < 0)
2593
		goto out;
2594

2595
	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2596 2597
	if (likely(memcg_cachep))
		return memcg_cachep;
2598 2599 2600 2601 2602 2603 2604 2605 2606

	/*
	 * If we are in a safe context (can wait, and not in interrupt
	 * context), we could be be predictable and return right away.
	 * This would guarantee that the allocation being performed
	 * already belongs in the new cache.
	 *
	 * However, there are some clashes that can arrive from locking.
	 * For instance, because we acquire the slab_mutex while doing
2607 2608 2609
	 * memcg_create_kmem_cache, this means no further allocation
	 * could happen with the slab_mutex held. So it's better to
	 * defer everything.
2610
	 */
2611
	memcg_schedule_kmem_cache_create(memcg, cachep);
2612
out:
2613
	css_put(&memcg->css);
2614
	return cachep;
2615 2616
}

2617 2618 2619 2620 2621
/**
 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
 * @cachep: the cache returned by memcg_kmem_get_cache
 */
void memcg_kmem_put_cache(struct kmem_cache *cachep)
2622 2623
{
	if (!is_root_cache(cachep))
2624
		css_put(&cachep->memcg_params.memcg->css);
2625 2626
}

2627
/**
2628
 * memcg_kmem_charge_memcg: charge a kmem page
2629 2630 2631 2632 2633 2634 2635 2636 2637
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
 * @memcg: memory cgroup to charge
 *
 * Returns 0 on success, an error code on failure.
 */
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			    struct mem_cgroup *memcg)
2638
{
2639 2640
	unsigned int nr_pages = 1 << order;
	struct page_counter *counter;
2641 2642
	int ret;

2643
	ret = try_charge(memcg, gfp, nr_pages);
2644
	if (ret)
2645
		return ret;
2646 2647 2648

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2649 2650 2651 2652 2653 2654 2655 2656 2657 2658

		/*
		 * Enforce __GFP_NOFAIL allocation because callers are not
		 * prepared to see failures and likely do not have any failure
		 * handling code.
		 */
		if (gfp & __GFP_NOFAIL) {
			page_counter_charge(&memcg->kmem, nr_pages);
			return 0;
		}
2659 2660
		cancel_charge(memcg, nr_pages);
		return -ENOMEM;
2661 2662
	}

2663
	page->mem_cgroup = memcg;
2664

2665
	return 0;
2666 2667
}

2668 2669 2670 2671 2672 2673 2674 2675 2676
/**
 * memcg_kmem_charge: charge a kmem page to the current memory cgroup
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
 *
 * Returns 0 on success, an error code on failure.
 */
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2677
{
2678
	struct mem_cgroup *memcg;
2679
	int ret = 0;
2680

2681 2682 2683
	if (memcg_kmem_bypass())
		return 0;

2684
	memcg = get_mem_cgroup_from_current();
2685
	if (!mem_cgroup_is_root(memcg)) {
2686
		ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2687 2688 2689
		if (!ret)
			__SetPageKmemcg(page);
	}
2690
	css_put(&memcg->css);
2691
	return ret;
2692
}
2693 2694 2695 2696 2697 2698
/**
 * memcg_kmem_uncharge: uncharge a kmem page
 * @page: page to uncharge
 * @order: allocation order
 */
void memcg_kmem_uncharge(struct page *page, int order)
2699
{
2700
	struct mem_cgroup *memcg = page->mem_cgroup;
2701
	unsigned int nr_pages = 1 << order;
2702 2703 2704 2705

	if (!memcg)
		return;

2706
	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2707

2708 2709 2710
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
		page_counter_uncharge(&memcg->kmem, nr_pages);

2711
	page_counter_uncharge(&memcg->memory, nr_pages);
2712
	if (do_memsw_account())
2713
		page_counter_uncharge(&memcg->memsw, nr_pages);
2714

2715
	page->mem_cgroup = NULL;
2716 2717 2718 2719 2720

	/* slab pages do not have PageKmemcg flag set */
	if (PageKmemcg(page))
		__ClearPageKmemcg(page);

2721
	css_put_many(&memcg->css, nr_pages);
2722
}
2723
#endif /* CONFIG_MEMCG_KMEM */
2724

2725 2726 2727 2728
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

/*
 * Because tail pages are not marked as "used", set it. We're under
2729
 * zone_lru_lock and migration entries setup in all page mappings.
2730
 */
2731
void mem_cgroup_split_huge_fixup(struct page *head)
2732
{
2733
	int i;
2734

2735 2736
	if (mem_cgroup_disabled())
		return;
2737

2738
	for (i = 1; i < HPAGE_PMD_NR; i++)
2739
		head[i].mem_cgroup = head->mem_cgroup;
2740

2741
	__mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
2742
}
2743
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2744

A
Andrew Morton 已提交
2745
#ifdef CONFIG_MEMCG_SWAP
2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756
/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
2757
 * The caller must have charged to @to, IOW, called page_counter_charge() about
2758 2759 2760
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
2761
				struct mem_cgroup *from, struct mem_cgroup *to)
2762 2763 2764
{
	unsigned short old_id, new_id;

L
Li Zefan 已提交
2765 2766
	old_id = mem_cgroup_id(from);
	new_id = mem_cgroup_id(to);
2767 2768

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2769 2770
		mod_memcg_state(from, MEMCG_SWAP, -1);
		mod_memcg_state(to, MEMCG_SWAP, 1);
2771 2772 2773 2774 2775 2776
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2777
				struct mem_cgroup *from, struct mem_cgroup *to)
2778 2779 2780
{
	return -EINVAL;
}
2781
#endif
K
KAMEZAWA Hiroyuki 已提交
2782

2783
static DEFINE_MUTEX(memcg_max_mutex);
2784

2785 2786
static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
				 unsigned long max, bool memsw)
2787
{
2788
	bool enlarge = false;
2789
	bool drained = false;
2790
	int ret;
2791 2792
	bool limits_invariant;
	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2793

2794
	do {
2795 2796 2797 2798
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2799

2800
		mutex_lock(&memcg_max_mutex);
2801 2802
		/*
		 * Make sure that the new limit (memsw or memory limit) doesn't
2803
		 * break our basic invariant rule memory.max <= memsw.max.
2804
		 */
2805 2806
		limits_invariant = memsw ? max >= memcg->memory.max :
					   max <= memcg->memsw.max;
2807
		if (!limits_invariant) {
2808
			mutex_unlock(&memcg_max_mutex);
2809 2810 2811
			ret = -EINVAL;
			break;
		}
2812
		if (max > counter->max)
2813
			enlarge = true;
2814 2815
		ret = page_counter_set_max(counter, max);
		mutex_unlock(&memcg_max_mutex);
2816 2817 2818 2819

		if (!ret)
			break;

2820 2821 2822 2823 2824 2825
		if (!drained) {
			drain_all_stock(memcg);
			drained = true;
			continue;
		}

2826 2827 2828 2829 2830 2831
		if (!try_to_free_mem_cgroup_pages(memcg, 1,
					GFP_KERNEL, !memsw)) {
			ret = -EBUSY;
			break;
		}
	} while (true);
2832

2833 2834
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
2835

2836 2837 2838
	return ret;
}

2839
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2840 2841 2842 2843
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
{
	unsigned long nr_reclaimed = 0;
2844
	struct mem_cgroup_per_node *mz, *next_mz = NULL;
2845 2846
	unsigned long reclaimed;
	int loop = 0;
2847
	struct mem_cgroup_tree_per_node *mctz;
2848
	unsigned long excess;
2849 2850 2851 2852 2853
	unsigned long nr_scanned;

	if (order > 0)
		return 0;

2854
	mctz = soft_limit_tree_node(pgdat->node_id);
2855 2856 2857 2858 2859 2860

	/*
	 * Do not even bother to check the largest node if the root
	 * is empty. Do it lockless to prevent lock bouncing. Races
	 * are acceptable as soft limit is best effort anyway.
	 */
2861
	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2862 2863
		return 0;

2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		nr_scanned = 0;
2878
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2879 2880 2881
						    gfp_mask, &nr_scanned);
		nr_reclaimed += reclaimed;
		*total_scanned += nr_scanned;
2882
		spin_lock_irq(&mctz->lock);
2883
		__mem_cgroup_remove_exceeded(mz, mctz);
2884 2885 2886 2887 2888 2889

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
2890 2891 2892
		if (!reclaimed)
			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);

2893
		excess = soft_limit_excess(mz->memcg);
2894 2895 2896 2897 2898 2899 2900 2901 2902
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
		/* If excess == 0, no tree ops */
2903
		__mem_cgroup_insert_exceeded(mz, mctz, excess);
2904
		spin_unlock_irq(&mctz->lock);
2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
		css_put(&mz->memcg->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->memcg->css);
	return nr_reclaimed;
}

2922 2923 2924 2925 2926 2927
/*
 * Test whether @memcg has children, dead or alive.  Note that this
 * function doesn't care whether @memcg has use_hierarchy enabled and
 * returns %true if there are child csses according to the cgroup
 * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
 */
2928 2929
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
2930 2931 2932 2933 2934 2935
	bool ret;

	rcu_read_lock();
	ret = css_next_child(NULL, &memcg->css);
	rcu_read_unlock();
	return ret;
2936 2937
}

2938
/*
2939
 * Reclaims as many pages from the given memcg as possible.
2940 2941 2942 2943 2944 2945 2946
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;

2947 2948
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
2949 2950 2951

	drain_all_stock(memcg);

2952
	/* try to free all pages in this cgroup */
2953
	while (nr_retries && page_counter_read(&memcg->memory)) {
2954
		int progress;
2955

2956 2957 2958
		if (signal_pending(current))
			return -EINTR;

2959 2960
		progress = try_to_free_mem_cgroup_pages(memcg, 1,
							GFP_KERNEL, true);
2961
		if (!progress) {
2962
			nr_retries--;
2963
			/* maybe some writeback is necessary */
2964
			congestion_wait(BLK_RW_ASYNC, HZ/10);
2965
		}
2966 2967

	}
2968 2969

	return 0;
2970 2971
}

2972 2973 2974
static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
2975
{
2976
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2977

2978 2979
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
2980
	return mem_cgroup_force_empty(memcg) ?: nbytes;
2981 2982
}

2983 2984
static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
				     struct cftype *cft)
2985
{
2986
	return mem_cgroup_from_css(css)->use_hierarchy;
2987 2988
}

2989 2990
static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, u64 val)
2991 2992
{
	int retval = 0;
2993
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
2994
	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2995

2996
	if (memcg->use_hierarchy == val)
2997
		return 0;
2998

2999
	/*
3000
	 * If parent's use_hierarchy is set, we can't make any modifications
3001 3002 3003 3004 3005 3006
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
3007
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3008
				(val == 1 || val == 0)) {
3009
		if (!memcg_has_children(memcg))
3010
			memcg->use_hierarchy = val;
3011 3012 3013 3014
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
3015

3016 3017 3018
	return retval;
}

3019 3020 3021 3022 3023 3024 3025 3026 3027
struct accumulated_stats {
	unsigned long stat[MEMCG_NR_STAT];
	unsigned long events[NR_VM_EVENT_ITEMS];
	unsigned long lru_pages[NR_LRU_LISTS];
	const unsigned int *stats_array;
	const unsigned int *events_array;
	int stats_size;
	int events_size;
};
3028

3029 3030
static void accumulate_memcg_tree(struct mem_cgroup *memcg,
				  struct accumulated_stats *acc)
3031
{
3032
	struct mem_cgroup *mi;
3033
	int i;
3034

3035 3036 3037 3038
	for_each_mem_cgroup_tree(mi, memcg) {
		for (i = 0; i < acc->stats_size; i++)
			acc->stat[i] += memcg_page_state(mi,
				acc->stats_array ? acc->stats_array[i] : i);
3039

3040 3041 3042 3043 3044 3045 3046
		for (i = 0; i < acc->events_size; i++)
			acc->events[i] += memcg_sum_events(mi,
				acc->events_array ? acc->events_array[i] : i);

		for (i = 0; i < NR_LRU_LISTS; i++)
			acc->lru_pages[i] +=
				mem_cgroup_nr_lru_pages(mi, BIT(i));
3047
	}
3048 3049
}

3050
static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3051
{
3052
	unsigned long val = 0;
3053

3054
	if (mem_cgroup_is_root(memcg)) {
3055 3056 3057
		struct mem_cgroup *iter;

		for_each_mem_cgroup_tree(iter, memcg) {
3058 3059
			val += memcg_page_state(iter, MEMCG_CACHE);
			val += memcg_page_state(iter, MEMCG_RSS);
3060
			if (swap)
3061
				val += memcg_page_state(iter, MEMCG_SWAP);
3062
		}
3063
	} else {
3064
		if (!swap)
3065
			val = page_counter_read(&memcg->memory);
3066
		else
3067
			val = page_counter_read(&memcg->memsw);
3068
	}
3069
	return val;
3070 3071
}

3072 3073 3074 3075 3076 3077 3078
enum {
	RES_USAGE,
	RES_LIMIT,
	RES_MAX_USAGE,
	RES_FAILCNT,
	RES_SOFT_LIMIT,
};
3079

3080
static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3081
			       struct cftype *cft)
B
Balbir Singh 已提交
3082
{
3083
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3084
	struct page_counter *counter;
3085

3086
	switch (MEMFILE_TYPE(cft->private)) {
3087
	case _MEM:
3088 3089
		counter = &memcg->memory;
		break;
3090
	case _MEMSWAP:
3091 3092
		counter = &memcg->memsw;
		break;
3093
	case _KMEM:
3094
		counter = &memcg->kmem;
3095
		break;
V
Vladimir Davydov 已提交
3096
	case _TCP:
3097
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
3098
		break;
3099 3100 3101
	default:
		BUG();
	}
3102 3103 3104 3105

	switch (MEMFILE_ATTR(cft->private)) {
	case RES_USAGE:
		if (counter == &memcg->memory)
3106
			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3107
		if (counter == &memcg->memsw)
3108
			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3109 3110
		return (u64)page_counter_read(counter) * PAGE_SIZE;
	case RES_LIMIT:
3111
		return (u64)counter->max * PAGE_SIZE;
3112 3113 3114 3115 3116 3117 3118 3119 3120
	case RES_MAX_USAGE:
		return (u64)counter->watermark * PAGE_SIZE;
	case RES_FAILCNT:
		return counter->failcnt;
	case RES_SOFT_LIMIT:
		return (u64)memcg->soft_limit * PAGE_SIZE;
	default:
		BUG();
	}
B
Balbir Singh 已提交
3121
}
3122

3123
#ifdef CONFIG_MEMCG_KMEM
3124
static int memcg_online_kmem(struct mem_cgroup *memcg)
3125 3126 3127
{
	int memcg_id;

3128 3129 3130
	if (cgroup_memory_nokmem)
		return 0;

3131
	BUG_ON(memcg->kmemcg_id >= 0);
3132
	BUG_ON(memcg->kmem_state);
3133

3134
	memcg_id = memcg_alloc_cache_id();
3135 3136
	if (memcg_id < 0)
		return memcg_id;
3137

3138
	static_branch_inc(&memcg_kmem_enabled_key);
3139
	/*
3140
	 * A memory cgroup is considered kmem-online as soon as it gets
V
Vladimir Davydov 已提交
3141
	 * kmemcg_id. Setting the id after enabling static branching will
3142 3143 3144
	 * guarantee no one starts accounting before all call sites are
	 * patched.
	 */
V
Vladimir Davydov 已提交
3145
	memcg->kmemcg_id = memcg_id;
3146
	memcg->kmem_state = KMEM_ONLINE;
3147
	INIT_LIST_HEAD(&memcg->kmem_caches);
3148 3149

	return 0;
3150 3151
}

3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
	struct cgroup_subsys_state *css;
	struct mem_cgroup *parent, *child;
	int kmemcg_id;

	if (memcg->kmem_state != KMEM_ONLINE)
		return;
	/*
	 * Clear the online state before clearing memcg_caches array
	 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
	 * guarantees that no cache will be created for this cgroup
	 * after we are done (see memcg_create_kmem_cache()).
	 */
	memcg->kmem_state = KMEM_ALLOCATED;

	memcg_deactivate_kmem_caches(memcg);

	kmemcg_id = memcg->kmemcg_id;
	BUG_ON(kmemcg_id < 0);

	parent = parent_mem_cgroup(memcg);
	if (!parent)
		parent = root_mem_cgroup;

	/*
	 * Change kmemcg_id of this cgroup and all its descendants to the
	 * parent's id, and then move all entries from this cgroup's list_lrus
	 * to ones of the parent. After we have finished, all list_lrus
	 * corresponding to this cgroup are guaranteed to remain empty. The
	 * ordering is imposed by list_lru_node->lock taken by
	 * memcg_drain_all_list_lrus().
	 */
3185
	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3186 3187 3188 3189 3190 3191 3192
	css_for_each_descendant_pre(css, &memcg->css) {
		child = mem_cgroup_from_css(css);
		BUG_ON(child->kmemcg_id != kmemcg_id);
		child->kmemcg_id = parent->kmemcg_id;
		if (!memcg->use_hierarchy)
			break;
	}
3193 3194
	rcu_read_unlock();

3195
	memcg_drain_all_list_lrus(kmemcg_id, parent);
3196 3197 3198 3199 3200 3201

	memcg_free_cache_id(kmemcg_id);
}

static void memcg_free_kmem(struct mem_cgroup *memcg)
{
3202 3203 3204 3205
	/* css_alloc() failed, offlining didn't happen */
	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
		memcg_offline_kmem(memcg);

3206 3207 3208 3209 3210 3211
	if (memcg->kmem_state == KMEM_ALLOCATED) {
		memcg_destroy_kmem_caches(memcg);
		static_branch_dec(&memcg_kmem_enabled_key);
		WARN_ON(page_counter_read(&memcg->kmem));
	}
}
3212
#else
3213
static int memcg_online_kmem(struct mem_cgroup *memcg)
3214 3215 3216 3217 3218 3219 3220 3221 3222
{
	return 0;
}
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
}
static void memcg_free_kmem(struct mem_cgroup *memcg)
{
}
3223
#endif /* CONFIG_MEMCG_KMEM */
3224

3225 3226
static int memcg_update_kmem_max(struct mem_cgroup *memcg,
				 unsigned long max)
3227
{
3228
	int ret;
3229

3230 3231 3232
	mutex_lock(&memcg_max_mutex);
	ret = page_counter_set_max(&memcg->kmem, max);
	mutex_unlock(&memcg_max_mutex);
3233
	return ret;
3234
}
3235

3236
static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
V
Vladimir Davydov 已提交
3237 3238 3239
{
	int ret;

3240
	mutex_lock(&memcg_max_mutex);
V
Vladimir Davydov 已提交
3241

3242
	ret = page_counter_set_max(&memcg->tcpmem, max);
V
Vladimir Davydov 已提交
3243 3244 3245
	if (ret)
		goto out;

3246
	if (!memcg->tcpmem_active) {
V
Vladimir Davydov 已提交
3247 3248 3249
		/*
		 * The active flag needs to be written after the static_key
		 * update. This is what guarantees that the socket activation
3250 3251 3252
		 * function is the last one to run. See mem_cgroup_sk_alloc()
		 * for details, and note that we don't mark any socket as
		 * belonging to this memcg until that flag is up.
V
Vladimir Davydov 已提交
3253 3254 3255 3256 3257 3258
		 *
		 * We need to do this, because static_keys will span multiple
		 * sites, but we can't control their order. If we mark a socket
		 * as accounted, but the accounting functions are not patched in
		 * yet, we'll lose accounting.
		 *
3259
		 * We never race with the readers in mem_cgroup_sk_alloc(),
V
Vladimir Davydov 已提交
3260 3261 3262 3263
		 * because when this value change, the code to process it is not
		 * patched in yet.
		 */
		static_branch_inc(&memcg_sockets_enabled_key);
3264
		memcg->tcpmem_active = true;
V
Vladimir Davydov 已提交
3265 3266
	}
out:
3267
	mutex_unlock(&memcg_max_mutex);
V
Vladimir Davydov 已提交
3268 3269 3270
	return ret;
}

3271 3272 3273 3274
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3275 3276
static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
B
Balbir Singh 已提交
3277
{
3278
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3279
	unsigned long nr_pages;
3280 3281
	int ret;

3282
	buf = strstrip(buf);
3283
	ret = page_counter_memparse(buf, "-1", &nr_pages);
3284 3285
	if (ret)
		return ret;
3286

3287
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3288
	case RES_LIMIT:
3289 3290 3291 3292
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3293 3294
		switch (MEMFILE_TYPE(of_cft(of)->private)) {
		case _MEM:
3295
			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3296
			break;
3297
		case _MEMSWAP:
3298
			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3299
			break;
3300
		case _KMEM:
3301
			ret = memcg_update_kmem_max(memcg, nr_pages);
3302
			break;
V
Vladimir Davydov 已提交
3303
		case _TCP:
3304
			ret = memcg_update_tcp_max(memcg, nr_pages);
V
Vladimir Davydov 已提交
3305
			break;
3306
		}
3307
		break;
3308 3309 3310
	case RES_SOFT_LIMIT:
		memcg->soft_limit = nr_pages;
		ret = 0;
3311 3312
		break;
	}
3313
	return ret ?: nbytes;
B
Balbir Singh 已提交
3314 3315
}

3316 3317
static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
				size_t nbytes, loff_t off)
3318
{
3319
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3320
	struct page_counter *counter;
3321

3322 3323 3324 3325 3326 3327 3328 3329 3330 3331
	switch (MEMFILE_TYPE(of_cft(of)->private)) {
	case _MEM:
		counter = &memcg->memory;
		break;
	case _MEMSWAP:
		counter = &memcg->memsw;
		break;
	case _KMEM:
		counter = &memcg->kmem;
		break;
V
Vladimir Davydov 已提交
3332
	case _TCP:
3333
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
3334
		break;
3335 3336 3337
	default:
		BUG();
	}
3338

3339
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3340
	case RES_MAX_USAGE:
3341
		page_counter_reset_watermark(counter);
3342 3343
		break;
	case RES_FAILCNT:
3344
		counter->failcnt = 0;
3345
		break;
3346 3347
	default:
		BUG();
3348
	}
3349

3350
	return nbytes;
3351 3352
}

3353
static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3354 3355
					struct cftype *cft)
{
3356
	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3357 3358
}

3359
#ifdef CONFIG_MMU
3360
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3361 3362
					struct cftype *cft, u64 val)
{
3363
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3364

3365
	if (val & ~MOVE_MASK)
3366
		return -EINVAL;
3367

3368
	/*
3369 3370 3371 3372
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
3373
	 */
3374
	memcg->move_charge_at_immigrate = val;
3375 3376
	return 0;
}
3377
#else
3378
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3379 3380 3381 3382 3383
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
3384

3385
#ifdef CONFIG_NUMA
3386
static int memcg_numa_stat_show(struct seq_file *m, void *v)
3387
{
3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399
	struct numa_stat {
		const char *name;
		unsigned int lru_mask;
	};

	static const struct numa_stat stats[] = {
		{ "total", LRU_ALL },
		{ "file", LRU_ALL_FILE },
		{ "anon", LRU_ALL_ANON },
		{ "unevictable", BIT(LRU_UNEVICTABLE) },
	};
	const struct numa_stat *stat;
3400
	int nid;
3401
	unsigned long nr;
3402
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3403

3404 3405 3406 3407 3408 3409 3410 3411 3412
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
		seq_printf(m, "%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
							  stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3413 3414
	}

3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		struct mem_cgroup *iter;

		nr = 0;
		for_each_mem_cgroup_tree(iter, memcg)
			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = 0;
			for_each_mem_cgroup_tree(iter, memcg)
				nr += mem_cgroup_node_nr_lru_pages(
					iter, nid, stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3430 3431 3432 3433 3434 3435
	}

	return 0;
}
#endif /* CONFIG_NUMA */

3436
/* Universal VM events cgroup1 shows, original sort order */
3437
static const unsigned int memcg1_events[] = {
3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450
	PGPGIN,
	PGPGOUT,
	PGFAULT,
	PGMAJFAULT,
};

static const char *const memcg1_event_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

3451
static int memcg_stat_show(struct seq_file *m, void *v)
3452
{
3453
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3454
	unsigned long memory, memsw;
3455 3456
	struct mem_cgroup *mi;
	unsigned int i;
3457
	struct accumulated_stats acc;
3458

3459
	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3460 3461
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);

3462 3463
	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3464
			continue;
3465
		seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3466
			   memcg_page_state(memcg, memcg1_stats[i]) *
3467
			   PAGE_SIZE);
3468
	}
L
Lee Schermerhorn 已提交
3469

3470 3471
	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
		seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3472
			   memcg_sum_events(memcg, memcg1_events[i]));
3473 3474 3475 3476 3477

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
3478
	/* Hierarchical information */
3479 3480
	memory = memsw = PAGE_COUNTER_MAX;
	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3481 3482
		memory = min(memory, mi->memory.max);
		memsw = min(memsw, mi->memsw.max);
3483
	}
3484 3485
	seq_printf(m, "hierarchical_memory_limit %llu\n",
		   (u64)memory * PAGE_SIZE);
3486
	if (do_memsw_account())
3487 3488
		seq_printf(m, "hierarchical_memsw_limit %llu\n",
			   (u64)memsw * PAGE_SIZE);
K
KOSAKI Motohiro 已提交
3489

3490 3491 3492 3493 3494 3495
	memset(&acc, 0, sizeof(acc));
	acc.stats_size = ARRAY_SIZE(memcg1_stats);
	acc.stats_array = memcg1_stats;
	acc.events_size = ARRAY_SIZE(memcg1_events);
	acc.events_array = memcg1_events;
	accumulate_memcg_tree(memcg, &acc);
3496

3497
	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3498
		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3499
			continue;
3500 3501
		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
			   (u64)acc.stat[i] * PAGE_SIZE);
3502 3503
	}

3504 3505 3506
	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
		seq_printf(m, "total_%s %llu\n", memcg1_event_names[i],
			   (u64)acc.events[i]);
3507

3508 3509 3510
	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i],
			   (u64)acc.lru_pages[i] * PAGE_SIZE);
K
KAMEZAWA Hiroyuki 已提交
3511

K
KOSAKI Motohiro 已提交
3512 3513
#ifdef CONFIG_DEBUG_VM
	{
3514 3515
		pg_data_t *pgdat;
		struct mem_cgroup_per_node *mz;
3516
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
3517 3518 3519
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

3520 3521 3522
		for_each_online_pgdat(pgdat) {
			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
			rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
3523

3524 3525 3526 3527 3528
			recent_rotated[0] += rstat->recent_rotated[0];
			recent_rotated[1] += rstat->recent_rotated[1];
			recent_scanned[0] += rstat->recent_scanned[0];
			recent_scanned[1] += rstat->recent_scanned[1];
		}
3529 3530 3531 3532
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
3533 3534 3535
	}
#endif

3536 3537 3538
	return 0;
}

3539 3540
static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
K
KOSAKI Motohiro 已提交
3541
{
3542
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3543

3544
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
3545 3546
}

3547 3548
static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
K
KOSAKI Motohiro 已提交
3549
{
3550
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3551

3552
	if (val > 100)
K
KOSAKI Motohiro 已提交
3553 3554
		return -EINVAL;

3555
	if (css->parent)
3556 3557 3558
		memcg->swappiness = val;
	else
		vm_swappiness = val;
3559

K
KOSAKI Motohiro 已提交
3560 3561 3562
	return 0;
}

3563 3564 3565
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
3566
	unsigned long usage;
3567 3568 3569 3570
	int i;

	rcu_read_lock();
	if (!swap)
3571
		t = rcu_dereference(memcg->thresholds.primary);
3572
	else
3573
		t = rcu_dereference(memcg->memsw_thresholds.primary);
3574 3575 3576 3577

	if (!t)
		goto unlock;

3578
	usage = mem_cgroup_usage(memcg, swap);
3579 3580

	/*
3581
	 * current_threshold points to threshold just below or equal to usage.
3582 3583 3584
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
3585
	i = t->current_threshold;
3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
3609
	t->current_threshold = i - 1;
3610 3611 3612 3613 3614 3615
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
3616 3617
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
3618
		if (do_memsw_account())
3619 3620 3621 3622
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
3623 3624 3625 3626 3627 3628 3629
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

3630 3631 3632 3633 3634 3635 3636
	if (_a->threshold > _b->threshold)
		return 1;

	if (_a->threshold < _b->threshold)
		return -1;

	return 0;
3637 3638
}

3639
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3640 3641 3642
{
	struct mem_cgroup_eventfd_list *ev;

3643 3644
	spin_lock(&memcg_oom_lock);

3645
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
3646
		eventfd_signal(ev->eventfd, 1);
3647 3648

	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3649 3650 3651
	return 0;
}

3652
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3653
{
K
KAMEZAWA Hiroyuki 已提交
3654 3655
	struct mem_cgroup *iter;

3656
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
3657
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
3658 3659
}

3660
static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3661
	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3662
{
3663 3664
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3665 3666
	unsigned long threshold;
	unsigned long usage;
3667
	int i, size, ret;
3668

3669
	ret = page_counter_memparse(args, "-1", &threshold);
3670 3671 3672 3673
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
3674

3675
	if (type == _MEM) {
3676
		thresholds = &memcg->thresholds;
3677
		usage = mem_cgroup_usage(memcg, false);
3678
	} else if (type == _MEMSWAP) {
3679
		thresholds = &memcg->memsw_thresholds;
3680
		usage = mem_cgroup_usage(memcg, true);
3681
	} else
3682 3683 3684
		BUG();

	/* Check if a threshold crossed before adding a new one */
3685
	if (thresholds->primary)
3686 3687
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

3688
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3689 3690

	/* Allocate memory for new array of thresholds */
3691
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3692
			GFP_KERNEL);
3693
	if (!new) {
3694 3695 3696
		ret = -ENOMEM;
		goto unlock;
	}
3697
	new->size = size;
3698 3699

	/* Copy thresholds (if any) to new array */
3700 3701
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3702
				sizeof(struct mem_cgroup_threshold));
3703 3704
	}

3705
	/* Add new threshold */
3706 3707
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
3708 3709

	/* Sort thresholds. Registering of new threshold isn't time-critical */
3710
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3711 3712 3713
			compare_thresholds, NULL);

	/* Find current threshold */
3714
	new->current_threshold = -1;
3715
	for (i = 0; i < size; i++) {
3716
		if (new->entries[i].threshold <= usage) {
3717
			/*
3718 3719
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
3720 3721
			 * it here.
			 */
3722
			++new->current_threshold;
3723 3724
		} else
			break;
3725 3726
	}

3727 3728 3729 3730 3731
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
3732

3733
	/* To be sure that nobody uses thresholds */
3734 3735 3736 3737 3738 3739 3740 3741
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

3742
static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3743 3744
	struct eventfd_ctx *eventfd, const char *args)
{
3745
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
T
Tejun Heo 已提交
3746 3747
}

3748
static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3749 3750
	struct eventfd_ctx *eventfd, const char *args)
{
3751
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
T
Tejun Heo 已提交
3752 3753
}

3754
static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3755
	struct eventfd_ctx *eventfd, enum res_type type)
3756
{
3757 3758
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3759
	unsigned long usage;
3760
	int i, j, size;
3761 3762

	mutex_lock(&memcg->thresholds_lock);
3763 3764

	if (type == _MEM) {
3765
		thresholds = &memcg->thresholds;
3766
		usage = mem_cgroup_usage(memcg, false);
3767
	} else if (type == _MEMSWAP) {
3768
		thresholds = &memcg->memsw_thresholds;
3769
		usage = mem_cgroup_usage(memcg, true);
3770
	} else
3771 3772
		BUG();

3773 3774 3775
	if (!thresholds->primary)
		goto unlock;

3776 3777 3778 3779
	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
3780 3781 3782
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
3783 3784 3785
			size++;
	}

3786
	new = thresholds->spare;
3787

3788 3789
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
3790 3791
		kfree(new);
		new = NULL;
3792
		goto swap_buffers;
3793 3794
	}

3795
	new->size = size;
3796 3797

	/* Copy thresholds and find current threshold */
3798 3799 3800
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
3801 3802
			continue;

3803
		new->entries[j] = thresholds->primary->entries[i];
3804
		if (new->entries[j].threshold <= usage) {
3805
			/*
3806
			 * new->current_threshold will not be used
3807 3808 3809
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
3810
			++new->current_threshold;
3811 3812 3813 3814
		}
		j++;
	}

3815
swap_buffers:
3816 3817
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
3818

3819
	rcu_assign_pointer(thresholds->primary, new);
3820

3821
	/* To be sure that nobody uses thresholds */
3822
	synchronize_rcu();
3823 3824 3825 3826 3827 3828

	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}
3829
unlock:
3830 3831
	mutex_unlock(&memcg->thresholds_lock);
}
3832

3833
static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3834 3835
	struct eventfd_ctx *eventfd)
{
3836
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
T
Tejun Heo 已提交
3837 3838
}

3839
static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3840 3841
	struct eventfd_ctx *eventfd)
{
3842
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
T
Tejun Heo 已提交
3843 3844
}

3845
static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3846
	struct eventfd_ctx *eventfd, const char *args)
K
KAMEZAWA Hiroyuki 已提交
3847 3848 3849 3850 3851 3852 3853
{
	struct mem_cgroup_eventfd_list *event;

	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

3854
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3855 3856 3857 3858 3859

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
3860
	if (memcg->under_oom)
K
KAMEZAWA Hiroyuki 已提交
3861
		eventfd_signal(eventfd, 1);
3862
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3863 3864 3865 3866

	return 0;
}

3867
static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3868
	struct eventfd_ctx *eventfd)
K
KAMEZAWA Hiroyuki 已提交
3869 3870 3871
{
	struct mem_cgroup_eventfd_list *ev, *tmp;

3872
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3873

3874
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
3875 3876 3877 3878 3879 3880
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

3881
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3882 3883
}

3884
static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3885
{
3886
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3887

3888
	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3889
	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
R
Roman Gushchin 已提交
3890 3891
	seq_printf(sf, "oom_kill %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
3892 3893 3894
	return 0;
}

3895
static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3896 3897
	struct cftype *cft, u64 val)
{
3898
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3899 3900

	/* cannot set to root cgroup and only 0 and 1 are allowed */
3901
	if (!css->parent || !((val == 0) || (val == 1)))
3902 3903
		return -EINVAL;

3904
	memcg->oom_kill_disable = val;
3905
	if (!val)
3906
		memcg_oom_recover(memcg);
3907

3908 3909 3910
	return 0;
}

3911 3912
#ifdef CONFIG_CGROUP_WRITEBACK

T
Tejun Heo 已提交
3913 3914 3915 3916 3917 3918 3919 3920 3921 3922
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return wb_domain_init(&memcg->cgwb_domain, gfp);
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
	wb_domain_exit(&memcg->cgwb_domain);
}

3923 3924 3925 3926 3927
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
	wb_domain_size_changed(&memcg->cgwb_domain);
}

T
Tejun Heo 已提交
3928 3929 3930 3931 3932 3933 3934 3935 3936 3937
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);

	if (!memcg->css.parent)
		return NULL;

	return &memcg->cgwb_domain;
}

3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page().
 */
static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
{
	long x = atomic_long_read(&memcg->stat[idx]);
	int cpu;

	for_each_online_cpu(cpu)
		x += per_cpu_ptr(memcg->stat_cpu, cpu)->count[idx];
	if (x < 0)
		x = 0;
	return x;
}

3954 3955 3956
/**
 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
 * @wb: bdi_writeback in question
3957 3958
 * @pfilepages: out parameter for number of file pages
 * @pheadroom: out parameter for number of allocatable pages according to memcg
3959 3960 3961
 * @pdirty: out parameter for number of dirty pages
 * @pwriteback: out parameter for number of pages under writeback
 *
3962 3963 3964
 * Determine the numbers of file, headroom, dirty, and writeback pages in
 * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
 * is a bit more involved.
3965
 *
3966 3967 3968 3969 3970
 * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
 * headroom is calculated as the lowest headroom of itself and the
 * ancestors.  Note that this doesn't consider the actual amount of
 * available memory in the system.  The caller should further cap
 * *@pheadroom accordingly.
3971
 */
3972 3973 3974
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback)
3975 3976 3977 3978
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
	struct mem_cgroup *parent;

3979
	*pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
3980 3981

	/* this should eventually include NR_UNSTABLE_NFS */
3982
	*pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
3983 3984 3985
	*pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
						     (1 << LRU_ACTIVE_FILE));
	*pheadroom = PAGE_COUNTER_MAX;
3986 3987

	while ((parent = parent_mem_cgroup(memcg))) {
3988
		unsigned long ceiling = min(memcg->memory.max, memcg->high);
3989 3990
		unsigned long used = page_counter_read(&memcg->memory);

3991
		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3992 3993 3994 3995
		memcg = parent;
	}
}

T
Tejun Heo 已提交
3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006
#else	/* CONFIG_CGROUP_WRITEBACK */

static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return 0;
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
}

4007 4008 4009 4010
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
}

4011 4012
#endif	/* CONFIG_CGROUP_WRITEBACK */

4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025
/*
 * DO NOT USE IN NEW FILES.
 *
 * "cgroup.event_control" implementation.
 *
 * This is way over-engineered.  It tries to support fully configurable
 * events for each user.  Such level of flexibility is completely
 * unnecessary especially in the light of the planned unified hierarchy.
 *
 * Please deprecate this and replace with something simpler if at all
 * possible.
 */

4026 4027 4028 4029 4030
/*
 * Unregister event and free resources.
 *
 * Gets called from workqueue.
 */
4031
static void memcg_event_remove(struct work_struct *work)
4032
{
4033 4034
	struct mem_cgroup_event *event =
		container_of(work, struct mem_cgroup_event, remove);
4035
	struct mem_cgroup *memcg = event->memcg;
4036 4037 4038

	remove_wait_queue(event->wqh, &event->wait);

4039
	event->unregister_event(memcg, event->eventfd);
4040 4041 4042 4043 4044 4045

	/* Notify userspace the event is going away. */
	eventfd_signal(event->eventfd, 1);

	eventfd_ctx_put(event->eventfd);
	kfree(event);
4046
	css_put(&memcg->css);
4047 4048 4049
}

/*
4050
 * Gets called on EPOLLHUP on eventfd when user closes it.
4051 4052 4053
 *
 * Called with wqh->lock held and interrupts disabled.
 */
4054
static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
4055
			    int sync, void *key)
4056
{
4057 4058
	struct mem_cgroup_event *event =
		container_of(wait, struct mem_cgroup_event, wait);
4059
	struct mem_cgroup *memcg = event->memcg;
A
Al Viro 已提交
4060
	__poll_t flags = key_to_poll(key);
4061

4062
	if (flags & EPOLLHUP) {
4063 4064 4065 4066 4067 4068 4069 4070 4071
		/*
		 * If the event has been detached at cgroup removal, we
		 * can simply return knowing the other side will cleanup
		 * for us.
		 *
		 * We can't race against event freeing since the other
		 * side will require wqh->lock via remove_wait_queue(),
		 * which we hold.
		 */
4072
		spin_lock(&memcg->event_list_lock);
4073 4074 4075 4076 4077 4078 4079 4080
		if (!list_empty(&event->list)) {
			list_del_init(&event->list);
			/*
			 * We are in atomic context, but cgroup_event_remove()
			 * may sleep, so we have to call it in workqueue.
			 */
			schedule_work(&event->remove);
		}
4081
		spin_unlock(&memcg->event_list_lock);
4082 4083 4084 4085 4086
	}

	return 0;
}

4087
static void memcg_event_ptable_queue_proc(struct file *file,
4088 4089
		wait_queue_head_t *wqh, poll_table *pt)
{
4090 4091
	struct mem_cgroup_event *event =
		container_of(pt, struct mem_cgroup_event, pt);
4092 4093 4094 4095 4096 4097

	event->wqh = wqh;
	add_wait_queue(wqh, &event->wait);
}

/*
4098 4099
 * DO NOT USE IN NEW FILES.
 *
4100 4101 4102 4103 4104
 * Parse input and register new cgroup event handler.
 *
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
4105 4106
static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
					 char *buf, size_t nbytes, loff_t off)
4107
{
4108
	struct cgroup_subsys_state *css = of_css(of);
4109
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4110
	struct mem_cgroup_event *event;
4111 4112 4113 4114
	struct cgroup_subsys_state *cfile_css;
	unsigned int efd, cfd;
	struct fd efile;
	struct fd cfile;
4115
	const char *name;
4116 4117 4118
	char *endp;
	int ret;

4119 4120 4121
	buf = strstrip(buf);

	efd = simple_strtoul(buf, &endp, 10);
4122 4123
	if (*endp != ' ')
		return -EINVAL;
4124
	buf = endp + 1;
4125

4126
	cfd = simple_strtoul(buf, &endp, 10);
4127 4128
	if ((*endp != ' ') && (*endp != '\0'))
		return -EINVAL;
4129
	buf = endp + 1;
4130 4131 4132 4133 4134

	event = kzalloc(sizeof(*event), GFP_KERNEL);
	if (!event)
		return -ENOMEM;

4135
	event->memcg = memcg;
4136
	INIT_LIST_HEAD(&event->list);
4137 4138 4139
	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
	INIT_WORK(&event->remove, memcg_event_remove);
4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164

	efile = fdget(efd);
	if (!efile.file) {
		ret = -EBADF;
		goto out_kfree;
	}

	event->eventfd = eventfd_ctx_fileget(efile.file);
	if (IS_ERR(event->eventfd)) {
		ret = PTR_ERR(event->eventfd);
		goto out_put_efile;
	}

	cfile = fdget(cfd);
	if (!cfile.file) {
		ret = -EBADF;
		goto out_put_eventfd;
	}

	/* the process need read permission on control file */
	/* AV: shouldn't we check that it's been opened for read instead? */
	ret = inode_permission(file_inode(cfile.file), MAY_READ);
	if (ret < 0)
		goto out_put_cfile;

4165 4166 4167 4168 4169
	/*
	 * Determine the event callbacks and set them in @event.  This used
	 * to be done via struct cftype but cgroup core no longer knows
	 * about these events.  The following is crude but the whole thing
	 * is for compatibility anyway.
4170 4171
	 *
	 * DO NOT ADD NEW FILES.
4172
	 */
A
Al Viro 已提交
4173
	name = cfile.file->f_path.dentry->d_name.name;
4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184

	if (!strcmp(name, "memory.usage_in_bytes")) {
		event->register_event = mem_cgroup_usage_register_event;
		event->unregister_event = mem_cgroup_usage_unregister_event;
	} else if (!strcmp(name, "memory.oom_control")) {
		event->register_event = mem_cgroup_oom_register_event;
		event->unregister_event = mem_cgroup_oom_unregister_event;
	} else if (!strcmp(name, "memory.pressure_level")) {
		event->register_event = vmpressure_register_event;
		event->unregister_event = vmpressure_unregister_event;
	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
T
Tejun Heo 已提交
4185 4186
		event->register_event = memsw_cgroup_usage_register_event;
		event->unregister_event = memsw_cgroup_usage_unregister_event;
4187 4188 4189 4190 4191
	} else {
		ret = -EINVAL;
		goto out_put_cfile;
	}

4192
	/*
4193 4194 4195
	 * Verify @cfile should belong to @css.  Also, remaining events are
	 * automatically removed on cgroup destruction but the removal is
	 * asynchronous, so take an extra ref on @css.
4196
	 */
A
Al Viro 已提交
4197
	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4198
					       &memory_cgrp_subsys);
4199
	ret = -EINVAL;
4200
	if (IS_ERR(cfile_css))
4201
		goto out_put_cfile;
4202 4203
	if (cfile_css != css) {
		css_put(cfile_css);
4204
		goto out_put_cfile;
4205
	}
4206

4207
	ret = event->register_event(memcg, event->eventfd, buf);
4208 4209 4210
	if (ret)
		goto out_put_css;

4211
	vfs_poll(efile.file, &event->pt);
4212

4213 4214 4215
	spin_lock(&memcg->event_list_lock);
	list_add(&event->list, &memcg->event_list);
	spin_unlock(&memcg->event_list_lock);
4216 4217 4218 4219

	fdput(cfile);
	fdput(efile);

4220
	return nbytes;
4221 4222

out_put_css:
4223
	css_put(css);
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235
out_put_cfile:
	fdput(cfile);
out_put_eventfd:
	eventfd_ctx_put(event->eventfd);
out_put_efile:
	fdput(efile);
out_kfree:
	kfree(event);

	return ret;
}

4236
static struct cftype mem_cgroup_legacy_files[] = {
B
Balbir Singh 已提交
4237
	{
4238
		.name = "usage_in_bytes",
4239
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4240
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4241
	},
4242 4243
	{
		.name = "max_usage_in_bytes",
4244
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4245
		.write = mem_cgroup_reset,
4246
		.read_u64 = mem_cgroup_read_u64,
4247
	},
B
Balbir Singh 已提交
4248
	{
4249
		.name = "limit_in_bytes",
4250
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4251
		.write = mem_cgroup_write,
4252
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4253
	},
4254 4255 4256
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4257
		.write = mem_cgroup_write,
4258
		.read_u64 = mem_cgroup_read_u64,
4259
	},
B
Balbir Singh 已提交
4260 4261
	{
		.name = "failcnt",
4262
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4263
		.write = mem_cgroup_reset,
4264
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4265
	},
4266 4267
	{
		.name = "stat",
4268
		.seq_show = memcg_stat_show,
4269
	},
4270 4271
	{
		.name = "force_empty",
4272
		.write = mem_cgroup_force_empty_write,
4273
	},
4274 4275 4276 4277 4278
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
4279
	{
4280
		.name = "cgroup.event_control",		/* XXX: for compat */
4281
		.write = memcg_write_event_control,
4282
		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4283
	},
K
KOSAKI Motohiro 已提交
4284 4285 4286 4287 4288
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4289 4290 4291 4292 4293
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4294 4295
	{
		.name = "oom_control",
4296
		.seq_show = mem_cgroup_oom_control_read,
4297
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4298 4299
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
4300 4301 4302
	{
		.name = "pressure_level",
	},
4303 4304 4305
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
4306
		.seq_show = memcg_numa_stat_show,
4307 4308
	},
#endif
4309 4310 4311
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4312
		.write = mem_cgroup_write,
4313
		.read_u64 = mem_cgroup_read_u64,
4314 4315 4316 4317
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4318
		.read_u64 = mem_cgroup_read_u64,
4319 4320 4321 4322
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4323
		.write = mem_cgroup_reset,
4324
		.read_u64 = mem_cgroup_read_u64,
4325 4326 4327 4328
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4329
		.write = mem_cgroup_reset,
4330
		.read_u64 = mem_cgroup_read_u64,
4331
	},
Y
Yang Shi 已提交
4332
#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
4333 4334
	{
		.name = "kmem.slabinfo",
4335 4336 4337
		.seq_start = memcg_slab_start,
		.seq_next = memcg_slab_next,
		.seq_stop = memcg_slab_stop,
4338
		.seq_show = memcg_slab_show,
4339 4340
	},
#endif
V
Vladimir Davydov 已提交
4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363
	{
		.name = "kmem.tcp.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.failcnt",
		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
4364
	{ },	/* terminate */
4365
};
4366

4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392
/*
 * Private memory cgroup IDR
 *
 * Swap-out records and page cache shadow entries need to store memcg
 * references in constrained space, so we maintain an ID space that is
 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
 * memory-controlled cgroups to 64k.
 *
 * However, there usually are many references to the oflline CSS after
 * the cgroup has been destroyed, such as page cache or reclaimable
 * slab objects, that don't need to hang on to the ID. We want to keep
 * those dead CSS from occupying IDs, or we might quickly exhaust the
 * relatively small ID space and prevent the creation of new cgroups
 * even when there are much fewer than 64k cgroups - possibly none.
 *
 * Maintain a private 16-bit ID space for memcg, and allow the ID to
 * be freed and recycled when it's no longer needed, which is usually
 * when the CSS is offlined.
 *
 * The only exception to that are records of swapped out tmpfs/shmem
 * pages that need to be attributed to live ancestors on swapin. But
 * those references are manageable from userspace.
 */

static DEFINE_IDR(mem_cgroup_idr);

4393 4394 4395 4396 4397 4398 4399 4400
static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
{
	if (memcg->id.id > 0) {
		idr_remove(&mem_cgroup_idr, memcg->id.id);
		memcg->id.id = 0;
	}
}

4401
static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4402
{
4403
	VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4404
	atomic_add(n, &memcg->id.ref);
4405 4406
}

4407
static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4408
{
4409
	VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4410
	if (atomic_sub_and_test(n, &memcg->id.ref)) {
4411
		mem_cgroup_id_remove(memcg);
4412 4413 4414 4415 4416 4417

		/* Memcg ID pins CSS */
		css_put(&memcg->css);
	}
}

4418 4419 4420 4421 4422 4423 4424 4425 4426 4427
static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
{
	mem_cgroup_id_get_many(memcg, 1);
}

static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
{
	mem_cgroup_id_put_many(memcg, 1);
}

4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439
/**
 * mem_cgroup_from_id - look up a memcg from a memcg id
 * @id: the memcg id to look up
 *
 * Caller must hold rcu_read_lock().
 */
struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return idr_find(&mem_cgroup_idr, id);
}

4440
static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4441 4442
{
	struct mem_cgroup_per_node *pn;
4443
	int tmp = node;
4444 4445 4446 4447 4448 4449 4450 4451
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4452 4453
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4454
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4455 4456
	if (!pn)
		return 1;
4457

4458 4459
	pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
	if (!pn->lruvec_stat_cpu) {
4460 4461 4462 4463
		kfree(pn);
		return 1;
	}

4464 4465 4466 4467 4468
	lruvec_init(&pn->lruvec);
	pn->usage_in_excess = 0;
	pn->on_tree = false;
	pn->memcg = memcg;

4469
	memcg->nodeinfo[node] = pn;
4470 4471 4472
	return 0;
}

4473
static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4474
{
4475 4476
	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];

M
Michal Hocko 已提交
4477 4478 4479
	if (!pn)
		return;

4480
	free_percpu(pn->lruvec_stat_cpu);
4481
	kfree(pn);
4482 4483
}

4484
static void __mem_cgroup_free(struct mem_cgroup *memcg)
4485
{
4486
	int node;
4487

4488
	for_each_node(node)
4489
		free_mem_cgroup_per_node_info(memcg, node);
4490
	free_percpu(memcg->stat_cpu);
4491
	kfree(memcg);
4492
}
4493

4494 4495 4496 4497 4498 4499
static void mem_cgroup_free(struct mem_cgroup *memcg)
{
	memcg_wb_domain_exit(memcg);
	__mem_cgroup_free(memcg);
}

4500
static struct mem_cgroup *mem_cgroup_alloc(void)
B
Balbir Singh 已提交
4501
{
4502
	struct mem_cgroup *memcg;
4503
	size_t size;
4504
	int node;
B
Balbir Singh 已提交
4505

4506 4507 4508 4509
	size = sizeof(struct mem_cgroup);
	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);

	memcg = kzalloc(size, GFP_KERNEL);
4510
	if (!memcg)
4511 4512
		return NULL;

4513 4514 4515 4516 4517 4518
	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
				 1, MEM_CGROUP_ID_MAX,
				 GFP_KERNEL);
	if (memcg->id.id < 0)
		goto fail;

4519 4520
	memcg->stat_cpu = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat_cpu)
4521
		goto fail;
4522

B
Bob Liu 已提交
4523
	for_each_node(node)
4524
		if (alloc_mem_cgroup_per_node_info(memcg, node))
4525
			goto fail;
4526

4527 4528
	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
		goto fail;
4529

4530
	INIT_WORK(&memcg->high_work, high_work_func);
4531 4532 4533 4534
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);
4535
	vmpressure_init(&memcg->vmpressure);
4536 4537
	INIT_LIST_HEAD(&memcg->event_list);
	spin_lock_init(&memcg->event_list_lock);
4538
	memcg->socket_pressure = jiffies;
4539
#ifdef CONFIG_MEMCG_KMEM
V
Vladimir Davydov 已提交
4540 4541
	memcg->kmemcg_id = -1;
#endif
4542 4543 4544
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&memcg->cgwb_list);
#endif
4545
	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4546 4547
	return memcg;
fail:
4548
	mem_cgroup_id_remove(memcg);
4549
	__mem_cgroup_free(memcg);
4550
	return NULL;
4551 4552
}

4553 4554
static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4555
{
4556 4557 4558
	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
	struct mem_cgroup *memcg;
	long error = -ENOMEM;
4559

4560 4561 4562
	memcg = mem_cgroup_alloc();
	if (!memcg)
		return ERR_PTR(error);
4563

4564 4565 4566 4567 4568 4569 4570 4571
	memcg->high = PAGE_COUNTER_MAX;
	memcg->soft_limit = PAGE_COUNTER_MAX;
	if (parent) {
		memcg->swappiness = mem_cgroup_swappiness(parent);
		memcg->oom_kill_disable = parent->oom_kill_disable;
	}
	if (parent && parent->use_hierarchy) {
		memcg->use_hierarchy = true;
4572
		page_counter_init(&memcg->memory, &parent->memory);
4573
		page_counter_init(&memcg->swap, &parent->swap);
4574 4575
		page_counter_init(&memcg->memsw, &parent->memsw);
		page_counter_init(&memcg->kmem, &parent->kmem);
4576
		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4577
	} else {
4578
		page_counter_init(&memcg->memory, NULL);
4579
		page_counter_init(&memcg->swap, NULL);
4580 4581
		page_counter_init(&memcg->memsw, NULL);
		page_counter_init(&memcg->kmem, NULL);
4582
		page_counter_init(&memcg->tcpmem, NULL);
4583 4584 4585 4586 4587
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
4588
		if (parent != root_mem_cgroup)
4589
			memory_cgrp_subsys.broken_hierarchy = true;
4590
	}
4591

4592 4593 4594 4595 4596 4597
	/* The following stuff does not apply to the root */
	if (!parent) {
		root_mem_cgroup = memcg;
		return &memcg->css;
	}

4598
	error = memcg_online_kmem(memcg);
4599 4600
	if (error)
		goto fail;
4601

4602
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4603
		static_branch_inc(&memcg_sockets_enabled_key);
4604

4605 4606
	return &memcg->css;
fail:
4607
	mem_cgroup_id_remove(memcg);
4608
	mem_cgroup_free(memcg);
4609
	return ERR_PTR(-ENOMEM);
4610 4611
}

4612
static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4613
{
4614 4615
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

4616 4617 4618 4619 4620 4621 4622 4623 4624 4625
	/*
	 * A memcg must be visible for memcg_expand_shrinker_maps()
	 * by the time the maps are allocated. So, we allocate maps
	 * here, when for_each_mem_cgroup() can't skip it.
	 */
	if (memcg_alloc_shrinker_maps(memcg)) {
		mem_cgroup_id_remove(memcg);
		return -ENOMEM;
	}

4626
	/* Online state pins memcg ID, memcg ID pins CSS */
4627
	atomic_set(&memcg->id.ref, 1);
4628
	css_get(css);
4629
	return 0;
B
Balbir Singh 已提交
4630 4631
}

4632
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4633
{
4634
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4635
	struct mem_cgroup_event *event, *tmp;
4636 4637 4638 4639 4640 4641

	/*
	 * Unregister events and notify userspace.
	 * Notify userspace about cgroup removing only after rmdir of cgroup
	 * directory to avoid race between userspace and kernelspace.
	 */
4642 4643
	spin_lock(&memcg->event_list_lock);
	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4644 4645 4646
		list_del_init(&event->list);
		schedule_work(&event->remove);
	}
4647
	spin_unlock(&memcg->event_list_lock);
4648

R
Roman Gushchin 已提交
4649
	page_counter_set_min(&memcg->memory, 0);
4650
	page_counter_set_low(&memcg->memory, 0);
4651

4652
	memcg_offline_kmem(memcg);
4653
	wb_memcg_offline(memcg);
4654 4655

	mem_cgroup_id_put(memcg);
4656 4657
}

4658 4659 4660 4661 4662 4663 4664
static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	invalidate_reclaim_iterators(memcg);
}

4665
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
B
Balbir Singh 已提交
4666
{
4667
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4668

4669
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4670
		static_branch_dec(&memcg_sockets_enabled_key);
4671

4672
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
V
Vladimir Davydov 已提交
4673
		static_branch_dec(&memcg_sockets_enabled_key);
4674

4675 4676 4677
	vmpressure_cleanup(&memcg->vmpressure);
	cancel_work_sync(&memcg->high_work);
	mem_cgroup_remove_from_trees(memcg);
4678
	memcg_free_shrinker_maps(memcg);
4679
	memcg_free_kmem(memcg);
4680
	mem_cgroup_free(memcg);
B
Balbir Singh 已提交
4681 4682
}

4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699
/**
 * mem_cgroup_css_reset - reset the states of a mem_cgroup
 * @css: the target css
 *
 * Reset the states of the mem_cgroup associated with @css.  This is
 * invoked when the userland requests disabling on the default hierarchy
 * but the memcg is pinned through dependency.  The memcg should stop
 * applying policies and should revert to the vanilla state as it may be
 * made visible again.
 *
 * The current implementation only resets the essential configurations.
 * This needs to be expanded to cover all the visible parts.
 */
static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

4700 4701 4702 4703 4704
	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
R
Roman Gushchin 已提交
4705
	page_counter_set_min(&memcg->memory, 0);
4706
	page_counter_set_low(&memcg->memory, 0);
4707
	memcg->high = PAGE_COUNTER_MAX;
4708
	memcg->soft_limit = PAGE_COUNTER_MAX;
4709
	memcg_wb_domain_size_changed(memcg);
4710 4711
}

4712
#ifdef CONFIG_MMU
4713
/* Handlers for move charge at task migration. */
4714
static int mem_cgroup_do_precharge(unsigned long count)
4715
{
4716
	int ret;
4717

4718 4719
	/* Try a single bulk charge without reclaim first, kswapd may wake */
	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4720
	if (!ret) {
4721 4722 4723
		mc.precharge += count;
		return ret;
	}
4724

4725
	/* Try charges one by one with reclaim, but do not retry */
4726
	while (count--) {
4727
		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4728 4729
		if (ret)
			return ret;
4730
		mc.precharge++;
4731
		cond_resched();
4732
	}
4733
	return 0;
4734 4735 4736 4737
}

union mc_target {
	struct page	*page;
4738
	swp_entry_t	ent;
4739 4740 4741
};

enum mc_target_type {
4742
	MC_TARGET_NONE = 0,
4743
	MC_TARGET_PAGE,
4744
	MC_TARGET_SWAP,
4745
	MC_TARGET_DEVICE,
4746 4747
};

D
Daisuke Nishimura 已提交
4748 4749
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
4750
{
4751
	struct page *page = _vm_normal_page(vma, addr, ptent, true);
4752

D
Daisuke Nishimura 已提交
4753 4754 4755
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
4756
		if (!(mc.flags & MOVE_ANON))
D
Daisuke Nishimura 已提交
4757
			return NULL;
4758 4759 4760 4761
	} else {
		if (!(mc.flags & MOVE_FILE))
			return NULL;
	}
D
Daisuke Nishimura 已提交
4762 4763 4764 4765 4766 4767
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

4768
#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
D
Daisuke Nishimura 已提交
4769
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4770
			pte_t ptent, swp_entry_t *entry)
D
Daisuke Nishimura 已提交
4771 4772 4773 4774
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

4775
	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
D
Daisuke Nishimura 已提交
4776
		return NULL;
4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793

	/*
	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
	 * a device and because they are not accessible by CPU they are store
	 * as special swap entry in the CPU page table.
	 */
	if (is_device_private_entry(ent)) {
		page = device_private_entry_to_page(ent);
		/*
		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
		 * a refcount of 1 when free (unlike normal page)
		 */
		if (!page_ref_add_unless(page, 1, 1))
			return NULL;
		return page;
	}

4794 4795 4796 4797
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
4798
	page = find_get_page(swap_address_space(ent), swp_offset(ent));
4799
	if (do_memsw_account())
D
Daisuke Nishimura 已提交
4800 4801 4802 4803
		entry->val = ent.val;

	return page;
}
4804 4805
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4806
			pte_t ptent, swp_entry_t *entry)
4807 4808 4809 4810
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
4811

4812 4813 4814 4815 4816 4817 4818 4819 4820
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
4821
	if (!(mc.flags & MOVE_FILE))
4822 4823 4824
		return NULL;

	mapping = vma->vm_file->f_mapping;
4825
	pgoff = linear_page_index(vma, addr);
4826 4827

	/* page is moved even if it's not RSS of this task(page-faulted). */
4828 4829
#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
4830 4831 4832 4833
	if (shmem_mapping(mapping)) {
		page = find_get_entry(mapping, pgoff);
		if (radix_tree_exceptional_entry(page)) {
			swp_entry_t swp = radix_to_swp_entry(page);
4834
			if (do_memsw_account())
4835
				*entry = swp;
4836 4837
			page = find_get_page(swap_address_space(swp),
					     swp_offset(swp));
4838 4839 4840 4841 4842
		}
	} else
		page = find_get_page(mapping, pgoff);
#else
	page = find_get_page(mapping, pgoff);
4843
#endif
4844 4845 4846
	return page;
}

4847 4848 4849
/**
 * mem_cgroup_move_account - move account of the page
 * @page: the page
4850
 * @compound: charge the page as compound or small page
4851 4852 4853
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
4854
 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4855 4856 4857 4858 4859
 *
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
 */
static int mem_cgroup_move_account(struct page *page,
4860
				   bool compound,
4861 4862 4863 4864
				   struct mem_cgroup *from,
				   struct mem_cgroup *to)
{
	unsigned long flags;
4865
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4866
	int ret;
4867
	bool anon;
4868 4869 4870

	VM_BUG_ON(from == to);
	VM_BUG_ON_PAGE(PageLRU(page), page);
4871
	VM_BUG_ON(compound && !PageTransHuge(page));
4872 4873

	/*
4874
	 * Prevent mem_cgroup_migrate() from looking at
4875
	 * page->mem_cgroup of its source page while we change it.
4876
	 */
4877
	ret = -EBUSY;
4878 4879 4880 4881 4882 4883 4884
	if (!trylock_page(page))
		goto out;

	ret = -EINVAL;
	if (page->mem_cgroup != from)
		goto out_unlock;

4885 4886
	anon = PageAnon(page);

4887 4888
	spin_lock_irqsave(&from->move_lock, flags);

4889
	if (!anon && page_mapped(page)) {
4890 4891
		__mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
		__mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
4892 4893
	}

4894 4895
	/*
	 * move_lock grabbed above and caller set from->moving_account, so
4896
	 * mod_memcg_page_state will serialize updates to PageDirty.
4897 4898 4899 4900 4901 4902
	 * So mapping should be stable for dirty pages.
	 */
	if (!anon && PageDirty(page)) {
		struct address_space *mapping = page_mapping(page);

		if (mapping_cap_account_dirty(mapping)) {
4903 4904
			__mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
			__mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
4905 4906 4907
		}
	}

4908
	if (PageWriteback(page)) {
4909 4910
		__mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
		__mod_memcg_state(to, NR_WRITEBACK, nr_pages);
4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925
	}

	/*
	 * It is safe to change page->mem_cgroup here because the page
	 * is referenced, charged, and isolated - we can't race with
	 * uncharging, charging, migration, or LRU putback.
	 */

	/* caller should have done css_get */
	page->mem_cgroup = to;
	spin_unlock_irqrestore(&from->move_lock, flags);

	ret = 0;

	local_irq_disable();
4926
	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4927
	memcg_check_events(to, page);
4928
	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4929 4930 4931 4932 4933 4934 4935 4936
	memcg_check_events(from, page);
	local_irq_enable();
out_unlock:
	unlock_page(page);
out:
	return ret;
}

4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951
/**
 * get_mctgt_type - get target type of moving charge
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
4952 4953 4954 4955 4956
 *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PUBLIC
 *     or MEMORY_DEVICE_PRIVATE (so ZONE_DEVICE page and thus not on the lru).
 *     For now we such page is charge like a regular page would be as for all
 *     intent and purposes it is just special memory taking the place of a
 *     regular page.
4957 4958
 *
 *     See Documentations/vm/hmm.txt and include/linux/hmm.h
4959 4960 4961 4962
 *
 * Called with pte lock held.
 */

4963
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
4964 4965 4966
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
4967
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
4968 4969 4970 4971 4972
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
4973
		page = mc_handle_swap_pte(vma, ptent, &ent);
4974
	else if (pte_none(ptent))
4975
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
4976 4977

	if (!page && !ent.val)
4978
		return ret;
4979 4980
	if (page) {
		/*
4981
		 * Do only loose check w/o serialization.
4982
		 * mem_cgroup_move_account() checks the page is valid or
4983
		 * not under LRU exclusion.
4984
		 */
4985
		if (page->mem_cgroup == mc.from) {
4986
			ret = MC_TARGET_PAGE;
4987 4988
			if (is_device_private_page(page) ||
			    is_device_public_page(page))
4989
				ret = MC_TARGET_DEVICE;
4990 4991 4992 4993 4994 4995
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
4996 4997 4998 4999 5000
	/*
	 * There is a swap entry and a page doesn't exist or isn't charged.
	 * But we cannot move a tail-page in a THP.
	 */
	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
L
Li Zefan 已提交
5001
	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
5002 5003 5004
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
5005 5006 5007 5008
	}
	return ret;
}

5009 5010
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
5011 5012
 * We don't consider PMD mapped swapping or file mapped pages because THP does
 * not support them for now.
5013 5014 5015 5016 5017 5018 5019 5020
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	enum mc_target_type ret = MC_TARGET_NONE;

5021 5022 5023 5024 5025
	if (unlikely(is_swap_pmd(pmd))) {
		VM_BUG_ON(thp_migration_supported() &&
				  !is_pmd_migration_entry(pmd));
		return ret;
	}
5026
	page = pmd_page(pmd);
5027
	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
5028
	if (!(mc.flags & MOVE_ANON))
5029
		return ret;
5030
	if (page->mem_cgroup == mc.from) {
5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

5047 5048 5049 5050
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
5051
	struct vm_area_struct *vma = walk->vma;
5052 5053 5054
	pte_t *pte;
	spinlock_t *ptl;

5055 5056
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
5057 5058 5059 5060 5061
		/*
		 * Note their can not be MC_TARGET_DEVICE for now as we do not
		 * support transparent huge page with MEMORY_DEVICE_PUBLIC or
		 * MEMORY_DEVICE_PRIVATE but this might change.
		 */
5062 5063
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
5064
		spin_unlock(ptl);
5065
		return 0;
5066
	}
5067

5068 5069
	if (pmd_trans_unstable(pmd))
		return 0;
5070 5071
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
5072
		if (get_mctgt_type(vma, addr, *pte, NULL))
5073 5074 5075 5076
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

5077 5078 5079
	return 0;
}

5080 5081 5082 5083
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;

5084 5085 5086 5087
	struct mm_walk mem_cgroup_count_precharge_walk = {
		.pmd_entry = mem_cgroup_count_precharge_pte_range,
		.mm = mm,
	};
5088
	down_read(&mm->mmap_sem);
5089 5090
	walk_page_range(0, mm->highest_vm_end,
			&mem_cgroup_count_precharge_walk);
5091
	up_read(&mm->mmap_sem);
5092 5093 5094 5095 5096 5097 5098 5099 5100

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
5101 5102 5103 5104 5105
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
5106 5107
}

5108 5109
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
5110
{
5111 5112 5113
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

5114
	/* we must uncharge all the leftover precharges from mc.to */
5115
	if (mc.precharge) {
5116
		cancel_charge(mc.to, mc.precharge);
5117 5118 5119 5120 5121 5122 5123
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
5124
		cancel_charge(mc.from, mc.moved_charge);
5125
		mc.moved_charge = 0;
5126
	}
5127 5128 5129
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
5130
		if (!mem_cgroup_is_root(mc.from))
5131
			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5132

5133 5134
		mem_cgroup_id_put_many(mc.from, mc.moved_swap);

5135
		/*
5136 5137
		 * we charged both to->memory and to->memsw, so we
		 * should uncharge to->memory.
5138
		 */
5139
		if (!mem_cgroup_is_root(mc.to))
5140 5141
			page_counter_uncharge(&mc.to->memory, mc.moved_swap);

5142 5143
		mem_cgroup_id_get_many(mc.to, mc.moved_swap);
		css_put_many(&mc.to->css, mc.moved_swap);
5144

5145 5146
		mc.moved_swap = 0;
	}
5147 5148 5149 5150 5151 5152 5153
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
5154 5155
	struct mm_struct *mm = mc.mm;

5156 5157 5158 5159 5160 5161
	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
5162
	spin_lock(&mc.lock);
5163 5164
	mc.from = NULL;
	mc.to = NULL;
5165
	mc.mm = NULL;
5166
	spin_unlock(&mc.lock);
5167 5168

	mmput(mm);
5169 5170
}

5171
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5172
{
5173
	struct cgroup_subsys_state *css;
5174
	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
5175
	struct mem_cgroup *from;
5176
	struct task_struct *leader, *p;
5177
	struct mm_struct *mm;
5178
	unsigned long move_flags;
5179
	int ret = 0;
5180

5181 5182
	/* charge immigration isn't supported on the default hierarchy */
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5183 5184
		return 0;

5185 5186 5187 5188 5189 5190 5191
	/*
	 * Multi-process migrations only happen on the default hierarchy
	 * where charge immigration is not used.  Perform charge
	 * immigration if @tset contains a leader and whine if there are
	 * multiple.
	 */
	p = NULL;
5192
	cgroup_taskset_for_each_leader(leader, css, tset) {
5193 5194
		WARN_ON_ONCE(p);
		p = leader;
5195
		memcg = mem_cgroup_from_css(css);
5196 5197 5198 5199
	}
	if (!p)
		return 0;

5200 5201 5202 5203 5204 5205 5206 5207 5208
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
	if (!move_flags)
		return 0;

5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224
	from = mem_cgroup_from_task(p);

	VM_BUG_ON(from == memcg);

	mm = get_task_mm(p);
	if (!mm)
		return 0;
	/* We move charges only when we move a owner of the mm */
	if (mm->owner == p) {
		VM_BUG_ON(mc.from);
		VM_BUG_ON(mc.to);
		VM_BUG_ON(mc.precharge);
		VM_BUG_ON(mc.moved_charge);
		VM_BUG_ON(mc.moved_swap);

		spin_lock(&mc.lock);
5225
		mc.mm = mm;
5226 5227 5228 5229 5230 5231 5232 5233 5234
		mc.from = from;
		mc.to = memcg;
		mc.flags = move_flags;
		spin_unlock(&mc.lock);
		/* We set mc.moving_task later */

		ret = mem_cgroup_precharge_mc(mm);
		if (ret)
			mem_cgroup_clear_mc();
5235 5236
	} else {
		mmput(mm);
5237 5238 5239 5240
	}
	return ret;
}

5241
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5242
{
5243 5244
	if (mc.to)
		mem_cgroup_clear_mc();
5245 5246
}

5247 5248 5249
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
5250
{
5251
	int ret = 0;
5252
	struct vm_area_struct *vma = walk->vma;
5253 5254
	pte_t *pte;
	spinlock_t *ptl;
5255 5256 5257
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
5258

5259 5260
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
5261
		if (mc.precharge < HPAGE_PMD_NR) {
5262
			spin_unlock(ptl);
5263 5264 5265 5266 5267 5268
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
5269
				if (!mem_cgroup_move_account(page, true,
5270
							     mc.from, mc.to)) {
5271 5272 5273 5274 5275 5276
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
5277 5278 5279 5280 5281 5282 5283 5284
		} else if (target_type == MC_TARGET_DEVICE) {
			page = target.page;
			if (!mem_cgroup_move_account(page, true,
						     mc.from, mc.to)) {
				mc.precharge -= HPAGE_PMD_NR;
				mc.moved_charge += HPAGE_PMD_NR;
			}
			put_page(page);
5285
		}
5286
		spin_unlock(ptl);
5287
		return 0;
5288 5289
	}

5290 5291
	if (pmd_trans_unstable(pmd))
		return 0;
5292 5293 5294 5295
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
5296
		bool device = false;
5297
		swp_entry_t ent;
5298 5299 5300 5301

		if (!mc.precharge)
			break;

5302
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
5303 5304 5305
		case MC_TARGET_DEVICE:
			device = true;
			/* fall through */
5306 5307
		case MC_TARGET_PAGE:
			page = target.page;
5308 5309 5310 5311 5312 5313 5314 5315
			/*
			 * We can have a part of the split pmd here. Moving it
			 * can be done but it would be too convoluted so simply
			 * ignore such a partial THP and keep it in original
			 * memcg. There should be somebody mapping the head.
			 */
			if (PageTransCompound(page))
				goto put;
5316
			if (!device && isolate_lru_page(page))
5317
				goto put;
5318 5319
			if (!mem_cgroup_move_account(page, false,
						mc.from, mc.to)) {
5320
				mc.precharge--;
5321 5322
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
5323
			}
5324 5325
			if (!device)
				putback_lru_page(page);
5326
put:			/* get_mctgt_type() gets the page */
5327 5328
			put_page(page);
			break;
5329 5330
		case MC_TARGET_SWAP:
			ent = target.ent;
5331
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5332
				mc.precharge--;
5333 5334 5335
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
5336
			break;
5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
5351
		ret = mem_cgroup_do_precharge(1);
5352 5353 5354 5355 5356 5357 5358
		if (!ret)
			goto retry;
	}

	return ret;
}

5359
static void mem_cgroup_move_charge(void)
5360
{
5361 5362
	struct mm_walk mem_cgroup_move_charge_walk = {
		.pmd_entry = mem_cgroup_move_charge_pte_range,
5363
		.mm = mc.mm,
5364
	};
5365 5366

	lru_add_drain_all();
5367
	/*
5368 5369 5370
	 * Signal lock_page_memcg() to take the memcg's move_lock
	 * while we're moving its pages to another memcg. Then wait
	 * for already started RCU-only updates to finish.
5371 5372 5373
	 */
	atomic_inc(&mc.from->moving_account);
	synchronize_rcu();
5374
retry:
5375
	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5387 5388 5389 5390
	/*
	 * When we have consumed all precharges and failed in doing
	 * additional charge, the page walk just aborts.
	 */
5391 5392
	walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);

5393
	up_read(&mc.mm->mmap_sem);
5394
	atomic_dec(&mc.from->moving_account);
5395 5396
}

5397
static void mem_cgroup_move_task(void)
B
Balbir Singh 已提交
5398
{
5399 5400
	if (mc.to) {
		mem_cgroup_move_charge();
5401
		mem_cgroup_clear_mc();
5402
	}
B
Balbir Singh 已提交
5403
}
5404
#else	/* !CONFIG_MMU */
5405
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5406 5407 5408
{
	return 0;
}
5409
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5410 5411
{
}
5412
static void mem_cgroup_move_task(void)
5413 5414 5415
{
}
#endif
B
Balbir Singh 已提交
5416

5417 5418
/*
 * Cgroup retains root cgroups across [un]mount cycles making it necessary
5419 5420
 * to verify whether we're attached to the default hierarchy on each mount
 * attempt.
5421
 */
5422
static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5423 5424
{
	/*
5425
	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5426 5427 5428
	 * guarantees that @root doesn't have any children, so turning it
	 * on for the root memcg is enough.
	 */
5429
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5430 5431 5432
		root_mem_cgroup->use_hierarchy = true;
	else
		root_mem_cgroup->use_hierarchy = false;
5433 5434
}

5435 5436 5437
static u64 memory_current_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
{
5438 5439 5440
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5441 5442
}

R
Roman Gushchin 已提交
5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472
static int memory_min_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
	unsigned long min = READ_ONCE(memcg->memory.min);

	if (min == PAGE_COUNTER_MAX)
		seq_puts(m, "max\n");
	else
		seq_printf(m, "%llu\n", (u64)min * PAGE_SIZE);

	return 0;
}

static ssize_t memory_min_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long min;
	int err;

	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &min);
	if (err)
		return err;

	page_counter_set_min(&memcg->memory, min);

	return nbytes;
}

5473 5474 5475
static int memory_low_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5476
	unsigned long low = READ_ONCE(memcg->memory.low);
5477 5478

	if (low == PAGE_COUNTER_MAX)
5479
		seq_puts(m, "max\n");
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493
	else
		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);

	return 0;
}

static ssize_t memory_low_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long low;
	int err;

	buf = strstrip(buf);
5494
	err = page_counter_memparse(buf, "max", &low);
5495 5496 5497
	if (err)
		return err;

5498
	page_counter_set_low(&memcg->memory, low);
5499 5500 5501 5502 5503 5504 5505

	return nbytes;
}

static int memory_high_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5506
	unsigned long high = READ_ONCE(memcg->high);
5507 5508

	if (high == PAGE_COUNTER_MAX)
5509
		seq_puts(m, "max\n");
5510 5511 5512 5513 5514 5515 5516 5517 5518 5519
	else
		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);

	return 0;
}

static ssize_t memory_high_write(struct kernfs_open_file *of,
				 char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5520
	unsigned long nr_pages;
5521 5522 5523 5524
	unsigned long high;
	int err;

	buf = strstrip(buf);
5525
	err = page_counter_memparse(buf, "max", &high);
5526 5527 5528 5529 5530
	if (err)
		return err;

	memcg->high = high;

5531 5532 5533 5534 5535
	nr_pages = page_counter_read(&memcg->memory);
	if (nr_pages > high)
		try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
					     GFP_KERNEL, true);

5536
	memcg_wb_domain_size_changed(memcg);
5537 5538 5539 5540 5541 5542
	return nbytes;
}

static int memory_max_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5543
	unsigned long max = READ_ONCE(memcg->memory.max);
5544 5545

	if (max == PAGE_COUNTER_MAX)
5546
		seq_puts(m, "max\n");
5547 5548 5549 5550 5551 5552 5553 5554 5555 5556
	else
		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);

	return 0;
}

static ssize_t memory_max_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5557 5558
	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
	bool drained = false;
5559 5560 5561 5562
	unsigned long max;
	int err;

	buf = strstrip(buf);
5563
	err = page_counter_memparse(buf, "max", &max);
5564 5565 5566
	if (err)
		return err;

5567
	xchg(&memcg->memory.max, max);
5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592

	for (;;) {
		unsigned long nr_pages = page_counter_read(&memcg->memory);

		if (nr_pages <= max)
			break;

		if (signal_pending(current)) {
			err = -EINTR;
			break;
		}

		if (!drained) {
			drain_all_stock(memcg);
			drained = true;
			continue;
		}

		if (nr_reclaims) {
			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
							  GFP_KERNEL, true))
				nr_reclaims--;
			continue;
		}

5593
		memcg_memory_event(memcg, MEMCG_OOM);
5594 5595 5596
		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
			break;
	}
5597

5598
	memcg_wb_domain_size_changed(memcg);
5599 5600 5601 5602 5603 5604 5605
	return nbytes;
}

static int memory_events_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));

5606 5607 5608 5609 5610 5611 5612 5613
	seq_printf(m, "low %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_LOW]));
	seq_printf(m, "high %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_HIGH]));
	seq_printf(m, "max %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_MAX]));
	seq_printf(m, "oom %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_OOM]));
R
Roman Gushchin 已提交
5614 5615
	seq_printf(m, "oom_kill %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
5616 5617 5618 5619

	return 0;
}

5620 5621 5622
static int memory_stat_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5623
	struct accumulated_stats acc;
5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636
	int i;

	/*
	 * Provide statistics on the state of the memory subsystem as
	 * well as cumulative event counters that show past behavior.
	 *
	 * This list is ordered following a combination of these gradients:
	 * 1) generic big picture -> specifics and details
	 * 2) reflecting userspace activity -> reflecting kernel heuristics
	 *
	 * Current memory state:
	 */

5637 5638 5639 5640
	memset(&acc, 0, sizeof(acc));
	acc.stats_size = MEMCG_NR_STAT;
	acc.events_size = NR_VM_EVENT_ITEMS;
	accumulate_memcg_tree(memcg, &acc);
5641

5642
	seq_printf(m, "anon %llu\n",
5643
		   (u64)acc.stat[MEMCG_RSS] * PAGE_SIZE);
5644
	seq_printf(m, "file %llu\n",
5645
		   (u64)acc.stat[MEMCG_CACHE] * PAGE_SIZE);
5646
	seq_printf(m, "kernel_stack %llu\n",
5647
		   (u64)acc.stat[MEMCG_KERNEL_STACK_KB] * 1024);
5648
	seq_printf(m, "slab %llu\n",
5649 5650
		   (u64)(acc.stat[NR_SLAB_RECLAIMABLE] +
			 acc.stat[NR_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5651
	seq_printf(m, "sock %llu\n",
5652
		   (u64)acc.stat[MEMCG_SOCK] * PAGE_SIZE);
5653

5654
	seq_printf(m, "shmem %llu\n",
5655
		   (u64)acc.stat[NR_SHMEM] * PAGE_SIZE);
5656
	seq_printf(m, "file_mapped %llu\n",
5657
		   (u64)acc.stat[NR_FILE_MAPPED] * PAGE_SIZE);
5658
	seq_printf(m, "file_dirty %llu\n",
5659
		   (u64)acc.stat[NR_FILE_DIRTY] * PAGE_SIZE);
5660
	seq_printf(m, "file_writeback %llu\n",
5661
		   (u64)acc.stat[NR_WRITEBACK] * PAGE_SIZE);
5662

5663 5664 5665
	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %llu\n", mem_cgroup_lru_names[i],
			   (u64)acc.lru_pages[i] * PAGE_SIZE);
5666

5667
	seq_printf(m, "slab_reclaimable %llu\n",
5668
		   (u64)acc.stat[NR_SLAB_RECLAIMABLE] * PAGE_SIZE);
5669
	seq_printf(m, "slab_unreclaimable %llu\n",
5670
		   (u64)acc.stat[NR_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5671

5672 5673
	/* Accumulated memory events */

5674 5675
	seq_printf(m, "pgfault %lu\n", acc.events[PGFAULT]);
	seq_printf(m, "pgmajfault %lu\n", acc.events[PGMAJFAULT]);
5676

5677 5678 5679 5680 5681 5682 5683 5684 5685
	seq_printf(m, "pgrefill %lu\n", acc.events[PGREFILL]);
	seq_printf(m, "pgscan %lu\n", acc.events[PGSCAN_KSWAPD] +
		   acc.events[PGSCAN_DIRECT]);
	seq_printf(m, "pgsteal %lu\n", acc.events[PGSTEAL_KSWAPD] +
		   acc.events[PGSTEAL_DIRECT]);
	seq_printf(m, "pgactivate %lu\n", acc.events[PGACTIVATE]);
	seq_printf(m, "pgdeactivate %lu\n", acc.events[PGDEACTIVATE]);
	seq_printf(m, "pglazyfree %lu\n", acc.events[PGLAZYFREE]);
	seq_printf(m, "pglazyfreed %lu\n", acc.events[PGLAZYFREED]);
5686

5687
	seq_printf(m, "workingset_refault %lu\n",
5688
		   acc.stat[WORKINGSET_REFAULT]);
5689
	seq_printf(m, "workingset_activate %lu\n",
5690
		   acc.stat[WORKINGSET_ACTIVATE]);
5691
	seq_printf(m, "workingset_nodereclaim %lu\n",
5692
		   acc.stat[WORKINGSET_NODERECLAIM]);
5693

5694 5695 5696
	return 0;
}

5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727
static int memory_oom_group_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));

	seq_printf(m, "%d\n", memcg->oom_group);

	return 0;
}

static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
				      char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	int ret, oom_group;

	buf = strstrip(buf);
	if (!buf)
		return -EINVAL;

	ret = kstrtoint(buf, 0, &oom_group);
	if (ret)
		return ret;

	if (oom_group != 0 && oom_group != 1)
		return -EINVAL;

	memcg->oom_group = oom_group;

	return nbytes;
}

5728 5729 5730
static struct cftype memory_files[] = {
	{
		.name = "current",
5731
		.flags = CFTYPE_NOT_ON_ROOT,
5732 5733
		.read_u64 = memory_current_read,
	},
R
Roman Gushchin 已提交
5734 5735 5736 5737 5738 5739
	{
		.name = "min",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_min_show,
		.write = memory_min_write,
	},
5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760
	{
		.name = "low",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_low_show,
		.write = memory_low_write,
	},
	{
		.name = "high",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_high_show,
		.write = memory_high_write,
	},
	{
		.name = "max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_max_show,
		.write = memory_max_write,
	},
	{
		.name = "events",
		.flags = CFTYPE_NOT_ON_ROOT,
5761
		.file_offset = offsetof(struct mem_cgroup, events_file),
5762 5763
		.seq_show = memory_events_show,
	},
5764 5765 5766 5767 5768
	{
		.name = "stat",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_stat_show,
	},
5769 5770 5771 5772 5773 5774
	{
		.name = "oom.group",
		.flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
		.seq_show = memory_oom_group_show,
		.write = memory_oom_group_write,
	},
5775 5776 5777
	{ }	/* terminate */
};

5778
struct cgroup_subsys memory_cgrp_subsys = {
5779
	.css_alloc = mem_cgroup_css_alloc,
5780
	.css_online = mem_cgroup_css_online,
5781
	.css_offline = mem_cgroup_css_offline,
5782
	.css_released = mem_cgroup_css_released,
5783
	.css_free = mem_cgroup_css_free,
5784
	.css_reset = mem_cgroup_css_reset,
5785 5786
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
5787
	.post_attach = mem_cgroup_move_task,
5788
	.bind = mem_cgroup_bind,
5789 5790
	.dfl_cftypes = memory_files,
	.legacy_cftypes = mem_cgroup_legacy_files,
5791
	.early_init = 0,
B
Balbir Singh 已提交
5792
};
5793

5794
/**
R
Roman Gushchin 已提交
5795
 * mem_cgroup_protected - check if memory consumption is in the normal range
5796
 * @root: the top ancestor of the sub-tree being checked
5797 5798
 * @memcg: the memory cgroup to check
 *
5799 5800
 * WARNING: This function is not stateless! It can only be used as part
 *          of a top-down tree iteration, not for isolated queries.
5801
 *
R
Roman Gushchin 已提交
5802 5803 5804 5805 5806
 * Returns one of the following:
 *   MEMCG_PROT_NONE: cgroup memory is not protected
 *   MEMCG_PROT_LOW: cgroup memory is protected as long there is
 *     an unprotected supply of reclaimable memory from other cgroups.
 *   MEMCG_PROT_MIN: cgroup memory is protected
5807
 *
R
Roman Gushchin 已提交
5808
 * @root is exclusive; it is never protected when looked at directly
5809
 *
R
Roman Gushchin 已提交
5810 5811 5812
 * To provide a proper hierarchical behavior, effective memory.min/low values
 * are used. Below is the description of how effective memory.low is calculated.
 * Effective memory.min values is calculated in the same way.
5813
 *
5814 5815 5816 5817 5818 5819 5820
 * Effective memory.low is always equal or less than the original memory.low.
 * If there is no memory.low overcommittment (which is always true for
 * top-level memory cgroups), these two values are equal.
 * Otherwise, it's a part of parent's effective memory.low,
 * calculated as a cgroup's memory.low usage divided by sum of sibling's
 * memory.low usages, where memory.low usage is the size of actually
 * protected memory.
5821
 *
5822 5823 5824
 *                                             low_usage
 * elow = min( memory.low, parent->elow * ------------------ ),
 *                                        siblings_low_usage
5825
 *
5826 5827 5828
 *             | memory.current, if memory.current < memory.low
 * low_usage = |
	       | 0, otherwise.
5829
 *
5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856
 *
 * Such definition of the effective memory.low provides the expected
 * hierarchical behavior: parent's memory.low value is limiting
 * children, unprotected memory is reclaimed first and cgroups,
 * which are not using their guarantee do not affect actual memory
 * distribution.
 *
 * For example, if there are memcgs A, A/B, A/C, A/D and A/E:
 *
 *     A      A/memory.low = 2G, A/memory.current = 6G
 *    //\\
 *   BC  DE   B/memory.low = 3G  B/memory.current = 2G
 *            C/memory.low = 1G  C/memory.current = 2G
 *            D/memory.low = 0   D/memory.current = 2G
 *            E/memory.low = 10G E/memory.current = 0
 *
 * and the memory pressure is applied, the following memory distribution
 * is expected (approximately):
 *
 *     A/memory.current = 2G
 *
 *     B/memory.current = 1.3G
 *     C/memory.current = 0.6G
 *     D/memory.current = 0
 *     E/memory.current = 0
 *
 * These calculations require constant tracking of the actual low usages
R
Roman Gushchin 已提交
5857 5858
 * (see propagate_protected_usage()), as well as recursive calculation of
 * effective memory.low values. But as we do call mem_cgroup_protected()
5859 5860 5861 5862
 * path for each memory cgroup top-down from the reclaim,
 * it's possible to optimize this part, and save calculated elow
 * for next usage. This part is intentionally racy, but it's ok,
 * as memory.low is a best-effort mechanism.
5863
 */
R
Roman Gushchin 已提交
5864 5865
enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
						struct mem_cgroup *memcg)
5866
{
5867
	struct mem_cgroup *parent;
R
Roman Gushchin 已提交
5868 5869 5870
	unsigned long emin, parent_emin;
	unsigned long elow, parent_elow;
	unsigned long usage;
5871

5872
	if (mem_cgroup_disabled())
R
Roman Gushchin 已提交
5873
		return MEMCG_PROT_NONE;
5874

5875 5876 5877
	if (!root)
		root = root_mem_cgroup;
	if (memcg == root)
R
Roman Gushchin 已提交
5878
		return MEMCG_PROT_NONE;
5879

5880
	usage = page_counter_read(&memcg->memory);
R
Roman Gushchin 已提交
5881 5882 5883 5884 5885
	if (!usage)
		return MEMCG_PROT_NONE;

	emin = memcg->memory.min;
	elow = memcg->memory.low;
5886

R
Roman Gushchin 已提交
5887
	parent = parent_mem_cgroup(memcg);
5888 5889 5890 5891
	/* No parent means a non-hierarchical mode on v1 memcg */
	if (!parent)
		return MEMCG_PROT_NONE;

5892 5893 5894
	if (parent == root)
		goto exit;

R
Roman Gushchin 已提交
5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908
	parent_emin = READ_ONCE(parent->memory.emin);
	emin = min(emin, parent_emin);
	if (emin && parent_emin) {
		unsigned long min_usage, siblings_min_usage;

		min_usage = min(usage, memcg->memory.min);
		siblings_min_usage = atomic_long_read(
			&parent->memory.children_min_usage);

		if (min_usage && siblings_min_usage)
			emin = min(emin, parent_emin * min_usage /
				   siblings_min_usage);
	}

5909 5910
	parent_elow = READ_ONCE(parent->memory.elow);
	elow = min(elow, parent_elow);
R
Roman Gushchin 已提交
5911 5912
	if (elow && parent_elow) {
		unsigned long low_usage, siblings_low_usage;
5913

R
Roman Gushchin 已提交
5914 5915 5916
		low_usage = min(usage, memcg->memory.low);
		siblings_low_usage = atomic_long_read(
			&parent->memory.children_low_usage);
5917

R
Roman Gushchin 已提交
5918 5919 5920 5921
		if (low_usage && siblings_low_usage)
			elow = min(elow, parent_elow * low_usage /
				   siblings_low_usage);
	}
5922 5923

exit:
R
Roman Gushchin 已提交
5924
	memcg->memory.emin = emin;
5925
	memcg->memory.elow = elow;
R
Roman Gushchin 已提交
5926 5927 5928 5929 5930 5931 5932

	if (usage <= emin)
		return MEMCG_PROT_MIN;
	else if (usage <= elow)
		return MEMCG_PROT_LOW;
	else
		return MEMCG_PROT_NONE;
5933 5934
}

5935 5936 5937 5938 5939 5940
/**
 * mem_cgroup_try_charge - try charging a page
 * @page: page to charge
 * @mm: mm context of the victim
 * @gfp_mask: reclaim mode
 * @memcgp: charged memcg return
5941
 * @compound: charge the page as compound or small page
5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953
 *
 * Try to charge @page to the memcg that @mm belongs to, reclaiming
 * pages according to @gfp_mask if necessary.
 *
 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
 * Otherwise, an error code is returned.
 *
 * After page->mapping has been set up, the caller must finalize the
 * charge with mem_cgroup_commit_charge().  Or abort the transaction
 * with mem_cgroup_cancel_charge() in case page instantiation fails.
 */
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5954 5955
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound)
5956 5957
{
	struct mem_cgroup *memcg = NULL;
5958
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971
	int ret = 0;

	if (mem_cgroup_disabled())
		goto out;

	if (PageSwapCache(page)) {
		/*
		 * Every swap fault against a single page tries to charge the
		 * page, bail as early as possible.  shmem_unuse() encounters
		 * already charged pages, too.  The USED bit is protected by
		 * the page lock, which serializes swap cache removal, which
		 * in turn serializes uncharging.
		 */
5972
		VM_BUG_ON_PAGE(!PageLocked(page), page);
5973
		if (compound_head(page)->mem_cgroup)
5974
			goto out;
5975

5976
		if (do_swap_account) {
5977 5978 5979 5980 5981 5982 5983 5984 5985
			swp_entry_t ent = { .val = page_private(page), };
			unsigned short id = lookup_swap_cgroup_id(ent);

			rcu_read_lock();
			memcg = mem_cgroup_from_id(id);
			if (memcg && !css_tryget_online(&memcg->css))
				memcg = NULL;
			rcu_read_unlock();
		}
5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998
	}

	if (!memcg)
		memcg = get_mem_cgroup_from_mm(mm);

	ret = try_charge(memcg, gfp_mask, nr_pages);

	css_put(&memcg->css);
out:
	*memcgp = memcg;
	return ret;
}

5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011
int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound)
{
	struct mem_cgroup *memcg;
	int ret;

	ret = mem_cgroup_try_charge(page, mm, gfp_mask, memcgp, compound);
	memcg = *memcgp;
	mem_cgroup_throttle_swaprate(memcg, page_to_nid(page), gfp_mask);
	return ret;
}

6012 6013 6014 6015 6016
/**
 * mem_cgroup_commit_charge - commit a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 * @lrucare: page might be on LRU already
6017
 * @compound: charge the page as compound or small page
6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029
 *
 * Finalize a charge transaction started by mem_cgroup_try_charge(),
 * after page->mapping has been set up.  This must happen atomically
 * as part of the page instantiation, i.e. under the page table lock
 * for anonymous pages, under the page lock for page and swap cache.
 *
 * In addition, the page must not be on the LRU during the commit, to
 * prevent racing with task migration.  If it might be, use @lrucare.
 *
 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
 */
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
6030
			      bool lrucare, bool compound)
6031
{
6032
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046

	VM_BUG_ON_PAGE(!page->mapping, page);
	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

6047 6048 6049
	commit_charge(page, memcg, lrucare);

	local_irq_disable();
6050
	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
6051 6052
	memcg_check_events(memcg, page);
	local_irq_enable();
6053

6054
	if (do_memsw_account() && PageSwapCache(page)) {
6055 6056 6057 6058 6059 6060
		swp_entry_t entry = { .val = page_private(page) };
		/*
		 * The swap entry might not get freed for a long time,
		 * let's not wait for it.  The page already received a
		 * memory+swap charge, drop the swap entry duplicate.
		 */
6061
		mem_cgroup_uncharge_swap(entry, nr_pages);
6062 6063 6064 6065 6066 6067 6068
	}
}

/**
 * mem_cgroup_cancel_charge - cancel a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
6069
 * @compound: charge the page as compound or small page
6070 6071 6072
 *
 * Cancel a charge transaction started by mem_cgroup_try_charge().
 */
6073 6074
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound)
6075
{
6076
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

	cancel_charge(memcg, nr_pages);
}

6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102
struct uncharge_gather {
	struct mem_cgroup *memcg;
	unsigned long pgpgout;
	unsigned long nr_anon;
	unsigned long nr_file;
	unsigned long nr_kmem;
	unsigned long nr_huge;
	unsigned long nr_shmem;
	struct page *dummy_page;
};

static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6103
{
6104 6105 6106 6107 6108 6109
	memset(ug, 0, sizeof(*ug));
}

static void uncharge_batch(const struct uncharge_gather *ug)
{
	unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
6110 6111
	unsigned long flags;

6112 6113
	if (!mem_cgroup_is_root(ug->memcg)) {
		page_counter_uncharge(&ug->memcg->memory, nr_pages);
6114
		if (do_memsw_account())
6115 6116 6117 6118
			page_counter_uncharge(&ug->memcg->memsw, nr_pages);
		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
		memcg_oom_recover(ug->memcg);
6119
	}
6120 6121

	local_irq_save(flags);
6122 6123 6124 6125 6126
	__mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
	__mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
	__mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
	__mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6127
	__this_cpu_add(ug->memcg->stat_cpu->nr_page_events, nr_pages);
6128
	memcg_check_events(ug->memcg, ug->dummy_page);
6129
	local_irq_restore(flags);
6130

6131 6132 6133 6134 6135 6136 6137
	if (!mem_cgroup_is_root(ug->memcg))
		css_put_many(&ug->memcg->css, nr_pages);
}

static void uncharge_page(struct page *page, struct uncharge_gather *ug)
{
	VM_BUG_ON_PAGE(PageLRU(page), page);
6138 6139
	VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
			!PageHWPoison(page) , page);
6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179

	if (!page->mem_cgroup)
		return;

	/*
	 * Nobody should be changing or seriously looking at
	 * page->mem_cgroup at this point, we have fully
	 * exclusive access to the page.
	 */

	if (ug->memcg != page->mem_cgroup) {
		if (ug->memcg) {
			uncharge_batch(ug);
			uncharge_gather_clear(ug);
		}
		ug->memcg = page->mem_cgroup;
	}

	if (!PageKmemcg(page)) {
		unsigned int nr_pages = 1;

		if (PageTransHuge(page)) {
			nr_pages <<= compound_order(page);
			ug->nr_huge += nr_pages;
		}
		if (PageAnon(page))
			ug->nr_anon += nr_pages;
		else {
			ug->nr_file += nr_pages;
			if (PageSwapBacked(page))
				ug->nr_shmem += nr_pages;
		}
		ug->pgpgout++;
	} else {
		ug->nr_kmem += 1 << compound_order(page);
		__ClearPageKmemcg(page);
	}

	ug->dummy_page = page;
	page->mem_cgroup = NULL;
6180 6181 6182 6183
}

static void uncharge_list(struct list_head *page_list)
{
6184
	struct uncharge_gather ug;
6185
	struct list_head *next;
6186 6187

	uncharge_gather_clear(&ug);
6188

6189 6190 6191 6192
	/*
	 * Note that the list can be a single page->lru; hence the
	 * do-while loop instead of a simple list_for_each_entry().
	 */
6193 6194
	next = page_list->next;
	do {
6195 6196
		struct page *page;

6197 6198 6199
		page = list_entry(next, struct page, lru);
		next = page->lru.next;

6200
		uncharge_page(page, &ug);
6201 6202
	} while (next != page_list);

6203 6204
	if (ug.memcg)
		uncharge_batch(&ug);
6205 6206
}

6207 6208 6209 6210 6211 6212 6213 6214 6215
/**
 * mem_cgroup_uncharge - uncharge a page
 * @page: page to uncharge
 *
 * Uncharge a page previously charged with mem_cgroup_try_charge() and
 * mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge(struct page *page)
{
6216 6217
	struct uncharge_gather ug;

6218 6219 6220
	if (mem_cgroup_disabled())
		return;

6221
	/* Don't touch page->lru of any random page, pre-check: */
6222
	if (!page->mem_cgroup)
6223 6224
		return;

6225 6226 6227
	uncharge_gather_clear(&ug);
	uncharge_page(page, &ug);
	uncharge_batch(&ug);
6228
}
6229

6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240
/**
 * mem_cgroup_uncharge_list - uncharge a list of page
 * @page_list: list of pages to uncharge
 *
 * Uncharge a list of pages previously charged with
 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge_list(struct list_head *page_list)
{
	if (mem_cgroup_disabled())
		return;
6241

6242 6243
	if (!list_empty(page_list))
		uncharge_list(page_list);
6244 6245 6246
}

/**
6247 6248 6249
 * mem_cgroup_migrate - charge a page's replacement
 * @oldpage: currently circulating page
 * @newpage: replacement page
6250
 *
6251 6252
 * Charge @newpage as a replacement page for @oldpage. @oldpage will
 * be uncharged upon free.
6253 6254 6255
 *
 * Both pages must be locked, @newpage->mapping must be set up.
 */
6256
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6257
{
6258
	struct mem_cgroup *memcg;
6259 6260
	unsigned int nr_pages;
	bool compound;
6261
	unsigned long flags;
6262 6263 6264 6265

	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6266 6267
	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
		       newpage);
6268 6269 6270 6271 6272

	if (mem_cgroup_disabled())
		return;

	/* Page cache replacement: new page already charged? */
6273
	if (newpage->mem_cgroup)
6274 6275
		return;

6276
	/* Swapcache readahead pages can get replaced before being charged */
6277
	memcg = oldpage->mem_cgroup;
6278
	if (!memcg)
6279 6280
		return;

6281 6282 6283 6284 6285 6286 6287 6288
	/* Force-charge the new page. The old one will be freed soon */
	compound = PageTransHuge(newpage);
	nr_pages = compound ? hpage_nr_pages(newpage) : 1;

	page_counter_charge(&memcg->memory, nr_pages);
	if (do_memsw_account())
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);
6289

6290
	commit_charge(newpage, memcg, false);
6291

6292
	local_irq_save(flags);
6293 6294
	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
	memcg_check_events(memcg, newpage);
6295
	local_irq_restore(flags);
6296 6297
}

6298
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
6299 6300
EXPORT_SYMBOL(memcg_sockets_enabled_key);

6301
void mem_cgroup_sk_alloc(struct sock *sk)
6302 6303 6304
{
	struct mem_cgroup *memcg;

6305 6306 6307
	if (!mem_cgroup_sockets_enabled)
		return;

6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321
	/*
	 * Socket cloning can throw us here with sk_memcg already
	 * filled. It won't however, necessarily happen from
	 * process context. So the test for root memcg given
	 * the current task's memcg won't help us in this case.
	 *
	 * Respecting the original socket's memcg is a better
	 * decision in this case.
	 */
	if (sk->sk_memcg) {
		css_get(&sk->sk_memcg->css);
		return;
	}

6322 6323
	rcu_read_lock();
	memcg = mem_cgroup_from_task(current);
6324 6325
	if (memcg == root_mem_cgroup)
		goto out;
6326
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
6327 6328
		goto out;
	if (css_tryget_online(&memcg->css))
6329
		sk->sk_memcg = memcg;
6330
out:
6331 6332 6333
	rcu_read_unlock();
}

6334
void mem_cgroup_sk_free(struct sock *sk)
6335
{
6336 6337
	if (sk->sk_memcg)
		css_put(&sk->sk_memcg->css);
6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349
}

/**
 * mem_cgroup_charge_skmem - charge socket memory
 * @memcg: memcg to charge
 * @nr_pages: number of pages to charge
 *
 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
 * @memcg's configured limit, %false if the charge had to be forced.
 */
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
6350
	gfp_t gfp_mask = GFP_KERNEL;
6351

6352
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6353
		struct page_counter *fail;
6354

6355 6356
		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
			memcg->tcpmem_pressure = 0;
6357 6358
			return true;
		}
6359 6360
		page_counter_charge(&memcg->tcpmem, nr_pages);
		memcg->tcpmem_pressure = 1;
6361
		return false;
6362
	}
6363

6364 6365 6366 6367
	/* Don't block in the packet receive path */
	if (in_softirq())
		gfp_mask = GFP_NOWAIT;

6368
	mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
6369

6370 6371 6372 6373
	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
		return true;

	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
6374 6375 6376 6377 6378
	return false;
}

/**
 * mem_cgroup_uncharge_skmem - uncharge socket memory
M
Mike Rapoport 已提交
6379 6380
 * @memcg: memcg to uncharge
 * @nr_pages: number of pages to uncharge
6381 6382 6383
 */
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
6384
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6385
		page_counter_uncharge(&memcg->tcpmem, nr_pages);
6386 6387
		return;
	}
6388

6389
	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
6390

6391
	refill_stock(memcg, nr_pages);
6392 6393
}

6394 6395 6396 6397 6398 6399 6400 6401 6402
static int __init cgroup_memory(char *s)
{
	char *token;

	while ((token = strsep(&s, ",")) != NULL) {
		if (!*token)
			continue;
		if (!strcmp(token, "nosocket"))
			cgroup_memory_nosocket = true;
6403 6404
		if (!strcmp(token, "nokmem"))
			cgroup_memory_nokmem = true;
6405 6406 6407 6408
	}
	return 0;
}
__setup("cgroup.memory=", cgroup_memory);
6409

6410
/*
6411 6412
 * subsys_initcall() for memory controller.
 *
6413 6414 6415 6416
 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
 * basically everything that doesn't depend on a specific mem_cgroup structure
 * should be initialized from here.
6417 6418 6419
 */
static int __init mem_cgroup_init(void)
{
6420 6421
	int cpu, node;

6422
#ifdef CONFIG_MEMCG_KMEM
6423 6424
	/*
	 * Kmem cache creation is mostly done with the slab_mutex held,
6425 6426 6427
	 * so use a workqueue with limited concurrency to avoid stalling
	 * all worker threads in case lots of cgroups are created and
	 * destroyed simultaneously.
6428
	 */
6429 6430
	memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
	BUG_ON(!memcg_kmem_cache_wq);
6431 6432
#endif

6433 6434
	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
				  memcg_hotplug_cpu_dead);
6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445

	for_each_possible_cpu(cpu)
		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
			  drain_local_stock);

	for_each_node(node) {
		struct mem_cgroup_tree_per_node *rtpn;

		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
				    node_online(node) ? node : NUMA_NO_NODE);

6446
		rtpn->rb_root = RB_ROOT;
6447
		rtpn->rb_rightmost = NULL;
6448
		spin_lock_init(&rtpn->lock);
6449 6450 6451
		soft_limit_tree.rb_tree_per_node[node] = rtpn;
	}

6452 6453 6454
	return 0;
}
subsys_initcall(mem_cgroup_init);
6455 6456

#ifdef CONFIG_MEMCG_SWAP
6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474
static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
{
	while (!atomic_inc_not_zero(&memcg->id.ref)) {
		/*
		 * The root cgroup cannot be destroyed, so it's refcount must
		 * always be >= 1.
		 */
		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
			VM_BUG_ON(1);
			break;
		}
		memcg = parent_mem_cgroup(memcg);
		if (!memcg)
			memcg = root_mem_cgroup;
	}
	return memcg;
}

6475 6476 6477 6478 6479 6480 6481 6482 6483
/**
 * mem_cgroup_swapout - transfer a memsw charge to swap
 * @page: page whose memsw charge to transfer
 * @entry: swap entry to move the charge to
 *
 * Transfer the memsw charge of @page to @entry.
 */
void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
{
6484
	struct mem_cgroup *memcg, *swap_memcg;
6485
	unsigned int nr_entries;
6486 6487 6488 6489 6490
	unsigned short oldid;

	VM_BUG_ON_PAGE(PageLRU(page), page);
	VM_BUG_ON_PAGE(page_count(page), page);

6491
	if (!do_memsw_account())
6492 6493 6494 6495 6496 6497 6498 6499
		return;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return;

6500 6501 6502 6503 6504 6505
	/*
	 * In case the memcg owning these pages has been offlined and doesn't
	 * have an ID allocated to it anymore, charge the closest online
	 * ancestor for the swap instead and transfer the memory+swap charge.
	 */
	swap_memcg = mem_cgroup_id_get_online(memcg);
6506 6507 6508 6509 6510 6511
	nr_entries = hpage_nr_pages(page);
	/* Get references for the tail pages, too */
	if (nr_entries > 1)
		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
				   nr_entries);
6512
	VM_BUG_ON_PAGE(oldid, page);
6513
	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
6514 6515 6516 6517

	page->mem_cgroup = NULL;

	if (!mem_cgroup_is_root(memcg))
6518
		page_counter_uncharge(&memcg->memory, nr_entries);
6519

6520 6521
	if (memcg != swap_memcg) {
		if (!mem_cgroup_is_root(swap_memcg))
6522 6523
			page_counter_charge(&swap_memcg->memsw, nr_entries);
		page_counter_uncharge(&memcg->memsw, nr_entries);
6524 6525
	}

6526 6527
	/*
	 * Interrupts should be disabled here because the caller holds the
M
Matthew Wilcox 已提交
6528
	 * i_pages lock which is taken with interrupts-off. It is
6529
	 * important here to have the interrupts disabled because it is the
M
Matthew Wilcox 已提交
6530
	 * only synchronisation we have for updating the per-CPU variables.
6531 6532
	 */
	VM_BUG_ON(!irqs_disabled());
6533 6534
	mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
				     -nr_entries);
6535
	memcg_check_events(memcg, page);
6536 6537

	if (!mem_cgroup_is_root(memcg))
6538
		css_put_many(&memcg->css, nr_entries);
6539 6540
}

6541 6542
/**
 * mem_cgroup_try_charge_swap - try charging swap space for a page
6543 6544 6545
 * @page: page being added to swap
 * @entry: swap entry to charge
 *
6546
 * Try to charge @page's memcg for the swap space at @entry.
6547 6548 6549 6550 6551
 *
 * Returns 0 on success, -ENOMEM on failure.
 */
int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
{
6552
	unsigned int nr_pages = hpage_nr_pages(page);
6553
	struct page_counter *counter;
6554
	struct mem_cgroup *memcg;
6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565
	unsigned short oldid;

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
		return 0;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return 0;

6566 6567
	if (!entry.val) {
		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6568
		return 0;
6569
	}
6570

6571 6572
	memcg = mem_cgroup_id_get_online(memcg);

6573
	if (!mem_cgroup_is_root(memcg) &&
6574
	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
6575 6576
		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
6577
		mem_cgroup_id_put(memcg);
6578
		return -ENOMEM;
6579
	}
6580

6581 6582 6583 6584
	/* Get references for the tail pages, too */
	if (nr_pages > 1)
		mem_cgroup_id_get_many(memcg, nr_pages - 1);
	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
6585
	VM_BUG_ON_PAGE(oldid, page);
6586
	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
6587 6588 6589 6590

	return 0;
}

6591
/**
6592
 * mem_cgroup_uncharge_swap - uncharge swap space
6593
 * @entry: swap entry to uncharge
6594
 * @nr_pages: the amount of swap space to uncharge
6595
 */
6596
void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
6597 6598 6599 6600
{
	struct mem_cgroup *memcg;
	unsigned short id;

6601
	if (!do_swap_account)
6602 6603
		return;

6604
	id = swap_cgroup_record(entry, 0, nr_pages);
6605
	rcu_read_lock();
6606
	memcg = mem_cgroup_from_id(id);
6607
	if (memcg) {
6608 6609
		if (!mem_cgroup_is_root(memcg)) {
			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
6610
				page_counter_uncharge(&memcg->swap, nr_pages);
6611
			else
6612
				page_counter_uncharge(&memcg->memsw, nr_pages);
6613
		}
6614
		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
6615
		mem_cgroup_id_put_many(memcg, nr_pages);
6616 6617 6618 6619
	}
	rcu_read_unlock();
}

6620 6621 6622 6623 6624 6625 6626 6627
long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
	long nr_swap_pages = get_nr_swap_pages();

	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return nr_swap_pages;
	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
		nr_swap_pages = min_t(long, nr_swap_pages,
6628
				      READ_ONCE(memcg->swap.max) -
6629 6630 6631 6632
				      page_counter_read(&memcg->swap));
	return nr_swap_pages;
}

6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648
bool mem_cgroup_swap_full(struct page *page)
{
	struct mem_cgroup *memcg;

	VM_BUG_ON_PAGE(!PageLocked(page), page);

	if (vm_swap_full())
		return true;
	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return false;

	memcg = page->mem_cgroup;
	if (!memcg)
		return false;

	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6649
		if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max)
6650 6651 6652 6653 6654
			return true;

	return false;
}

6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671
/* for remember boot option*/
#ifdef CONFIG_MEMCG_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata;
#endif

static int __init enable_swap_account(char *s)
{
	if (!strcmp(s, "1"))
		really_do_swap_account = 1;
	else if (!strcmp(s, "0"))
		really_do_swap_account = 0;
	return 1;
}
__setup("swapaccount=", enable_swap_account);

6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682
static u64 swap_current_read(struct cgroup_subsys_state *css,
			     struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
}

static int swap_max_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6683
	unsigned long max = READ_ONCE(memcg->swap.max);
6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704

	if (max == PAGE_COUNTER_MAX)
		seq_puts(m, "max\n");
	else
		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);

	return 0;
}

static ssize_t swap_max_write(struct kernfs_open_file *of,
			      char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long max;
	int err;

	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &max);
	if (err)
		return err;

6705
	xchg(&memcg->swap.max, max);
6706 6707 6708 6709

	return nbytes;
}

6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721
static int swap_events_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));

	seq_printf(m, "max %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
	seq_printf(m, "fail %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));

	return 0;
}

6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733
static struct cftype swap_files[] = {
	{
		.name = "swap.current",
		.flags = CFTYPE_NOT_ON_ROOT,
		.read_u64 = swap_current_read,
	},
	{
		.name = "swap.max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = swap_max_show,
		.write = swap_max_write,
	},
6734 6735 6736 6737 6738 6739
	{
		.name = "swap.events",
		.flags = CFTYPE_NOT_ON_ROOT,
		.file_offset = offsetof(struct mem_cgroup, swap_events_file),
		.seq_show = swap_events_show,
	},
6740 6741 6742
	{ }	/* terminate */
};

6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{ },	/* terminate */
};

static int __init mem_cgroup_swap_init(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
6774 6775
		WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
					       swap_files));
6776 6777 6778 6779 6780 6781 6782 6783
		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
						  memsw_cgroup_files));
	}
	return 0;
}
subsys_initcall(mem_cgroup_swap_init);

#endif /* CONFIG_MEMCG_SWAP */