memcontrol.c 187.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
B
Balbir Singh 已提交
2 3 4 5 6
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
7 8 9
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
10 11 12 13
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
14 15 16 17
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
18 19 20 21 22
 * Native page reclaim
 * Charge lifetime sanitation
 * Lockless page tracking & accounting
 * Unified hierarchy configuration model
 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
B
Balbir Singh 已提交
23 24
 */

25
#include <linux/page_counter.h>
B
Balbir Singh 已提交
26 27
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
28
#include <linux/pagewalk.h>
29
#include <linux/sched/mm.h>
30
#include <linux/shmem_fs.h>
31
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
32
#include <linux/pagemap.h>
33
#include <linux/vm_event_item.h>
34
#include <linux/smp.h>
35
#include <linux/page-flags.h>
36
#include <linux/backing-dev.h>
37 38
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
39
#include <linux/limits.h>
40
#include <linux/export.h>
41
#include <linux/mutex.h>
42
#include <linux/rbtree.h>
43
#include <linux/slab.h>
44
#include <linux/swap.h>
45
#include <linux/swapops.h>
46
#include <linux/spinlock.h>
47
#include <linux/eventfd.h>
48
#include <linux/poll.h>
49
#include <linux/sort.h>
50
#include <linux/fs.h>
51
#include <linux/seq_file.h>
52
#include <linux/vmpressure.h>
53
#include <linux/mm_inline.h>
54
#include <linux/swap_cgroup.h>
55
#include <linux/cpu.h>
56
#include <linux/oom.h>
57
#include <linux/lockdep.h>
58
#include <linux/file.h>
59
#include <linux/tracehook.h>
60
#include <linux/psi.h>
61
#include <linux/seq_buf.h>
K
KAMEZAWA Hiroyuki 已提交
62
#include "internal.h"
G
Glauber Costa 已提交
63
#include <net/sock.h>
M
Michal Hocko 已提交
64
#include <net/ip.h>
65
#include "slab.h"
B
Balbir Singh 已提交
66

67
#include <linux/uaccess.h>
68

69 70
#include <trace/events/vmscan.h>

71 72
struct cgroup_subsys memory_cgrp_subsys __read_mostly;
EXPORT_SYMBOL(memory_cgrp_subsys);
73

74 75
struct mem_cgroup *root_mem_cgroup __read_mostly;

76
#define MEM_CGROUP_RECLAIM_RETRIES	5
B
Balbir Singh 已提交
77

78 79 80
/* Socket memory accounting disabled? */
static bool cgroup_memory_nosocket;

81 82 83
/* Kernel memory accounting disabled? */
static bool cgroup_memory_nokmem;

84
/* Whether the swap controller is active */
A
Andrew Morton 已提交
85
#ifdef CONFIG_MEMCG_SWAP
86 87
int do_swap_account __read_mostly;
#else
88
#define do_swap_account		0
89 90
#endif

91 92 93 94
#ifdef CONFIG_CGROUP_WRITEBACK
static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
#endif

95 96 97 98 99 100
/* Whether legacy memory+swap accounting is active */
static bool do_memsw_account(void)
{
	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
}

101
static const char *const mem_cgroup_lru_names[] = {
102 103 104 105 106 107 108
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

109 110 111
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
112

113 114 115 116 117
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

118
struct mem_cgroup_tree_per_node {
119
	struct rb_root rb_root;
120
	struct rb_node *rb_rightmost;
121 122 123 124 125 126 127 128 129
	spinlock_t lock;
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

K
KAMEZAWA Hiroyuki 已提交
130 131 132 133 134
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
135

136 137 138
/*
 * cgroup_event represents events which userspace want to receive.
 */
139
struct mem_cgroup_event {
140
	/*
141
	 * memcg which the event belongs to.
142
	 */
143
	struct mem_cgroup *memcg;
144 145 146 147 148 149 150 151
	/*
	 * eventfd to signal userspace about the event.
	 */
	struct eventfd_ctx *eventfd;
	/*
	 * Each of these stored in a list by the cgroup.
	 */
	struct list_head list;
152 153 154 155 156
	/*
	 * register_event() callback will be used to add new userspace
	 * waiter for changes related to this event.  Use eventfd_signal()
	 * on eventfd to send notification to userspace.
	 */
157
	int (*register_event)(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
158
			      struct eventfd_ctx *eventfd, const char *args);
159 160 161 162 163
	/*
	 * unregister_event() callback will be called when userspace closes
	 * the eventfd or on cgroup removing.  This callback must be set,
	 * if you want provide notification functionality.
	 */
164
	void (*unregister_event)(struct mem_cgroup *memcg,
165
				 struct eventfd_ctx *eventfd);
166 167 168 169 170 171
	/*
	 * All fields below needed to unregister event when
	 * userspace closes eventfd.
	 */
	poll_table pt;
	wait_queue_head_t *wqh;
172
	wait_queue_entry_t wait;
173 174 175
	struct work_struct remove;
};

176 177
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
178

179 180
/* Stuffs for move charges at task migration. */
/*
181
 * Types of charges to be moved.
182
 */
183 184 185
#define MOVE_ANON	0x1U
#define MOVE_FILE	0x2U
#define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
186

187 188
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
189
	spinlock_t	  lock; /* for from, to */
190
	struct mm_struct  *mm;
191 192
	struct mem_cgroup *from;
	struct mem_cgroup *to;
193
	unsigned long flags;
194
	unsigned long precharge;
195
	unsigned long moved_charge;
196
	unsigned long moved_swap;
197 198 199
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
200
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
201 202
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
203

204 205 206 207
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
208
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
209
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
210

211 212
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
213
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
214
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
215
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
216 217 218
	NR_CHARGE_TYPE,
};

219
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
220 221 222 223
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
224
	_KMEM,
V
Vladimir Davydov 已提交
225
	_TCP,
G
Glauber Costa 已提交
226 227
};

228 229
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
230
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
231 232
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
233

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
	     iter != NULL;				\
	     iter = mem_cgroup_iter(root, iter, NULL))

#define for_each_mem_cgroup(iter)			\
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
	     iter != NULL;				\
	     iter = mem_cgroup_iter(NULL, iter, NULL))

249 250 251 252 253 254
static inline bool should_force_charge(void)
{
	return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
		(current->flags & PF_EXITING);
}

255 256 257 258 259 260 261 262 263 264 265 266 267
/* Some nice accessors for the vmpressure. */
struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
{
	if (!memcg)
		memcg = root_mem_cgroup;
	return &memcg->vmpressure;
}

struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
{
	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}

268
#ifdef CONFIG_MEMCG_KMEM
269
/*
270
 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
L
Li Zefan 已提交
271 272 273 274 275
 * The main reason for not using cgroup id for this:
 *  this works better in sparse environments, where we have a lot of memcgs,
 *  but only a few kmem-limited. Or also, if we have, for instance, 200
 *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *  200 entry array for that.
276
 *
277 278
 * The current size of the caches array is stored in memcg_nr_cache_ids. It
 * will double each time we have to increase it.
279
 */
280 281
static DEFINE_IDA(memcg_cache_ida);
int memcg_nr_cache_ids;
282

283 284 285 286 287 288 289 290 291 292 293 294 295
/* Protects memcg_nr_cache_ids */
static DECLARE_RWSEM(memcg_cache_ids_sem);

void memcg_get_cache_ids(void)
{
	down_read(&memcg_cache_ids_sem);
}

void memcg_put_cache_ids(void)
{
	up_read(&memcg_cache_ids_sem);
}

296 297 298 299 300 301
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
L
Li Zefan 已提交
302
 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
303 304
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
L
Li Zefan 已提交
305
 * cgrp_id space is not getting any smaller, and we don't have to necessarily
306 307 308
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
L
Li Zefan 已提交
309
#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
310

311 312 313 314 315 316
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
317
DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
318
EXPORT_SYMBOL(memcg_kmem_enabled_key);
319

320 321
struct workqueue_struct *memcg_kmem_cache_wq;

322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
static int memcg_shrinker_map_size;
static DEFINE_MUTEX(memcg_shrinker_map_mutex);

static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
{
	kvfree(container_of(head, struct memcg_shrinker_map, rcu));
}

static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
					 int size, int old_size)
{
	struct memcg_shrinker_map *new, *old;
	int nid;

	lockdep_assert_held(&memcg_shrinker_map_mutex);

	for_each_node(nid) {
		old = rcu_dereference_protected(
			mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
		/* Not yet online memcg */
		if (!old)
			return 0;

		new = kvmalloc(sizeof(*new) + size, GFP_KERNEL);
		if (!new)
			return -ENOMEM;

		/* Set all old bits, clear all new bits */
		memset(new->map, (int)0xff, old_size);
		memset((void *)new->map + old_size, 0, size - old_size);

		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
	}

	return 0;
}

static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_node *pn;
	struct memcg_shrinker_map *map;
	int nid;

	if (mem_cgroup_is_root(memcg))
		return;

	for_each_node(nid) {
		pn = mem_cgroup_nodeinfo(memcg, nid);
		map = rcu_dereference_protected(pn->shrinker_map, true);
		if (map)
			kvfree(map);
		rcu_assign_pointer(pn->shrinker_map, NULL);
	}
}

static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
{
	struct memcg_shrinker_map *map;
	int nid, size, ret = 0;

	if (mem_cgroup_is_root(memcg))
		return 0;

	mutex_lock(&memcg_shrinker_map_mutex);
	size = memcg_shrinker_map_size;
	for_each_node(nid) {
		map = kvzalloc(sizeof(*map) + size, GFP_KERNEL);
		if (!map) {
			memcg_free_shrinker_maps(memcg);
			ret = -ENOMEM;
			break;
		}
		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
	}
	mutex_unlock(&memcg_shrinker_map_mutex);

	return ret;
}

int memcg_expand_shrinker_maps(int new_id)
{
	int size, old_size, ret = 0;
	struct mem_cgroup *memcg;

	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
	old_size = memcg_shrinker_map_size;
	if (size <= old_size)
		return 0;

	mutex_lock(&memcg_shrinker_map_mutex);
	if (!root_mem_cgroup)
		goto unlock;

	for_each_mem_cgroup(memcg) {
		if (mem_cgroup_is_root(memcg))
			continue;
		ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
		if (ret)
			goto unlock;
	}
unlock:
	if (!ret)
		memcg_shrinker_map_size = size;
	mutex_unlock(&memcg_shrinker_map_mutex);
	return ret;
}
429 430 431 432 433 434 435 436

void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
{
	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
		struct memcg_shrinker_map *map;

		rcu_read_lock();
		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
437 438
		/* Pairs with smp mb in shrink_slab() */
		smp_mb__before_atomic();
439 440 441 442 443
		set_bit(shrinker_id, map->map);
		rcu_read_unlock();
	}
}

444 445 446 447 448 449
#else /* CONFIG_MEMCG_KMEM */
static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
{
	return 0;
}
static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { }
450
#endif /* CONFIG_MEMCG_KMEM */
451

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
/**
 * mem_cgroup_css_from_page - css of the memcg associated with a page
 * @page: page of interest
 *
 * If memcg is bound to the default hierarchy, css of the memcg associated
 * with @page is returned.  The returned css remains associated with @page
 * until it is released.
 *
 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
 * is returned.
 */
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
{
	struct mem_cgroup *memcg;

	memcg = page->mem_cgroup;

469
	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
470 471 472 473 474
		memcg = root_mem_cgroup;

	return &memcg->css;
}

475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
/**
 * page_cgroup_ino - return inode number of the memcg a page is charged to
 * @page: the page
 *
 * Look up the closest online ancestor of the memory cgroup @page is charged to
 * and return its inode number or 0 if @page is not charged to any cgroup. It
 * is safe to call this function without holding a reference to @page.
 *
 * Note, this function is inherently racy, because there is nothing to prevent
 * the cgroup inode from getting torn down and potentially reallocated a moment
 * after page_cgroup_ino() returns, so it only should be used by callers that
 * do not care (such as procfs interfaces).
 */
ino_t page_cgroup_ino(struct page *page)
{
	struct mem_cgroup *memcg;
	unsigned long ino = 0;

	rcu_read_lock();
494 495 496 497
	if (PageHead(page) && PageSlab(page))
		memcg = memcg_from_slab_page(page);
	else
		memcg = READ_ONCE(page->mem_cgroup);
498 499 500 501 502 503 504 505
	while (memcg && !(memcg->css.flags & CSS_ONLINE))
		memcg = parent_mem_cgroup(memcg);
	if (memcg)
		ino = cgroup_ino(memcg->css.cgroup);
	rcu_read_unlock();
	return ino;
}

506 507
static struct mem_cgroup_per_node *
mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
508
{
509
	int nid = page_to_nid(page);
510

511
	return memcg->nodeinfo[nid];
512 513
}

514 515
static struct mem_cgroup_tree_per_node *
soft_limit_tree_node(int nid)
516
{
517
	return soft_limit_tree.rb_tree_per_node[nid];
518 519
}

520
static struct mem_cgroup_tree_per_node *
521 522 523 524
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);

525
	return soft_limit_tree.rb_tree_per_node[nid];
526 527
}

528 529
static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
					 struct mem_cgroup_tree_per_node *mctz,
530
					 unsigned long new_usage_in_excess)
531 532 533
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
534
	struct mem_cgroup_per_node *mz_node;
535
	bool rightmost = true;
536 537 538 539 540 541 542 543 544

	if (mz->on_tree)
		return;

	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
	while (*p) {
		parent = *p;
545
		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
546
					tree_node);
547
		if (mz->usage_in_excess < mz_node->usage_in_excess) {
548
			p = &(*p)->rb_left;
549 550 551
			rightmost = false;
		}

552 553 554 555 556 557 558
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
559 560 561 562

	if (rightmost)
		mctz->rb_rightmost = &mz->tree_node;

563 564 565 566 567
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
}

568 569
static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
					 struct mem_cgroup_tree_per_node *mctz)
570 571 572
{
	if (!mz->on_tree)
		return;
573 574 575 576

	if (&mz->tree_node == mctz->rb_rightmost)
		mctz->rb_rightmost = rb_prev(&mz->tree_node);

577 578 579 580
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

581 582
static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
				       struct mem_cgroup_tree_per_node *mctz)
583
{
584 585 586
	unsigned long flags;

	spin_lock_irqsave(&mctz->lock, flags);
587
	__mem_cgroup_remove_exceeded(mz, mctz);
588
	spin_unlock_irqrestore(&mctz->lock, flags);
589 590
}

591 592 593
static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
{
	unsigned long nr_pages = page_counter_read(&memcg->memory);
594
	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
595 596 597 598 599 600 601
	unsigned long excess = 0;

	if (nr_pages > soft_limit)
		excess = nr_pages - soft_limit;

	return excess;
}
602 603 604

static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
{
605
	unsigned long excess;
606 607
	struct mem_cgroup_per_node *mz;
	struct mem_cgroup_tree_per_node *mctz;
608

609
	mctz = soft_limit_tree_from_page(page);
610 611
	if (!mctz)
		return;
612 613 614 615 616
	/*
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
617
		mz = mem_cgroup_page_nodeinfo(memcg, page);
618
		excess = soft_limit_excess(memcg);
619 620 621 622 623
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
		if (excess || mz->on_tree) {
624 625 626
			unsigned long flags;

			spin_lock_irqsave(&mctz->lock, flags);
627 628
			/* if on-tree, remove it */
			if (mz->on_tree)
629
				__mem_cgroup_remove_exceeded(mz, mctz);
630 631 632 633
			/*
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
			 */
634
			__mem_cgroup_insert_exceeded(mz, mctz, excess);
635
			spin_unlock_irqrestore(&mctz->lock, flags);
636 637 638 639 640 641
		}
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
{
642 643 644
	struct mem_cgroup_tree_per_node *mctz;
	struct mem_cgroup_per_node *mz;
	int nid;
645

646
	for_each_node(nid) {
647 648
		mz = mem_cgroup_nodeinfo(memcg, nid);
		mctz = soft_limit_tree_node(nid);
649 650
		if (mctz)
			mem_cgroup_remove_exceeded(mz, mctz);
651 652 653
	}
}

654 655
static struct mem_cgroup_per_node *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
656
{
657
	struct mem_cgroup_per_node *mz;
658 659 660

retry:
	mz = NULL;
661
	if (!mctz->rb_rightmost)
662 663
		goto done;		/* Nothing to reclaim from */

664 665
	mz = rb_entry(mctz->rb_rightmost,
		      struct mem_cgroup_per_node, tree_node);
666 667 668 669 670
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
671
	__mem_cgroup_remove_exceeded(mz, mctz);
672
	if (!soft_limit_excess(mz->memcg) ||
673
	    !css_tryget_online(&mz->memcg->css))
674 675 676 677 678
		goto retry;
done:
	return mz;
}

679 680
static struct mem_cgroup_per_node *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
681
{
682
	struct mem_cgroup_per_node *mz;
683

684
	spin_lock_irq(&mctz->lock);
685
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
686
	spin_unlock_irq(&mctz->lock);
687 688 689
	return mz;
}

690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
/**
 * __mod_memcg_state - update cgroup memory statistics
 * @memcg: the memory cgroup
 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
 * @val: delta to add to the counter, can be negative
 */
void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
{
	long x;

	if (mem_cgroup_disabled())
		return;

	x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
705 706
		struct mem_cgroup *mi;

707 708 709 710 711
		/*
		 * Batch local counters to keep them in sync with
		 * the hierarchical ones.
		 */
		__this_cpu_add(memcg->vmstats_local->stat[idx], x);
712 713
		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
			atomic_long_add(x, &mi->vmstats[idx]);
714 715 716 717 718
		x = 0;
	}
	__this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
}

719 720 721 722 723 724 725 726 727 728 729
static struct mem_cgroup_per_node *
parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
{
	struct mem_cgroup *parent;

	parent = parent_mem_cgroup(pn->memcg);
	if (!parent)
		return NULL;
	return mem_cgroup_nodeinfo(parent, nid);
}

730 731 732 733 734 735 736 737 738 739 740 741 742
/**
 * __mod_lruvec_state - update lruvec memory statistics
 * @lruvec: the lruvec
 * @idx: the stat item
 * @val: delta to add to the counter, can be negative
 *
 * The lruvec is the intersection of the NUMA node and a cgroup. This
 * function updates the all three counters that are affected by a
 * change of state at this level: per-node, per-cgroup, per-lruvec.
 */
void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
			int val)
{
743
	pg_data_t *pgdat = lruvec_pgdat(lruvec);
744
	struct mem_cgroup_per_node *pn;
745
	struct mem_cgroup *memcg;
746 747 748
	long x;

	/* Update node */
749
	__mod_node_page_state(pgdat, idx, val);
750 751 752 753 754

	if (mem_cgroup_disabled())
		return;

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
755
	memcg = pn->memcg;
756 757

	/* Update memcg */
758
	__mod_memcg_state(memcg, idx, val);
759

760 761 762
	/* Update lruvec */
	__this_cpu_add(pn->lruvec_stat_local->count[idx], val);

763 764
	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
765 766 767 768
		struct mem_cgroup_per_node *pi;

		for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
			atomic_long_add(x, &pi->lruvec_stat[idx]);
769 770 771 772 773
		x = 0;
	}
	__this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
}

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
{
	struct page *page = virt_to_head_page(p);
	pg_data_t *pgdat = page_pgdat(page);
	struct mem_cgroup *memcg;
	struct lruvec *lruvec;

	rcu_read_lock();
	memcg = memcg_from_slab_page(page);

	/* Untracked pages have no memcg, no lruvec. Update only the node */
	if (!memcg || memcg == root_mem_cgroup) {
		__mod_node_page_state(pgdat, idx, val);
	} else {
		lruvec = mem_cgroup_lruvec(pgdat, memcg);
		__mod_lruvec_state(lruvec, idx, val);
	}
	rcu_read_unlock();
}

794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
/**
 * __count_memcg_events - account VM events in a cgroup
 * @memcg: the memory cgroup
 * @idx: the event item
 * @count: the number of events that occured
 */
void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
			  unsigned long count)
{
	unsigned long x;

	if (mem_cgroup_disabled())
		return;

	x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
	if (unlikely(x > MEMCG_CHARGE_BATCH)) {
810 811
		struct mem_cgroup *mi;

812 813 814 815 816
		/*
		 * Batch local counters to keep them in sync with
		 * the hierarchical ones.
		 */
		__this_cpu_add(memcg->vmstats_local->events[idx], x);
817 818
		for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
			atomic_long_add(x, &mi->vmevents[idx]);
819 820 821 822 823
		x = 0;
	}
	__this_cpu_write(memcg->vmstats_percpu->events[idx], x);
}

824
static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
825
{
826
	return atomic_long_read(&memcg->vmevents[event]);
827 828
}

829 830
static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
{
831 832 833 834 835 836
	long x = 0;
	int cpu;

	for_each_possible_cpu(cpu)
		x += per_cpu(memcg->vmstats_local->events[event], cpu);
	return x;
837 838
}

839
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
840
					 struct page *page,
841
					 bool compound, int nr_pages)
842
{
843 844 845 846
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
847
	if (PageAnon(page))
848
		__mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
849
	else {
850
		__mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
851
		if (PageSwapBacked(page))
852
			__mod_memcg_state(memcg, NR_SHMEM, nr_pages);
853
	}
854

855 856
	if (compound) {
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
857
		__mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
858
	}
859

860 861
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
862
		__count_memcg_events(memcg, PGPGIN, 1);
863
	else {
864
		__count_memcg_events(memcg, PGPGOUT, 1);
865 866
		nr_pages = -nr_pages; /* for event */
	}
867

868
	__this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
869 870
}

871 872
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
873 874 875
{
	unsigned long val, next;

876 877
	val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
	next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
878
	/* from time_after() in jiffies.h */
879
	if ((long)(next - val) < 0) {
880 881 882 883
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
884 885 886
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
887 888 889 890 891 892
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
893
		__this_cpu_write(memcg->vmstats_percpu->targets[target], next);
894
		return true;
895
	}
896
	return false;
897 898 899 900 901 902
}

/*
 * Check events in order.
 *
 */
903
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
904 905
{
	/* threshold event is triggered in finer grain than soft limit */
906 907
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
908
		bool do_softlimit;
909
		bool do_numainfo __maybe_unused;
910

911 912
		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
913 914 915 916
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
917
		mem_cgroup_threshold(memcg);
918 919
		if (unlikely(do_softlimit))
			mem_cgroup_update_tree(memcg, page);
920
#if MAX_NUMNODES > 1
921
		if (unlikely(do_numainfo))
922
			atomic_inc(&memcg->numainfo_events);
923
#endif
924
	}
925 926
}

927
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
928
{
929 930 931 932 933 934 935 936
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

937
	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
938
}
M
Michal Hocko 已提交
939
EXPORT_SYMBOL(mem_cgroup_from_task);
940

941 942 943 944 945 946 947 948 949
/**
 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
 * @mm: mm from which memcg should be extracted. It can be NULL.
 *
 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
 * returned.
 */
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
950
{
951 952 953 954
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return NULL;
955

956 957
	rcu_read_lock();
	do {
958 959 960 961 962 963
		/*
		 * Page cache insertions can happen withou an
		 * actual mm context, e.g. during disk probing
		 * on boot, loopback IO, acct() writes etc.
		 */
		if (unlikely(!mm))
964
			memcg = root_mem_cgroup;
965 966 967 968 969
		else {
			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
			if (unlikely(!memcg))
				memcg = root_mem_cgroup;
		}
970
	} while (!css_tryget_online(&memcg->css));
971
	rcu_read_unlock();
972
	return memcg;
973
}
974 975
EXPORT_SYMBOL(get_mem_cgroup_from_mm);

976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
/**
 * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
 * @page: page from which memcg should be extracted.
 *
 * Obtain a reference on page->memcg and returns it if successful. Otherwise
 * root_mem_cgroup is returned.
 */
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
	struct mem_cgroup *memcg = page->mem_cgroup;

	if (mem_cgroup_disabled())
		return NULL;

	rcu_read_lock();
	if (!memcg || !css_tryget_online(&memcg->css))
		memcg = root_mem_cgroup;
	rcu_read_unlock();
	return memcg;
}
EXPORT_SYMBOL(get_mem_cgroup_from_page);

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
/**
 * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
 */
static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
{
	if (unlikely(current->active_memcg)) {
		struct mem_cgroup *memcg = root_mem_cgroup;

		rcu_read_lock();
		if (css_tryget_online(&current->active_memcg->css))
			memcg = current->active_memcg;
		rcu_read_unlock();
		return memcg;
	}
	return get_mem_cgroup_from_mm(current->mm);
}
1014

1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
1028
 * Reclaimers can specify a node and a priority level in @reclaim to
1029
 * divide up the memcgs in the hierarchy among all concurrent
1030
 * reclaimers operating on the same node and priority.
1031
 */
1032
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1033
				   struct mem_cgroup *prev,
1034
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
1035
{
M
Michal Hocko 已提交
1036
	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1037
	struct cgroup_subsys_state *css = NULL;
1038
	struct mem_cgroup *memcg = NULL;
1039
	struct mem_cgroup *pos = NULL;
1040

1041 1042
	if (mem_cgroup_disabled())
		return NULL;
1043

1044 1045
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
1046

1047
	if (prev && !reclaim)
1048
		pos = prev;
K
KAMEZAWA Hiroyuki 已提交
1049

1050 1051
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
1052
			goto out;
1053
		return root;
1054
	}
K
KAMEZAWA Hiroyuki 已提交
1055

1056
	rcu_read_lock();
M
Michal Hocko 已提交
1057

1058
	if (reclaim) {
1059
		struct mem_cgroup_per_node *mz;
1060

1061
		mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
1062 1063 1064 1065 1066
		iter = &mz->iter[reclaim->priority];

		if (prev && reclaim->generation != iter->generation)
			goto out_unlock;

1067
		while (1) {
1068
			pos = READ_ONCE(iter->position);
1069 1070
			if (!pos || css_tryget(&pos->css))
				break;
1071
			/*
1072 1073 1074 1075 1076 1077
			 * css reference reached zero, so iter->position will
			 * be cleared by ->css_released. However, we should not
			 * rely on this happening soon, because ->css_released
			 * is called from a work queue, and by busy-waiting we
			 * might block it. So we clear iter->position right
			 * away.
1078
			 */
1079 1080
			(void)cmpxchg(&iter->position, pos, NULL);
		}
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
	}

	if (pos)
		css = &pos->css;

	for (;;) {
		css = css_next_descendant_pre(css, &root->css);
		if (!css) {
			/*
			 * Reclaimers share the hierarchy walk, and a
			 * new one might jump in right at the end of
			 * the hierarchy - make sure they see at least
			 * one group and restart from the beginning.
			 */
			if (!prev)
				continue;
			break;
1098
		}
K
KAMEZAWA Hiroyuki 已提交
1099

1100 1101 1102 1103 1104 1105
		/*
		 * Verify the css and acquire a reference.  The root
		 * is provided by the caller, so we know it's alive
		 * and kicking, and don't take an extra reference.
		 */
		memcg = mem_cgroup_from_css(css);
K
KAMEZAWA Hiroyuki 已提交
1106

1107 1108
		if (css == &root->css)
			break;
K
KAMEZAWA Hiroyuki 已提交
1109

1110 1111
		if (css_tryget(css))
			break;
1112

1113
		memcg = NULL;
1114
	}
1115 1116 1117

	if (reclaim) {
		/*
1118 1119 1120
		 * The position could have already been updated by a competing
		 * thread, so check that the value hasn't changed since we read
		 * it to avoid reclaiming from the same cgroup twice.
1121
		 */
1122 1123
		(void)cmpxchg(&iter->position, pos, memcg);

1124 1125 1126 1127 1128 1129 1130
		if (pos)
			css_put(&pos->css);

		if (!memcg)
			iter->generation++;
		else if (!prev)
			reclaim->generation = iter->generation;
1131
	}
1132

1133 1134
out_unlock:
	rcu_read_unlock();
1135
out:
1136 1137 1138
	if (prev && prev != root)
		css_put(&prev->css);

1139
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
1140
}
K
KAMEZAWA Hiroyuki 已提交
1141

1142 1143 1144 1145 1146 1147 1148
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
1149 1150 1151 1152 1153 1154
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
1155

1156 1157
static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
					struct mem_cgroup *dead_memcg)
1158 1159
{
	struct mem_cgroup_reclaim_iter *iter;
1160 1161
	struct mem_cgroup_per_node *mz;
	int nid;
1162 1163
	int i;

1164 1165 1166 1167 1168 1169
	for_each_node(nid) {
		mz = mem_cgroup_nodeinfo(from, nid);
		for (i = 0; i <= DEF_PRIORITY; i++) {
			iter = &mz->iter[i];
			cmpxchg(&iter->position,
				dead_memcg, NULL);
1170 1171 1172 1173
		}
	}
}

1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194
static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
{
	struct mem_cgroup *memcg = dead_memcg;
	struct mem_cgroup *last;

	do {
		__invalidate_reclaim_iterators(memcg, dead_memcg);
		last = memcg;
	} while ((memcg = parent_mem_cgroup(memcg)));

	/*
	 * When cgruop1 non-hierarchy mode is used,
	 * parent_mem_cgroup() does not walk all the way up to the
	 * cgroup root (root_mem_cgroup). So we have to handle
	 * dead_memcg from cgroup root separately.
	 */
	if (last != root_mem_cgroup)
		__invalidate_reclaim_iterators(root_mem_cgroup,
						dead_memcg);
}

1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
/**
 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
 * @memcg: hierarchy root
 * @fn: function to call for each task
 * @arg: argument passed to @fn
 *
 * This function iterates over tasks attached to @memcg or to any of its
 * descendants and calls @fn for each task. If @fn returns a non-zero
 * value, the function breaks the iteration loop and returns the value.
 * Otherwise, it will iterate over all tasks and return 0.
 *
 * This function must not be called for the root memory cgroup.
 */
int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
			  int (*fn)(struct task_struct *, void *), void *arg)
{
	struct mem_cgroup *iter;
	int ret = 0;

	BUG_ON(memcg == root_mem_cgroup);

	for_each_mem_cgroup_tree(iter, memcg) {
		struct css_task_iter it;
		struct task_struct *task;

1220
		css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
		while (!ret && (task = css_task_iter_next(&it)))
			ret = fn(task, arg);
		css_task_iter_end(&it);
		if (ret) {
			mem_cgroup_iter_break(memcg, iter);
			break;
		}
	}
	return ret;
}

1232
/**
1233
 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1234
 * @page: the page
1235
 * @pgdat: pgdat of the page
1236 1237 1238 1239
 *
 * This function is only safe when following the LRU page isolation
 * and putback protocol: the LRU lock must be held, and the page must
 * either be PageLRU() or the caller must have isolated/allocated it.
1240
 */
M
Mel Gorman 已提交
1241
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
K
KAMEZAWA Hiroyuki 已提交
1242
{
1243
	struct mem_cgroup_per_node *mz;
1244
	struct mem_cgroup *memcg;
1245
	struct lruvec *lruvec;
1246

1247
	if (mem_cgroup_disabled()) {
M
Mel Gorman 已提交
1248
		lruvec = &pgdat->lruvec;
1249 1250
		goto out;
	}
1251

1252
	memcg = page->mem_cgroup;
1253
	/*
1254
	 * Swapcache readahead pages are added to the LRU - and
1255
	 * possibly migrated - before they are charged.
1256
	 */
1257 1258
	if (!memcg)
		memcg = root_mem_cgroup;
1259

1260
	mz = mem_cgroup_page_nodeinfo(memcg, page);
1261 1262 1263 1264 1265 1266 1267
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
M
Mel Gorman 已提交
1268 1269
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
1270
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1271
}
1272

1273
/**
1274 1275 1276
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
1277
 * @zid: zone id of the accounted pages
1278
 * @nr_pages: positive when adding or negative when removing
1279
 *
1280 1281 1282
 * This function must be called under lru_lock, just before a page is added
 * to or just after a page is removed from an lru list (that ordering being
 * so as to allow it to check that lru_size 0 is consistent with list_empty).
1283
 */
1284
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1285
				int zid, int nr_pages)
1286
{
1287
	struct mem_cgroup_per_node *mz;
1288
	unsigned long *lru_size;
1289
	long size;
1290 1291 1292 1293

	if (mem_cgroup_disabled())
		return;

1294
	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1295
	lru_size = &mz->lru_zone_size[zid][lru];
1296 1297 1298 1299 1300

	if (nr_pages < 0)
		*lru_size += nr_pages;

	size = *lru_size;
1301 1302 1303
	if (WARN_ONCE(size < 0,
		"%s(%p, %d, %d): lru_size %ld\n",
		__func__, lruvec, lru, nr_pages, size)) {
1304 1305 1306 1307 1308 1309
		VM_BUG_ON(1);
		*lru_size = 0;
	}

	if (nr_pages > 0)
		*lru_size += nr_pages;
K
KAMEZAWA Hiroyuki 已提交
1310
}
1311

1312
/**
1313
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1314
 * @memcg: the memory cgroup
1315
 *
1316
 * Returns the maximum amount of memory @mem can be charged with, in
1317
 * pages.
1318
 */
1319
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1320
{
1321 1322 1323
	unsigned long margin = 0;
	unsigned long count;
	unsigned long limit;
1324

1325
	count = page_counter_read(&memcg->memory);
1326
	limit = READ_ONCE(memcg->memory.max);
1327 1328 1329
	if (count < limit)
		margin = limit - count;

1330
	if (do_memsw_account()) {
1331
		count = page_counter_read(&memcg->memsw);
1332
		limit = READ_ONCE(memcg->memsw.max);
1333 1334
		if (count <= limit)
			margin = min(margin, limit - count);
1335 1336
		else
			margin = 0;
1337 1338 1339
	}

	return margin;
1340 1341
}

1342
/*
Q
Qiang Huang 已提交
1343
 * A routine for checking "mem" is under move_account() or not.
1344
 *
Q
Qiang Huang 已提交
1345 1346 1347
 * Checking a cgroup is mc.from or mc.to or under hierarchy of
 * moving cgroups. This is for waiting at high-memory pressure
 * caused by "move".
1348
 */
1349
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1350
{
1351 1352
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1353
	bool ret = false;
1354 1355 1356 1357 1358 1359 1360 1361 1362
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1363

1364 1365
	ret = mem_cgroup_is_descendant(from, memcg) ||
		mem_cgroup_is_descendant(to, memcg);
1366 1367
unlock:
	spin_unlock(&mc.lock);
1368 1369 1370
	return ret;
}

1371
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1372 1373
{
	if (mc.moving_task && current != mc.moving_task) {
1374
		if (mem_cgroup_under_move(memcg)) {
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1387 1388 1389 1390
static char *memory_stat_format(struct mem_cgroup *memcg)
{
	struct seq_buf s;
	int i;
1391

1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
	seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
	if (!s.buffer)
		return NULL;

	/*
	 * Provide statistics on the state of the memory subsystem as
	 * well as cumulative event counters that show past behavior.
	 *
	 * This list is ordered following a combination of these gradients:
	 * 1) generic big picture -> specifics and details
	 * 2) reflecting userspace activity -> reflecting kernel heuristics
	 *
	 * Current memory state:
	 */

	seq_buf_printf(&s, "anon %llu\n",
		       (u64)memcg_page_state(memcg, MEMCG_RSS) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "file %llu\n",
		       (u64)memcg_page_state(memcg, MEMCG_CACHE) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "kernel_stack %llu\n",
		       (u64)memcg_page_state(memcg, MEMCG_KERNEL_STACK_KB) *
		       1024);
	seq_buf_printf(&s, "slab %llu\n",
		       (u64)(memcg_page_state(memcg, NR_SLAB_RECLAIMABLE) +
			     memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE)) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "sock %llu\n",
		       (u64)memcg_page_state(memcg, MEMCG_SOCK) *
		       PAGE_SIZE);

	seq_buf_printf(&s, "shmem %llu\n",
		       (u64)memcg_page_state(memcg, NR_SHMEM) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "file_mapped %llu\n",
		       (u64)memcg_page_state(memcg, NR_FILE_MAPPED) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "file_dirty %llu\n",
		       (u64)memcg_page_state(memcg, NR_FILE_DIRTY) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "file_writeback %llu\n",
		       (u64)memcg_page_state(memcg, NR_WRITEBACK) *
		       PAGE_SIZE);

	/*
	 * TODO: We should eventually replace our own MEMCG_RSS_HUGE counter
	 * with the NR_ANON_THP vm counter, but right now it's a pain in the
	 * arse because it requires migrating the work out of rmap to a place
	 * where the page->mem_cgroup is set up and stable.
	 */
	seq_buf_printf(&s, "anon_thp %llu\n",
		       (u64)memcg_page_state(memcg, MEMCG_RSS_HUGE) *
		       PAGE_SIZE);

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_buf_printf(&s, "%s %llu\n", mem_cgroup_lru_names[i],
			       (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
			       PAGE_SIZE);

	seq_buf_printf(&s, "slab_reclaimable %llu\n",
		       (u64)memcg_page_state(memcg, NR_SLAB_RECLAIMABLE) *
		       PAGE_SIZE);
	seq_buf_printf(&s, "slab_unreclaimable %llu\n",
		       (u64)memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE) *
		       PAGE_SIZE);

	/* Accumulated memory events */

	seq_buf_printf(&s, "pgfault %lu\n", memcg_events(memcg, PGFAULT));
	seq_buf_printf(&s, "pgmajfault %lu\n", memcg_events(memcg, PGMAJFAULT));

	seq_buf_printf(&s, "workingset_refault %lu\n",
		       memcg_page_state(memcg, WORKINGSET_REFAULT));
	seq_buf_printf(&s, "workingset_activate %lu\n",
		       memcg_page_state(memcg, WORKINGSET_ACTIVATE));
	seq_buf_printf(&s, "workingset_nodereclaim %lu\n",
		       memcg_page_state(memcg, WORKINGSET_NODERECLAIM));

	seq_buf_printf(&s, "pgrefill %lu\n", memcg_events(memcg, PGREFILL));
	seq_buf_printf(&s, "pgscan %lu\n",
		       memcg_events(memcg, PGSCAN_KSWAPD) +
		       memcg_events(memcg, PGSCAN_DIRECT));
	seq_buf_printf(&s, "pgsteal %lu\n",
		       memcg_events(memcg, PGSTEAL_KSWAPD) +
		       memcg_events(memcg, PGSTEAL_DIRECT));
	seq_buf_printf(&s, "pgactivate %lu\n", memcg_events(memcg, PGACTIVATE));
	seq_buf_printf(&s, "pgdeactivate %lu\n", memcg_events(memcg, PGDEACTIVATE));
	seq_buf_printf(&s, "pglazyfree %lu\n", memcg_events(memcg, PGLAZYFREE));
	seq_buf_printf(&s, "pglazyfreed %lu\n", memcg_events(memcg, PGLAZYFREED));

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	seq_buf_printf(&s, "thp_fault_alloc %lu\n",
		       memcg_events(memcg, THP_FAULT_ALLOC));
	seq_buf_printf(&s, "thp_collapse_alloc %lu\n",
		       memcg_events(memcg, THP_COLLAPSE_ALLOC));
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

	/* The above should easily fit into one page */
	WARN_ON_ONCE(seq_buf_has_overflowed(&s));

	return s.buffer;
}
1495

1496
#define K(x) ((x) << (PAGE_SHIFT-10))
1497
/**
1498 1499
 * mem_cgroup_print_oom_context: Print OOM information relevant to
 * memory controller.
1500 1501 1502 1503 1504 1505
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
1506
void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1507 1508 1509
{
	rcu_read_lock();

1510 1511 1512 1513 1514
	if (memcg) {
		pr_cont(",oom_memcg=");
		pr_cont_cgroup_path(memcg->css.cgroup);
	} else
		pr_cont(",global_oom");
1515
	if (p) {
1516
		pr_cont(",task_memcg=");
1517 1518
		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
	}
1519
	rcu_read_unlock();
1520 1521 1522 1523 1524 1525 1526 1527 1528
}

/**
 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
 * memory controller.
 * @memcg: The memory cgroup that went over limit
 */
void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
{
1529
	char *buf;
1530

1531 1532
	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memory)),
1533
		K((u64)memcg->memory.max), memcg->memory.failcnt);
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
		pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
			K((u64)page_counter_read(&memcg->swap)),
			K((u64)memcg->swap.max), memcg->swap.failcnt);
	else {
		pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
			K((u64)page_counter_read(&memcg->memsw)),
			K((u64)memcg->memsw.max), memcg->memsw.failcnt);
		pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
			K((u64)page_counter_read(&memcg->kmem)),
			K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1545
	}
1546 1547 1548 1549 1550 1551 1552 1553 1554

	pr_info("Memory cgroup stats for ");
	pr_cont_cgroup_path(memcg->css.cgroup);
	pr_cont(":");
	buf = memory_stat_format(memcg);
	if (!buf)
		return;
	pr_info("%s", buf);
	kfree(buf);
1555 1556
}

D
David Rientjes 已提交
1557 1558 1559
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1560
unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1561
{
1562
	unsigned long max;
1563

1564
	max = memcg->memory.max;
1565
	if (mem_cgroup_swappiness(memcg)) {
1566 1567
		unsigned long memsw_max;
		unsigned long swap_max;
1568

1569 1570 1571 1572
		memsw_max = memcg->memsw.max;
		swap_max = memcg->swap.max;
		swap_max = min(swap_max, (unsigned long)total_swap_pages);
		max = min(max + swap_max, memsw_max);
1573
	}
1574
	return max;
D
David Rientjes 已提交
1575 1576
}

1577
static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1578
				     int order)
1579
{
1580 1581 1582
	struct oom_control oc = {
		.zonelist = NULL,
		.nodemask = NULL,
1583
		.memcg = memcg,
1584 1585 1586
		.gfp_mask = gfp_mask,
		.order = order,
	};
1587
	bool ret;
1588

1589 1590 1591 1592 1593 1594 1595
	if (mutex_lock_killable(&oom_lock))
		return true;
	/*
	 * A few threads which were not waiting at mutex_lock_killable() can
	 * fail to bail out. Therefore, check again after holding oom_lock.
	 */
	ret = should_force_charge() || out_of_memory(&oc);
1596
	mutex_unlock(&oom_lock);
1597
	return ret;
1598 1599
}

1600 1601
#if MAX_NUMNODES > 1

1602 1603
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1604
 * @memcg: the target memcg
1605 1606 1607 1608 1609 1610 1611
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1612
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1613 1614
		int nid, bool noswap)
{
1615 1616
	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);

1617 1618
	if (lruvec_page_state(lruvec, NR_INACTIVE_FILE) ||
	    lruvec_page_state(lruvec, NR_ACTIVE_FILE))
1619 1620 1621
		return true;
	if (noswap || !total_swap_pages)
		return false;
1622 1623
	if (lruvec_page_state(lruvec, NR_INACTIVE_ANON) ||
	    lruvec_page_state(lruvec, NR_ACTIVE_ANON))
1624 1625 1626 1627
		return true;
	return false;

}
1628 1629 1630 1631 1632 1633 1634

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1635
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1636 1637
{
	int nid;
1638 1639 1640 1641
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1642
	if (!atomic_read(&memcg->numainfo_events))
1643
		return;
1644
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1645 1646 1647
		return;

	/* make a nodemask where this memcg uses memory from */
1648
	memcg->scan_nodes = node_states[N_MEMORY];
1649

1650
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1651

1652 1653
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1654
	}
1655

1656 1657
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1672
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1673 1674 1675
{
	int node;

1676 1677
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1678

1679
	node = next_node_in(node, memcg->scan_nodes);
1680
	/*
1681 1682 1683
	 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
	 * last time it really checked all the LRUs due to rate limiting.
	 * Fallback to the current node in that case for simplicity.
1684 1685 1686 1687
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1688
	memcg->last_scanned_node = node;
1689 1690 1691
	return node;
}
#else
1692
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1693 1694 1695 1696 1697
{
	return 0;
}
#endif

1698
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1699
				   pg_data_t *pgdat,
1700 1701 1702 1703 1704 1705 1706 1707 1708
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
{
	struct mem_cgroup *victim = NULL;
	int total = 0;
	int loop = 0;
	unsigned long excess;
	unsigned long nr_scanned;
	struct mem_cgroup_reclaim_cookie reclaim = {
1709
		.pgdat = pgdat,
1710 1711 1712
		.priority = 0,
	};

1713
	excess = soft_limit_excess(root_memcg);
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738

	while (1) {
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
		if (!victim) {
			loop++;
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!total)
					break;
				/*
				 * We want to do more targeted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
					break;
			}
			continue;
		}
1739
		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1740
					pgdat, &nr_scanned);
1741
		*total_scanned += nr_scanned;
1742
		if (!soft_limit_excess(root_memcg))
1743
			break;
1744
	}
1745 1746
	mem_cgroup_iter_break(root_memcg, victim);
	return total;
1747 1748
}

1749 1750 1751 1752 1753 1754
#ifdef CONFIG_LOCKDEP
static struct lockdep_map memcg_oom_lock_dep_map = {
	.name = "memcg_oom_lock",
};
#endif

1755 1756
static DEFINE_SPINLOCK(memcg_oom_lock);

K
KAMEZAWA Hiroyuki 已提交
1757 1758 1759 1760
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
 */
1761
static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1762
{
1763
	struct mem_cgroup *iter, *failed = NULL;
1764

1765 1766
	spin_lock(&memcg_oom_lock);

1767
	for_each_mem_cgroup_tree(iter, memcg) {
1768
		if (iter->oom_lock) {
1769 1770 1771 1772 1773
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1774 1775
			mem_cgroup_iter_break(memcg, iter);
			break;
1776 1777
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1778
	}
K
KAMEZAWA Hiroyuki 已提交
1779

1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
	if (failed) {
		/*
		 * OK, we failed to lock the whole subtree so we have
		 * to clean up what we set up to the failing subtree
		 */
		for_each_mem_cgroup_tree(iter, memcg) {
			if (iter == failed) {
				mem_cgroup_iter_break(memcg, iter);
				break;
			}
			iter->oom_lock = false;
1791
		}
1792 1793
	} else
		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1794 1795 1796 1797

	spin_unlock(&memcg_oom_lock);

	return !failed;
1798
}
1799

1800
static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1801
{
K
KAMEZAWA Hiroyuki 已提交
1802 1803
	struct mem_cgroup *iter;

1804
	spin_lock(&memcg_oom_lock);
1805
	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1806
	for_each_mem_cgroup_tree(iter, memcg)
1807
		iter->oom_lock = false;
1808
	spin_unlock(&memcg_oom_lock);
1809 1810
}

1811
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1812 1813 1814
{
	struct mem_cgroup *iter;

1815
	spin_lock(&memcg_oom_lock);
1816
	for_each_mem_cgroup_tree(iter, memcg)
1817 1818
		iter->under_oom++;
	spin_unlock(&memcg_oom_lock);
1819 1820
}

1821
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1822 1823 1824
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1825 1826
	/*
	 * When a new child is created while the hierarchy is under oom,
1827
	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
K
KAMEZAWA Hiroyuki 已提交
1828
	 */
1829
	spin_lock(&memcg_oom_lock);
1830
	for_each_mem_cgroup_tree(iter, memcg)
1831 1832 1833
		if (iter->under_oom > 0)
			iter->under_oom--;
	spin_unlock(&memcg_oom_lock);
1834 1835
}

K
KAMEZAWA Hiroyuki 已提交
1836 1837
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1838
struct oom_wait_info {
1839
	struct mem_cgroup *memcg;
1840
	wait_queue_entry_t	wait;
K
KAMEZAWA Hiroyuki 已提交
1841 1842
};

1843
static int memcg_oom_wake_function(wait_queue_entry_t *wait,
K
KAMEZAWA Hiroyuki 已提交
1844 1845
	unsigned mode, int sync, void *arg)
{
1846 1847
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1848 1849 1850
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1851
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
1852

1853 1854
	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
K
KAMEZAWA Hiroyuki 已提交
1855 1856 1857 1858
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1859
static void memcg_oom_recover(struct mem_cgroup *memcg)
1860
{
1861 1862 1863 1864 1865 1866 1867 1868 1869
	/*
	 * For the following lockless ->under_oom test, the only required
	 * guarantee is that it must see the state asserted by an OOM when
	 * this function is called as a result of userland actions
	 * triggered by the notification of the OOM.  This is trivially
	 * achieved by invoking mem_cgroup_mark_under_oom() before
	 * triggering notification.
	 */
	if (memcg && memcg->under_oom)
1870
		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1871 1872
}

1873 1874 1875 1876 1877 1878 1879 1880
enum oom_status {
	OOM_SUCCESS,
	OOM_FAILED,
	OOM_ASYNC,
	OOM_SKIPPED
};

static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1881
{
1882 1883 1884
	enum oom_status ret;
	bool locked;

1885 1886 1887
	if (order > PAGE_ALLOC_COSTLY_ORDER)
		return OOM_SKIPPED;

1888 1889
	memcg_memory_event(memcg, MEMCG_OOM);

K
KAMEZAWA Hiroyuki 已提交
1890
	/*
1891 1892 1893 1894
	 * We are in the middle of the charge context here, so we
	 * don't want to block when potentially sitting on a callstack
	 * that holds all kinds of filesystem and mm locks.
	 *
1895 1896 1897 1898
	 * cgroup1 allows disabling the OOM killer and waiting for outside
	 * handling until the charge can succeed; remember the context and put
	 * the task to sleep at the end of the page fault when all locks are
	 * released.
1899
	 *
1900 1901 1902 1903 1904 1905 1906
	 * On the other hand, in-kernel OOM killer allows for an async victim
	 * memory reclaim (oom_reaper) and that means that we are not solely
	 * relying on the oom victim to make a forward progress and we can
	 * invoke the oom killer here.
	 *
	 * Please note that mem_cgroup_out_of_memory might fail to find a
	 * victim and then we have to bail out from the charge path.
K
KAMEZAWA Hiroyuki 已提交
1907
	 */
1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
	if (memcg->oom_kill_disable) {
		if (!current->in_user_fault)
			return OOM_SKIPPED;
		css_get(&memcg->css);
		current->memcg_in_oom = memcg;
		current->memcg_oom_gfp_mask = mask;
		current->memcg_oom_order = order;

		return OOM_ASYNC;
	}

1919 1920 1921 1922 1923 1924 1925 1926
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	mem_cgroup_unmark_under_oom(memcg);
1927
	if (mem_cgroup_out_of_memory(memcg, mask, order))
1928 1929 1930 1931 1932 1933
		ret = OOM_SUCCESS;
	else
		ret = OOM_FAILED;

	if (locked)
		mem_cgroup_oom_unlock(memcg);
1934

1935
	return ret;
1936 1937 1938 1939
}

/**
 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1940
 * @handle: actually kill/wait or just clean up the OOM state
1941
 *
1942 1943
 * This has to be called at the end of a page fault if the memcg OOM
 * handler was enabled.
1944
 *
1945
 * Memcg supports userspace OOM handling where failed allocations must
1946 1947 1948 1949
 * sleep on a waitqueue until the userspace task resolves the
 * situation.  Sleeping directly in the charge context with all kinds
 * of locks held is not a good idea, instead we remember an OOM state
 * in the task and mem_cgroup_oom_synchronize() has to be called at
1950
 * the end of the page fault to complete the OOM handling.
1951 1952
 *
 * Returns %true if an ongoing memcg OOM situation was detected and
1953
 * completed, %false otherwise.
1954
 */
1955
bool mem_cgroup_oom_synchronize(bool handle)
1956
{
T
Tejun Heo 已提交
1957
	struct mem_cgroup *memcg = current->memcg_in_oom;
1958
	struct oom_wait_info owait;
1959
	bool locked;
1960 1961 1962

	/* OOM is global, do not handle */
	if (!memcg)
1963
		return false;
1964

1965
	if (!handle)
1966
		goto cleanup;
1967 1968 1969 1970 1971

	owait.memcg = memcg;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
1972
	INIT_LIST_HEAD(&owait.wait.entry);
K
KAMEZAWA Hiroyuki 已提交
1973

1974
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	if (locked && !memcg->oom_kill_disable) {
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
T
Tejun Heo 已提交
1985 1986
		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
					 current->memcg_oom_order);
1987
	} else {
1988
		schedule();
1989 1990 1991 1992 1993
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
	}

	if (locked) {
1994 1995 1996 1997 1998 1999 2000 2001
		mem_cgroup_oom_unlock(memcg);
		/*
		 * There is no guarantee that an OOM-lock contender
		 * sees the wakeups triggered by the OOM kill
		 * uncharges.  Wake any sleepers explicitely.
		 */
		memcg_oom_recover(memcg);
	}
2002
cleanup:
T
Tejun Heo 已提交
2003
	current->memcg_in_oom = NULL;
2004
	css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2005
	return true;
2006 2007
}

2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
/**
 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
 * @victim: task to be killed by the OOM killer
 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
 *
 * Returns a pointer to a memory cgroup, which has to be cleaned up
 * by killing all belonging OOM-killable tasks.
 *
 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
 */
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
					    struct mem_cgroup *oom_domain)
{
	struct mem_cgroup *oom_group = NULL;
	struct mem_cgroup *memcg;

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return NULL;

	if (!oom_domain)
		oom_domain = root_mem_cgroup;

	rcu_read_lock();

	memcg = mem_cgroup_from_task(victim);
	if (memcg == root_mem_cgroup)
		goto out;

	/*
	 * Traverse the memory cgroup hierarchy from the victim task's
	 * cgroup up to the OOMing cgroup (or root) to find the
	 * highest-level memory cgroup with oom.group set.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		if (memcg->oom_group)
			oom_group = memcg;

		if (memcg == oom_domain)
			break;
	}

	if (oom_group)
		css_get(&oom_group->css);
out:
	rcu_read_unlock();

	return oom_group;
}

void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
{
	pr_info("Tasks in ");
	pr_cont_cgroup_path(memcg->css.cgroup);
	pr_cont(" are going to be killed due to memory.oom.group set\n");
}

2064
/**
2065 2066
 * lock_page_memcg - lock a page->mem_cgroup binding
 * @page: the page
2067
 *
2068
 * This function protects unlocked LRU pages from being moved to
2069 2070 2071 2072 2073
 * another cgroup.
 *
 * It ensures lifetime of the returned memcg. Caller is responsible
 * for the lifetime of the page; __unlock_page_memcg() is available
 * when @page might get freed inside the locked section.
2074
 */
2075
struct mem_cgroup *lock_page_memcg(struct page *page)
2076 2077
{
	struct mem_cgroup *memcg;
2078
	unsigned long flags;
2079

2080 2081 2082 2083
	/*
	 * The RCU lock is held throughout the transaction.  The fast
	 * path can get away without acquiring the memcg->move_lock
	 * because page moving starts with an RCU grace period.
2084 2085 2086 2087 2088 2089 2090
	 *
	 * The RCU lock also protects the memcg from being freed when
	 * the page state that is going to change is the only thing
	 * preventing the page itself from being freed. E.g. writeback
	 * doesn't hold a page reference and relies on PG_writeback to
	 * keep off truncation, migration and so forth.
         */
2091 2092 2093
	rcu_read_lock();

	if (mem_cgroup_disabled())
2094
		return NULL;
2095
again:
2096
	memcg = page->mem_cgroup;
2097
	if (unlikely(!memcg))
2098
		return NULL;
2099

Q
Qiang Huang 已提交
2100
	if (atomic_read(&memcg->moving_account) <= 0)
2101
		return memcg;
2102

2103
	spin_lock_irqsave(&memcg->move_lock, flags);
2104
	if (memcg != page->mem_cgroup) {
2105
		spin_unlock_irqrestore(&memcg->move_lock, flags);
2106 2107
		goto again;
	}
2108 2109 2110 2111

	/*
	 * When charge migration first begins, we can have locked and
	 * unlocked page stat updates happening concurrently.  Track
2112
	 * the task who has the lock for unlock_page_memcg().
2113 2114 2115
	 */
	memcg->move_lock_task = current;
	memcg->move_lock_flags = flags;
2116

2117
	return memcg;
2118
}
2119
EXPORT_SYMBOL(lock_page_memcg);
2120

2121
/**
2122 2123 2124 2125
 * __unlock_page_memcg - unlock and unpin a memcg
 * @memcg: the memcg
 *
 * Unlock and unpin a memcg returned by lock_page_memcg().
2126
 */
2127
void __unlock_page_memcg(struct mem_cgroup *memcg)
2128
{
2129 2130 2131 2132 2133 2134 2135 2136
	if (memcg && memcg->move_lock_task == current) {
		unsigned long flags = memcg->move_lock_flags;

		memcg->move_lock_task = NULL;
		memcg->move_lock_flags = 0;

		spin_unlock_irqrestore(&memcg->move_lock, flags);
	}
2137

2138
	rcu_read_unlock();
2139
}
2140 2141 2142 2143 2144 2145 2146 2147 2148

/**
 * unlock_page_memcg - unlock a page->mem_cgroup binding
 * @page: the page
 */
void unlock_page_memcg(struct page *page)
{
	__unlock_page_memcg(page->mem_cgroup);
}
2149
EXPORT_SYMBOL(unlock_page_memcg);
2150

2151 2152
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
2153
	unsigned int nr_pages;
2154
	struct work_struct work;
2155
	unsigned long flags;
2156
#define FLUSHING_CACHED_CHARGE	0
2157 2158
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2159
static DEFINE_MUTEX(percpu_charge_mutex);
2160

2161 2162 2163 2164 2165 2166 2167 2168 2169 2170
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
2171
 */
2172
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2173 2174
{
	struct memcg_stock_pcp *stock;
2175
	unsigned long flags;
2176
	bool ret = false;
2177

2178
	if (nr_pages > MEMCG_CHARGE_BATCH)
2179
		return ret;
2180

2181 2182 2183
	local_irq_save(flags);

	stock = this_cpu_ptr(&memcg_stock);
2184
	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
2185
		stock->nr_pages -= nr_pages;
2186 2187
		ret = true;
	}
2188 2189 2190

	local_irq_restore(flags);

2191 2192 2193 2194
	return ret;
}

/*
2195
 * Returns stocks cached in percpu and reset cached information.
2196 2197 2198 2199 2200
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2201
	if (stock->nr_pages) {
2202
		page_counter_uncharge(&old->memory, stock->nr_pages);
2203
		if (do_memsw_account())
2204
			page_counter_uncharge(&old->memsw, stock->nr_pages);
2205
		css_put_many(&old->css, stock->nr_pages);
2206
		stock->nr_pages = 0;
2207 2208 2209 2210 2211 2212
	}
	stock->cached = NULL;
}

static void drain_local_stock(struct work_struct *dummy)
{
2213 2214 2215
	struct memcg_stock_pcp *stock;
	unsigned long flags;

2216 2217 2218 2219
	/*
	 * The only protection from memory hotplug vs. drain_stock races is
	 * that we always operate on local CPU stock here with IRQ disabled
	 */
2220 2221 2222
	local_irq_save(flags);

	stock = this_cpu_ptr(&memcg_stock);
2223
	drain_stock(stock);
2224
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2225 2226

	local_irq_restore(flags);
2227 2228 2229
}

/*
2230
 * Cache charges(val) to local per_cpu area.
2231
 * This will be consumed by consume_stock() function, later.
2232
 */
2233
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2234
{
2235 2236 2237 2238
	struct memcg_stock_pcp *stock;
	unsigned long flags;

	local_irq_save(flags);
2239

2240
	stock = this_cpu_ptr(&memcg_stock);
2241
	if (stock->cached != memcg) { /* reset if necessary */
2242
		drain_stock(stock);
2243
		stock->cached = memcg;
2244
	}
2245
	stock->nr_pages += nr_pages;
2246

2247
	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2248 2249
		drain_stock(stock);

2250
	local_irq_restore(flags);
2251 2252 2253
}

/*
2254
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2255
 * of the hierarchy under it.
2256
 */
2257
static void drain_all_stock(struct mem_cgroup *root_memcg)
2258
{
2259
	int cpu, curcpu;
2260

2261 2262 2263
	/* If someone's already draining, avoid adding running more workers. */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2264 2265 2266 2267 2268 2269
	/*
	 * Notify other cpus that system-wide "drain" is running
	 * We do not care about races with the cpu hotplug because cpu down
	 * as well as workers from this path always operate on the local
	 * per-cpu data. CPU up doesn't touch memcg_stock at all.
	 */
2270
	curcpu = get_cpu();
2271 2272
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2273
		struct mem_cgroup *memcg;
2274

2275
		memcg = stock->cached;
2276
		if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
2277
			continue;
2278 2279
		if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
			css_put(&memcg->css);
2280
			continue;
2281
		}
2282 2283 2284 2285 2286 2287
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2288
		css_put(&memcg->css);
2289
	}
2290
	put_cpu();
2291
	mutex_unlock(&percpu_charge_mutex);
2292 2293
}

2294
static int memcg_hotplug_cpu_dead(unsigned int cpu)
2295 2296
{
	struct memcg_stock_pcp *stock;
2297
	struct mem_cgroup *memcg, *mi;
2298 2299 2300

	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
2301 2302 2303 2304 2305 2306 2307 2308

	for_each_mem_cgroup(memcg) {
		int i;

		for (i = 0; i < MEMCG_NR_STAT; i++) {
			int nid;
			long x;

2309
			x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
2310
			if (x)
2311 2312
				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
					atomic_long_add(x, &memcg->vmstats[i]);
2313 2314 2315 2316 2317 2318 2319 2320 2321

			if (i >= NR_VM_NODE_STAT_ITEMS)
				continue;

			for_each_node(nid) {
				struct mem_cgroup_per_node *pn;

				pn = mem_cgroup_nodeinfo(memcg, nid);
				x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
2322
				if (x)
2323 2324 2325
					do {
						atomic_long_add(x, &pn->lruvec_stat[i]);
					} while ((pn = parent_nodeinfo(pn, nid)));
2326 2327 2328
			}
		}

2329
		for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2330 2331
			long x;

2332
			x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
2333
			if (x)
2334 2335
				for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
					atomic_long_add(x, &memcg->vmevents[i]);
2336 2337 2338
		}
	}

2339
	return 0;
2340 2341
}

2342 2343 2344 2345 2346 2347 2348
static void reclaim_high(struct mem_cgroup *memcg,
			 unsigned int nr_pages,
			 gfp_t gfp_mask)
{
	do {
		if (page_counter_read(&memcg->memory) <= memcg->high)
			continue;
2349
		memcg_memory_event(memcg, MEMCG_HIGH);
2350 2351 2352 2353 2354 2355 2356 2357 2358
		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
	} while ((memcg = parent_mem_cgroup(memcg)));
}

static void high_work_func(struct work_struct *work)
{
	struct mem_cgroup *memcg;

	memcg = container_of(work, struct mem_cgroup, high_work);
2359
	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2360 2361
}

2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414
/*
 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
 * enough to still cause a significant slowdown in most cases, while still
 * allowing diagnostics and tracing to proceed without becoming stuck.
 */
#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)

/*
 * When calculating the delay, we use these either side of the exponentiation to
 * maintain precision and scale to a reasonable number of jiffies (see the table
 * below.
 *
 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
 *   overage ratio to a delay.
 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down down the
 *   proposed penalty in order to reduce to a reasonable number of jiffies, and
 *   to produce a reasonable delay curve.
 *
 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
 * reasonable delay curve compared to precision-adjusted overage, not
 * penalising heavily at first, but still making sure that growth beyond the
 * limit penalises misbehaviour cgroups by slowing them down exponentially. For
 * example, with a high of 100 megabytes:
 *
 *  +-------+------------------------+
 *  | usage | time to allocate in ms |
 *  +-------+------------------------+
 *  | 100M  |                      0 |
 *  | 101M  |                      6 |
 *  | 102M  |                     25 |
 *  | 103M  |                     57 |
 *  | 104M  |                    102 |
 *  | 105M  |                    159 |
 *  | 106M  |                    230 |
 *  | 107M  |                    313 |
 *  | 108M  |                    409 |
 *  | 109M  |                    518 |
 *  | 110M  |                    639 |
 *  | 111M  |                    774 |
 *  | 112M  |                    921 |
 *  | 113M  |                   1081 |
 *  | 114M  |                   1254 |
 *  | 115M  |                   1439 |
 *  | 116M  |                   1638 |
 *  | 117M  |                   1849 |
 *  | 118M  |                   2000 |
 *  | 119M  |                   2000 |
 *  | 120M  |                   2000 |
 *  +-------+------------------------+
 */
 #define MEMCG_DELAY_PRECISION_SHIFT 20
 #define MEMCG_DELAY_SCALING_SHIFT 14

2415 2416 2417 2418 2419 2420
/*
 * Scheduled by try_charge() to be executed from the userland return path
 * and reclaims memory over the high limit.
 */
void mem_cgroup_handle_over_high(void)
{
2421 2422 2423
	unsigned long usage, high, clamped_high;
	unsigned long pflags;
	unsigned long penalty_jiffies, overage;
2424
	unsigned int nr_pages = current->memcg_nr_pages_over_high;
2425
	struct mem_cgroup *memcg;
2426 2427 2428 2429

	if (likely(!nr_pages))
		return;

2430 2431
	memcg = get_mem_cgroup_from_mm(current->mm);
	reclaim_high(memcg, nr_pages, GFP_KERNEL);
2432
	current->memcg_nr_pages_over_high = 0;
2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500

	/*
	 * memory.high is breached and reclaim is unable to keep up. Throttle
	 * allocators proactively to slow down excessive growth.
	 *
	 * We use overage compared to memory.high to calculate the number of
	 * jiffies to sleep (penalty_jiffies). Ideally this value should be
	 * fairly lenient on small overages, and increasingly harsh when the
	 * memcg in question makes it clear that it has no intention of stopping
	 * its crazy behaviour, so we exponentially increase the delay based on
	 * overage amount.
	 */

	usage = page_counter_read(&memcg->memory);
	high = READ_ONCE(memcg->high);

	if (usage <= high)
		goto out;

	/*
	 * Prevent division by 0 in overage calculation by acting as if it was a
	 * threshold of 1 page
	 */
	clamped_high = max(high, 1UL);

	overage = div_u64((u64)(usage - high) << MEMCG_DELAY_PRECISION_SHIFT,
			  clamped_high);

	penalty_jiffies = ((u64)overage * overage * HZ)
		>> (MEMCG_DELAY_PRECISION_SHIFT + MEMCG_DELAY_SCALING_SHIFT);

	/*
	 * Factor in the task's own contribution to the overage, such that four
	 * N-sized allocations are throttled approximately the same as one
	 * 4N-sized allocation.
	 *
	 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
	 * larger the current charge patch is than that.
	 */
	penalty_jiffies = penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;

	/*
	 * Clamp the max delay per usermode return so as to still keep the
	 * application moving forwards and also permit diagnostics, albeit
	 * extremely slowly.
	 */
	penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);

	/*
	 * Don't sleep if the amount of jiffies this memcg owes us is so low
	 * that it's not even worth doing, in an attempt to be nice to those who
	 * go only a small amount over their memory.high value and maybe haven't
	 * been aggressively reclaimed enough yet.
	 */
	if (penalty_jiffies <= HZ / 100)
		goto out;

	/*
	 * If we exit early, we're guaranteed to die (since
	 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
	 * need to account for any ill-begotten jiffies to pay them off later.
	 */
	psi_memstall_enter(&pflags);
	schedule_timeout_killable(penalty_jiffies);
	psi_memstall_leave(&pflags);

out:
	css_put(&memcg->css);
2501 2502
}

2503 2504
static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
		      unsigned int nr_pages)
2505
{
2506
	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2507
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2508
	struct mem_cgroup *mem_over_limit;
2509
	struct page_counter *counter;
2510
	unsigned long nr_reclaimed;
2511 2512
	bool may_swap = true;
	bool drained = false;
2513
	enum oom_status oom_status;
2514

2515
	if (mem_cgroup_is_root(memcg))
2516
		return 0;
2517
retry:
2518
	if (consume_stock(memcg, nr_pages))
2519
		return 0;
2520

2521
	if (!do_memsw_account() ||
2522 2523
	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
		if (page_counter_try_charge(&memcg->memory, batch, &counter))
2524
			goto done_restock;
2525
		if (do_memsw_account())
2526 2527
			page_counter_uncharge(&memcg->memsw, batch);
		mem_over_limit = mem_cgroup_from_counter(counter, memory);
2528
	} else {
2529
		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2530
		may_swap = false;
2531
	}
2532

2533 2534 2535 2536
	if (batch > nr_pages) {
		batch = nr_pages;
		goto retry;
	}
2537

2538 2539 2540 2541 2542 2543
	/*
	 * Unlike in global OOM situations, memcg is not in a physical
	 * memory shortage.  Allow dying and OOM-killed tasks to
	 * bypass the last charges so that they can exit quickly and
	 * free their memory.
	 */
2544
	if (unlikely(should_force_charge()))
2545
		goto force;
2546

2547 2548 2549 2550 2551 2552 2553 2554 2555
	/*
	 * Prevent unbounded recursion when reclaim operations need to
	 * allocate memory. This might exceed the limits temporarily,
	 * but we prefer facilitating memory reclaim and getting back
	 * under the limit over triggering OOM kills in these cases.
	 */
	if (unlikely(current->flags & PF_MEMALLOC))
		goto force;

2556 2557 2558
	if (unlikely(task_in_memcg_oom(current)))
		goto nomem;

2559
	if (!gfpflags_allow_blocking(gfp_mask))
2560
		goto nomem;
2561

2562
	memcg_memory_event(mem_over_limit, MEMCG_MAX);
2563

2564 2565
	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
						    gfp_mask, may_swap);
2566

2567
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2568
		goto retry;
2569

2570
	if (!drained) {
2571
		drain_all_stock(mem_over_limit);
2572 2573 2574 2575
		drained = true;
		goto retry;
	}

2576 2577
	if (gfp_mask & __GFP_NORETRY)
		goto nomem;
2578 2579 2580 2581 2582 2583 2584 2585 2586
	/*
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
	 */
2587
	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2588 2589 2590 2591 2592 2593 2594 2595
		goto retry;
	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		goto retry;

2596 2597 2598
	if (nr_retries--)
		goto retry;

2599
	if (gfp_mask & __GFP_RETRY_MAYFAIL)
2600 2601
		goto nomem;

2602
	if (gfp_mask & __GFP_NOFAIL)
2603
		goto force;
2604

2605
	if (fatal_signal_pending(current))
2606
		goto force;
2607

2608 2609 2610 2611 2612 2613
	/*
	 * keep retrying as long as the memcg oom killer is able to make
	 * a forward progress or bypass the charge if the oom killer
	 * couldn't make any progress.
	 */
	oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2614
		       get_order(nr_pages * PAGE_SIZE));
2615 2616 2617 2618 2619 2620 2621 2622 2623
	switch (oom_status) {
	case OOM_SUCCESS:
		nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
		goto retry;
	case OOM_FAILED:
		goto force;
	default:
		goto nomem;
	}
2624
nomem:
2625
	if (!(gfp_mask & __GFP_NOFAIL))
2626
		return -ENOMEM;
2627 2628 2629 2630 2631 2632 2633
force:
	/*
	 * The allocation either can't fail or will lead to more memory
	 * being freed very soon.  Allow memory usage go over the limit
	 * temporarily by force charging it.
	 */
	page_counter_charge(&memcg->memory, nr_pages);
2634
	if (do_memsw_account())
2635 2636 2637 2638
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);

	return 0;
2639 2640

done_restock:
2641
	css_get_many(&memcg->css, batch);
2642 2643
	if (batch > nr_pages)
		refill_stock(memcg, batch - nr_pages);
2644

2645
	/*
2646 2647
	 * If the hierarchy is above the normal consumption range, schedule
	 * reclaim on returning to userland.  We can perform reclaim here
2648
	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2649 2650 2651 2652
	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
	 * not recorded as it most likely matches current's and won't
	 * change in the meantime.  As high limit is checked again before
	 * reclaim, the cost of mismatch is negligible.
2653 2654
	 */
	do {
2655
		if (page_counter_read(&memcg->memory) > memcg->high) {
2656 2657 2658 2659 2660
			/* Don't bother a random interrupted task */
			if (in_interrupt()) {
				schedule_work(&memcg->high_work);
				break;
			}
V
Vladimir Davydov 已提交
2661
			current->memcg_nr_pages_over_high += batch;
2662 2663 2664
			set_notify_resume(current);
			break;
		}
2665
	} while ((memcg = parent_mem_cgroup(memcg)));
2666 2667

	return 0;
2668
}
2669

2670
static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2671
{
2672 2673 2674
	if (mem_cgroup_is_root(memcg))
		return;

2675
	page_counter_uncharge(&memcg->memory, nr_pages);
2676
	if (do_memsw_account())
2677
		page_counter_uncharge(&memcg->memsw, nr_pages);
2678

2679
	css_put_many(&memcg->css, nr_pages);
2680 2681
}

2682 2683
static void lock_page_lru(struct page *page, int *isolated)
{
2684
	pg_data_t *pgdat = page_pgdat(page);
2685

2686
	spin_lock_irq(&pgdat->lru_lock);
2687 2688 2689
	if (PageLRU(page)) {
		struct lruvec *lruvec;

2690
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
2691 2692 2693 2694 2695 2696 2697 2698 2699
		ClearPageLRU(page);
		del_page_from_lru_list(page, lruvec, page_lru(page));
		*isolated = 1;
	} else
		*isolated = 0;
}

static void unlock_page_lru(struct page *page, int isolated)
{
2700
	pg_data_t *pgdat = page_pgdat(page);
2701 2702 2703 2704

	if (isolated) {
		struct lruvec *lruvec;

2705
		lruvec = mem_cgroup_page_lruvec(page, pgdat);
2706 2707 2708 2709
		VM_BUG_ON_PAGE(PageLRU(page), page);
		SetPageLRU(page);
		add_page_to_lru_list(page, lruvec, page_lru(page));
	}
2710
	spin_unlock_irq(&pgdat->lru_lock);
2711 2712
}

2713
static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2714
			  bool lrucare)
2715
{
2716
	int isolated;
2717

2718
	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2719 2720 2721 2722 2723

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
2724 2725
	if (lrucare)
		lock_page_lru(page, &isolated);
2726

2727 2728
	/*
	 * Nobody should be changing or seriously looking at
2729
	 * page->mem_cgroup at this point:
2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740
	 *
	 * - the page is uncharged
	 *
	 * - the page is off-LRU
	 *
	 * - an anonymous fault has exclusive page access, except for
	 *   a locked page table
	 *
	 * - a page cache insertion, a swapin fault, or a migration
	 *   have the page locked
	 */
2741
	page->mem_cgroup = memcg;
2742

2743 2744
	if (lrucare)
		unlock_page_lru(page, isolated);
2745
}
2746

2747
#ifdef CONFIG_MEMCG_KMEM
2748
static int memcg_alloc_cache_id(void)
2749
{
2750 2751 2752
	int id, size;
	int err;

2753
	id = ida_simple_get(&memcg_cache_ida,
2754 2755 2756
			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (id < 0)
		return id;
2757

2758
	if (id < memcg_nr_cache_ids)
2759 2760 2761 2762 2763 2764
		return id;

	/*
	 * There's no space for the new id in memcg_caches arrays,
	 * so we have to grow them.
	 */
2765
	down_write(&memcg_cache_ids_sem);
2766 2767

	size = 2 * (id + 1);
2768 2769 2770 2771 2772
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

2773
	err = memcg_update_all_caches(size);
2774 2775
	if (!err)
		err = memcg_update_all_list_lrus(size);
2776 2777 2778 2779 2780
	if (!err)
		memcg_nr_cache_ids = size;

	up_write(&memcg_cache_ids_sem);

2781
	if (err) {
2782
		ida_simple_remove(&memcg_cache_ida, id);
2783 2784 2785 2786 2787 2788 2789
		return err;
	}
	return id;
}

static void memcg_free_cache_id(int id)
{
2790
	ida_simple_remove(&memcg_cache_ida, id);
2791 2792
}

2793
struct memcg_kmem_cache_create_work {
2794 2795 2796 2797 2798
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

2799
static void memcg_kmem_cache_create_func(struct work_struct *w)
2800
{
2801 2802
	struct memcg_kmem_cache_create_work *cw =
		container_of(w, struct memcg_kmem_cache_create_work, work);
2803 2804
	struct mem_cgroup *memcg = cw->memcg;
	struct kmem_cache *cachep = cw->cachep;
2805

2806
	memcg_create_kmem_cache(memcg, cachep);
2807

2808
	css_put(&memcg->css);
2809 2810 2811 2812 2813 2814
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 */
2815
static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2816
					       struct kmem_cache *cachep)
2817
{
2818
	struct memcg_kmem_cache_create_work *cw;
2819

2820 2821 2822
	if (!css_tryget_online(&memcg->css))
		return;

2823
	cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2824
	if (!cw)
2825
		return;
2826

2827 2828
	cw->memcg = memcg;
	cw->cachep = cachep;
2829
	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2830

2831
	queue_work(memcg_kmem_cache_wq, &cw->work);
2832 2833
}

2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844
static inline bool memcg_kmem_bypass(void)
{
	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
		return true;
	return false;
}

/**
 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
 * @cachep: the original global kmem cache
 *
2845 2846 2847
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
2848 2849 2850
 * If the cache does not exist yet, if we are the first user of it, we
 * create it asynchronously in a workqueue and let the current allocation
 * go through with the original cache.
2851
 *
2852 2853 2854 2855
 * This function takes a reference to the cache it returns to assure it
 * won't get destroyed while we are working with it. Once the caller is
 * done with it, memcg_kmem_put_cache() must be called to release the
 * reference.
2856
 */
2857
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2858 2859
{
	struct mem_cgroup *memcg;
2860
	struct kmem_cache *memcg_cachep;
2861
	struct memcg_cache_array *arr;
2862
	int kmemcg_id;
2863

2864
	VM_BUG_ON(!is_root_cache(cachep));
2865

2866
	if (memcg_kmem_bypass())
V
Vladimir Davydov 已提交
2867 2868
		return cachep;

2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
	rcu_read_lock();

	if (unlikely(current->active_memcg))
		memcg = current->active_memcg;
	else
		memcg = mem_cgroup_from_task(current);

	if (!memcg || memcg == root_mem_cgroup)
		goto out_unlock;

2879
	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2880
	if (kmemcg_id < 0)
2881
		goto out_unlock;
2882

2883 2884 2885 2886 2887 2888 2889 2890
	arr = rcu_dereference(cachep->memcg_params.memcg_caches);

	/*
	 * Make sure we will access the up-to-date value. The code updating
	 * memcg_caches issues a write barrier to match the data dependency
	 * barrier inside READ_ONCE() (see memcg_create_kmem_cache()).
	 */
	memcg_cachep = READ_ONCE(arr->entries[kmemcg_id]);
2891 2892 2893 2894 2895 2896 2897 2898 2899

	/*
	 * If we are in a safe context (can wait, and not in interrupt
	 * context), we could be be predictable and return right away.
	 * This would guarantee that the allocation being performed
	 * already belongs in the new cache.
	 *
	 * However, there are some clashes that can arrive from locking.
	 * For instance, because we acquire the slab_mutex while doing
2900 2901 2902
	 * memcg_create_kmem_cache, this means no further allocation
	 * could happen with the slab_mutex held. So it's better to
	 * defer everything.
2903 2904 2905 2906 2907 2908 2909
	 *
	 * If the memcg is dying or memcg_cache is about to be released,
	 * don't bother creating new kmem_caches. Because memcg_cachep
	 * is ZEROed as the fist step of kmem offlining, we don't need
	 * percpu_ref_tryget_live() here. css_tryget_online() check in
	 * memcg_schedule_kmem_cache_create() will prevent us from
	 * creation of a new kmem_cache.
2910
	 */
2911 2912 2913 2914 2915 2916
	if (unlikely(!memcg_cachep))
		memcg_schedule_kmem_cache_create(memcg, cachep);
	else if (percpu_ref_tryget(&memcg_cachep->memcg_params.refcnt))
		cachep = memcg_cachep;
out_unlock:
	rcu_read_unlock();
2917
	return cachep;
2918 2919
}

2920 2921 2922 2923 2924
/**
 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
 * @cachep: the cache returned by memcg_kmem_get_cache
 */
void memcg_kmem_put_cache(struct kmem_cache *cachep)
2925 2926
{
	if (!is_root_cache(cachep))
2927
		percpu_ref_put(&cachep->memcg_params.refcnt);
2928 2929
}

2930
/**
2931
 * __memcg_kmem_charge_memcg: charge a kmem page
2932 2933 2934 2935 2936 2937 2938
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
 * @memcg: memory cgroup to charge
 *
 * Returns 0 on success, an error code on failure.
 */
2939
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2940
			    struct mem_cgroup *memcg)
2941
{
2942 2943
	unsigned int nr_pages = 1 << order;
	struct page_counter *counter;
2944 2945
	int ret;

2946
	ret = try_charge(memcg, gfp, nr_pages);
2947
	if (ret)
2948
		return ret;
2949 2950 2951 2952 2953

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
		cancel_charge(memcg, nr_pages);
		return -ENOMEM;
2954
	}
2955
	return 0;
2956 2957
}

2958
/**
2959
 * __memcg_kmem_charge: charge a kmem page to the current memory cgroup
2960 2961 2962 2963 2964 2965
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
 *
 * Returns 0 on success, an error code on failure.
 */
2966
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2967
{
2968
	struct mem_cgroup *memcg;
2969
	int ret = 0;
2970

2971
	if (memcg_kmem_bypass())
2972 2973
		return 0;

2974
	memcg = get_mem_cgroup_from_current();
2975
	if (!mem_cgroup_is_root(memcg)) {
2976
		ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
2977 2978
		if (!ret) {
			page->mem_cgroup = memcg;
2979
			__SetPageKmemcg(page);
2980
		}
2981
	}
2982
	css_put(&memcg->css);
2983
	return ret;
2984
}
2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000

/**
 * __memcg_kmem_uncharge_memcg: uncharge a kmem page
 * @memcg: memcg to uncharge
 * @nr_pages: number of pages to uncharge
 */
void __memcg_kmem_uncharge_memcg(struct mem_cgroup *memcg,
				 unsigned int nr_pages)
{
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
		page_counter_uncharge(&memcg->kmem, nr_pages);

	page_counter_uncharge(&memcg->memory, nr_pages);
	if (do_memsw_account())
		page_counter_uncharge(&memcg->memsw, nr_pages);
}
3001
/**
3002
 * __memcg_kmem_uncharge: uncharge a kmem page
3003 3004 3005
 * @page: page to uncharge
 * @order: allocation order
 */
3006
void __memcg_kmem_uncharge(struct page *page, int order)
3007
{
3008
	struct mem_cgroup *memcg = page->mem_cgroup;
3009
	unsigned int nr_pages = 1 << order;
3010 3011 3012 3013

	if (!memcg)
		return;

3014
	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
3015
	__memcg_kmem_uncharge_memcg(memcg, nr_pages);
3016
	page->mem_cgroup = NULL;
3017 3018 3019 3020 3021

	/* slab pages do not have PageKmemcg flag set */
	if (PageKmemcg(page))
		__ClearPageKmemcg(page);

3022
	css_put_many(&memcg->css, nr_pages);
3023
}
3024
#endif /* CONFIG_MEMCG_KMEM */
3025

3026 3027 3028 3029
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

/*
 * Because tail pages are not marked as "used", set it. We're under
3030
 * pgdat->lru_lock and migration entries setup in all page mappings.
3031
 */
3032
void mem_cgroup_split_huge_fixup(struct page *head)
3033
{
3034
	int i;
3035

3036 3037
	if (mem_cgroup_disabled())
		return;
3038

3039
	for (i = 1; i < HPAGE_PMD_NR; i++)
3040
		head[i].mem_cgroup = head->mem_cgroup;
3041

3042
	__mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
3043
}
3044
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3045

A
Andrew Morton 已提交
3046
#ifdef CONFIG_MEMCG_SWAP
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
3058
 * The caller must have charged to @to, IOW, called page_counter_charge() about
3059 3060 3061
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
3062
				struct mem_cgroup *from, struct mem_cgroup *to)
3063 3064 3065
{
	unsigned short old_id, new_id;

L
Li Zefan 已提交
3066 3067
	old_id = mem_cgroup_id(from);
	new_id = mem_cgroup_id(to);
3068 3069

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
3070 3071
		mod_memcg_state(from, MEMCG_SWAP, -1);
		mod_memcg_state(to, MEMCG_SWAP, 1);
3072 3073 3074 3075 3076 3077
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
3078
				struct mem_cgroup *from, struct mem_cgroup *to)
3079 3080 3081
{
	return -EINVAL;
}
3082
#endif
K
KAMEZAWA Hiroyuki 已提交
3083

3084
static DEFINE_MUTEX(memcg_max_mutex);
3085

3086 3087
static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
				 unsigned long max, bool memsw)
3088
{
3089
	bool enlarge = false;
3090
	bool drained = false;
3091
	int ret;
3092 3093
	bool limits_invariant;
	struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
3094

3095
	do {
3096 3097 3098 3099
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
3100

3101
		mutex_lock(&memcg_max_mutex);
3102 3103
		/*
		 * Make sure that the new limit (memsw or memory limit) doesn't
3104
		 * break our basic invariant rule memory.max <= memsw.max.
3105
		 */
3106 3107
		limits_invariant = memsw ? max >= memcg->memory.max :
					   max <= memcg->memsw.max;
3108
		if (!limits_invariant) {
3109
			mutex_unlock(&memcg_max_mutex);
3110 3111 3112
			ret = -EINVAL;
			break;
		}
3113
		if (max > counter->max)
3114
			enlarge = true;
3115 3116
		ret = page_counter_set_max(counter, max);
		mutex_unlock(&memcg_max_mutex);
3117 3118 3119 3120

		if (!ret)
			break;

3121 3122 3123 3124 3125 3126
		if (!drained) {
			drain_all_stock(memcg);
			drained = true;
			continue;
		}

3127 3128 3129 3130 3131 3132
		if (!try_to_free_mem_cgroup_pages(memcg, 1,
					GFP_KERNEL, !memsw)) {
			ret = -EBUSY;
			break;
		}
	} while (true);
3133

3134 3135
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
3136

3137 3138 3139
	return ret;
}

3140
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
3141 3142 3143 3144
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
{
	unsigned long nr_reclaimed = 0;
3145
	struct mem_cgroup_per_node *mz, *next_mz = NULL;
3146 3147
	unsigned long reclaimed;
	int loop = 0;
3148
	struct mem_cgroup_tree_per_node *mctz;
3149
	unsigned long excess;
3150 3151 3152 3153 3154
	unsigned long nr_scanned;

	if (order > 0)
		return 0;

3155
	mctz = soft_limit_tree_node(pgdat->node_id);
3156 3157 3158 3159 3160 3161

	/*
	 * Do not even bother to check the largest node if the root
	 * is empty. Do it lockless to prevent lock bouncing. Races
	 * are acceptable as soft limit is best effort anyway.
	 */
3162
	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
3163 3164
		return 0;

3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		nr_scanned = 0;
3179
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
3180 3181 3182
						    gfp_mask, &nr_scanned);
		nr_reclaimed += reclaimed;
		*total_scanned += nr_scanned;
3183
		spin_lock_irq(&mctz->lock);
3184
		__mem_cgroup_remove_exceeded(mz, mctz);
3185 3186 3187 3188 3189 3190

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
3191 3192 3193
		if (!reclaimed)
			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);

3194
		excess = soft_limit_excess(mz->memcg);
3195 3196 3197 3198 3199 3200 3201 3202 3203
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
		/* If excess == 0, no tree ops */
3204
		__mem_cgroup_insert_exceeded(mz, mctz, excess);
3205
		spin_unlock_irq(&mctz->lock);
3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222
		css_put(&mz->memcg->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->memcg->css);
	return nr_reclaimed;
}

3223 3224 3225 3226 3227 3228
/*
 * Test whether @memcg has children, dead or alive.  Note that this
 * function doesn't care whether @memcg has use_hierarchy enabled and
 * returns %true if there are child csses according to the cgroup
 * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
 */
3229 3230
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
3231 3232 3233 3234 3235 3236
	bool ret;

	rcu_read_lock();
	ret = css_next_child(NULL, &memcg->css);
	rcu_read_unlock();
	return ret;
3237 3238
}

3239
/*
3240
 * Reclaims as many pages from the given memcg as possible.
3241 3242 3243 3244 3245 3246 3247
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;

3248 3249
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
3250 3251 3252

	drain_all_stock(memcg);

3253
	/* try to free all pages in this cgroup */
3254
	while (nr_retries && page_counter_read(&memcg->memory)) {
3255
		int progress;
3256

3257 3258 3259
		if (signal_pending(current))
			return -EINTR;

3260 3261
		progress = try_to_free_mem_cgroup_pages(memcg, 1,
							GFP_KERNEL, true);
3262
		if (!progress) {
3263
			nr_retries--;
3264
			/* maybe some writeback is necessary */
3265
			congestion_wait(BLK_RW_ASYNC, HZ/10);
3266
		}
3267 3268

	}
3269 3270

	return 0;
3271 3272
}

3273 3274 3275
static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
3276
{
3277
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3278

3279 3280
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
3281
	return mem_cgroup_force_empty(memcg) ?: nbytes;
3282 3283
}

3284 3285
static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
				     struct cftype *cft)
3286
{
3287
	return mem_cgroup_from_css(css)->use_hierarchy;
3288 3289
}

3290 3291
static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, u64 val)
3292 3293
{
	int retval = 0;
3294
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
3295
	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
3296

3297
	if (memcg->use_hierarchy == val)
3298
		return 0;
3299

3300
	/*
3301
	 * If parent's use_hierarchy is set, we can't make any modifications
3302 3303 3304 3305 3306 3307
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
3308
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
3309
				(val == 1 || val == 0)) {
3310
		if (!memcg_has_children(memcg))
3311
			memcg->use_hierarchy = val;
3312 3313 3314 3315
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
3316

3317 3318 3319
	return retval;
}

3320
static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
3321
{
3322
	unsigned long val;
3323

3324
	if (mem_cgroup_is_root(memcg)) {
3325 3326 3327 3328
		val = memcg_page_state(memcg, MEMCG_CACHE) +
			memcg_page_state(memcg, MEMCG_RSS);
		if (swap)
			val += memcg_page_state(memcg, MEMCG_SWAP);
3329
	} else {
3330
		if (!swap)
3331
			val = page_counter_read(&memcg->memory);
3332
		else
3333
			val = page_counter_read(&memcg->memsw);
3334
	}
3335
	return val;
3336 3337
}

3338 3339 3340 3341 3342 3343 3344
enum {
	RES_USAGE,
	RES_LIMIT,
	RES_MAX_USAGE,
	RES_FAILCNT,
	RES_SOFT_LIMIT,
};
3345

3346
static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3347
			       struct cftype *cft)
B
Balbir Singh 已提交
3348
{
3349
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3350
	struct page_counter *counter;
3351

3352
	switch (MEMFILE_TYPE(cft->private)) {
3353
	case _MEM:
3354 3355
		counter = &memcg->memory;
		break;
3356
	case _MEMSWAP:
3357 3358
		counter = &memcg->memsw;
		break;
3359
	case _KMEM:
3360
		counter = &memcg->kmem;
3361
		break;
V
Vladimir Davydov 已提交
3362
	case _TCP:
3363
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
3364
		break;
3365 3366 3367
	default:
		BUG();
	}
3368 3369 3370 3371

	switch (MEMFILE_ATTR(cft->private)) {
	case RES_USAGE:
		if (counter == &memcg->memory)
3372
			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3373
		if (counter == &memcg->memsw)
3374
			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3375 3376
		return (u64)page_counter_read(counter) * PAGE_SIZE;
	case RES_LIMIT:
3377
		return (u64)counter->max * PAGE_SIZE;
3378 3379 3380 3381 3382 3383 3384 3385 3386
	case RES_MAX_USAGE:
		return (u64)counter->watermark * PAGE_SIZE;
	case RES_FAILCNT:
		return counter->failcnt;
	case RES_SOFT_LIMIT:
		return (u64)memcg->soft_limit * PAGE_SIZE;
	default:
		BUG();
	}
B
Balbir Singh 已提交
3387
}
3388

3389
static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg, bool slab_only)
3390 3391 3392 3393
{
	unsigned long stat[MEMCG_NR_STAT];
	struct mem_cgroup *mi;
	int node, cpu, i;
3394
	int min_idx, max_idx;
3395

3396 3397 3398 3399 3400 3401 3402 3403 3404
	if (slab_only) {
		min_idx = NR_SLAB_RECLAIMABLE;
		max_idx = NR_SLAB_UNRECLAIMABLE;
	} else {
		min_idx = 0;
		max_idx = MEMCG_NR_STAT;
	}

	for (i = min_idx; i < max_idx; i++)
3405 3406 3407
		stat[i] = 0;

	for_each_online_cpu(cpu)
3408
		for (i = min_idx; i < max_idx; i++)
3409
			stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
3410 3411

	for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3412
		for (i = min_idx; i < max_idx; i++)
3413 3414
			atomic_long_add(stat[i], &mi->vmstats[i]);

3415 3416 3417
	if (!slab_only)
		max_idx = NR_VM_NODE_STAT_ITEMS;

3418 3419 3420 3421
	for_each_node(node) {
		struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
		struct mem_cgroup_per_node *pi;

3422
		for (i = min_idx; i < max_idx; i++)
3423 3424 3425
			stat[i] = 0;

		for_each_online_cpu(cpu)
3426
			for (i = min_idx; i < max_idx; i++)
3427 3428
				stat[i] += per_cpu(
					pn->lruvec_stat_cpu->count[i], cpu);
3429 3430

		for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
3431
			for (i = min_idx; i < max_idx; i++)
3432 3433 3434 3435
				atomic_long_add(stat[i], &pi->lruvec_stat[i]);
	}
}

3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446
static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct mem_cgroup *mi;
	int cpu, i;

	for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
		events[i] = 0;

	for_each_online_cpu(cpu)
		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3447 3448
			events[i] += per_cpu(memcg->vmstats_percpu->events[i],
					     cpu);
3449 3450 3451 3452 3453 3454

	for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
		for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
			atomic_long_add(events[i], &mi->vmevents[i]);
}

3455
#ifdef CONFIG_MEMCG_KMEM
3456
static int memcg_online_kmem(struct mem_cgroup *memcg)
3457 3458 3459
{
	int memcg_id;

3460 3461 3462
	if (cgroup_memory_nokmem)
		return 0;

3463
	BUG_ON(memcg->kmemcg_id >= 0);
3464
	BUG_ON(memcg->kmem_state);
3465

3466
	memcg_id = memcg_alloc_cache_id();
3467 3468
	if (memcg_id < 0)
		return memcg_id;
3469

3470
	static_branch_inc(&memcg_kmem_enabled_key);
3471
	/*
3472
	 * A memory cgroup is considered kmem-online as soon as it gets
V
Vladimir Davydov 已提交
3473
	 * kmemcg_id. Setting the id after enabling static branching will
3474 3475 3476
	 * guarantee no one starts accounting before all call sites are
	 * patched.
	 */
V
Vladimir Davydov 已提交
3477
	memcg->kmemcg_id = memcg_id;
3478
	memcg->kmem_state = KMEM_ONLINE;
3479
	INIT_LIST_HEAD(&memcg->kmem_caches);
3480 3481

	return 0;
3482 3483
}

3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
	struct cgroup_subsys_state *css;
	struct mem_cgroup *parent, *child;
	int kmemcg_id;

	if (memcg->kmem_state != KMEM_ONLINE)
		return;
	/*
	 * Clear the online state before clearing memcg_caches array
	 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
	 * guarantees that no cache will be created for this cgroup
	 * after we are done (see memcg_create_kmem_cache()).
	 */
	memcg->kmem_state = KMEM_ALLOCATED;

	parent = parent_mem_cgroup(memcg);
	if (!parent)
		parent = root_mem_cgroup;

3504 3505 3506 3507 3508 3509
	/*
	 * Deactivate and reparent kmem_caches. Then flush percpu
	 * slab statistics to have precise values at the parent and
	 * all ancestor levels. It's required to keep slab stats
	 * accurate after the reparenting of kmem_caches.
	 */
3510
	memcg_deactivate_kmem_caches(memcg, parent);
3511
	memcg_flush_percpu_vmstats(memcg, true);
3512 3513 3514 3515

	kmemcg_id = memcg->kmemcg_id;
	BUG_ON(kmemcg_id < 0);

3516 3517 3518 3519 3520 3521 3522 3523
	/*
	 * Change kmemcg_id of this cgroup and all its descendants to the
	 * parent's id, and then move all entries from this cgroup's list_lrus
	 * to ones of the parent. After we have finished, all list_lrus
	 * corresponding to this cgroup are guaranteed to remain empty. The
	 * ordering is imposed by list_lru_node->lock taken by
	 * memcg_drain_all_list_lrus().
	 */
3524
	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3525 3526 3527 3528 3529 3530 3531
	css_for_each_descendant_pre(css, &memcg->css) {
		child = mem_cgroup_from_css(css);
		BUG_ON(child->kmemcg_id != kmemcg_id);
		child->kmemcg_id = parent->kmemcg_id;
		if (!memcg->use_hierarchy)
			break;
	}
3532 3533
	rcu_read_unlock();

3534
	memcg_drain_all_list_lrus(kmemcg_id, parent);
3535 3536 3537 3538 3539 3540

	memcg_free_cache_id(kmemcg_id);
}

static void memcg_free_kmem(struct mem_cgroup *memcg)
{
3541 3542 3543 3544
	/* css_alloc() failed, offlining didn't happen */
	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
		memcg_offline_kmem(memcg);

3545
	if (memcg->kmem_state == KMEM_ALLOCATED) {
3546
		WARN_ON(!list_empty(&memcg->kmem_caches));
3547 3548 3549
		static_branch_dec(&memcg_kmem_enabled_key);
	}
}
3550
#else
3551
static int memcg_online_kmem(struct mem_cgroup *memcg)
3552 3553 3554 3555 3556 3557 3558 3559 3560
{
	return 0;
}
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
}
static void memcg_free_kmem(struct mem_cgroup *memcg)
{
}
3561
#endif /* CONFIG_MEMCG_KMEM */
3562

3563 3564
static int memcg_update_kmem_max(struct mem_cgroup *memcg,
				 unsigned long max)
3565
{
3566
	int ret;
3567

3568 3569 3570
	mutex_lock(&memcg_max_mutex);
	ret = page_counter_set_max(&memcg->kmem, max);
	mutex_unlock(&memcg_max_mutex);
3571
	return ret;
3572
}
3573

3574
static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
V
Vladimir Davydov 已提交
3575 3576 3577
{
	int ret;

3578
	mutex_lock(&memcg_max_mutex);
V
Vladimir Davydov 已提交
3579

3580
	ret = page_counter_set_max(&memcg->tcpmem, max);
V
Vladimir Davydov 已提交
3581 3582 3583
	if (ret)
		goto out;

3584
	if (!memcg->tcpmem_active) {
V
Vladimir Davydov 已提交
3585 3586 3587
		/*
		 * The active flag needs to be written after the static_key
		 * update. This is what guarantees that the socket activation
3588 3589 3590
		 * function is the last one to run. See mem_cgroup_sk_alloc()
		 * for details, and note that we don't mark any socket as
		 * belonging to this memcg until that flag is up.
V
Vladimir Davydov 已提交
3591 3592 3593 3594 3595 3596
		 *
		 * We need to do this, because static_keys will span multiple
		 * sites, but we can't control their order. If we mark a socket
		 * as accounted, but the accounting functions are not patched in
		 * yet, we'll lose accounting.
		 *
3597
		 * We never race with the readers in mem_cgroup_sk_alloc(),
V
Vladimir Davydov 已提交
3598 3599 3600 3601
		 * because when this value change, the code to process it is not
		 * patched in yet.
		 */
		static_branch_inc(&memcg_sockets_enabled_key);
3602
		memcg->tcpmem_active = true;
V
Vladimir Davydov 已提交
3603 3604
	}
out:
3605
	mutex_unlock(&memcg_max_mutex);
V
Vladimir Davydov 已提交
3606 3607 3608
	return ret;
}

3609 3610 3611 3612
/*
 * The user of this function is...
 * RES_LIMIT.
 */
3613 3614
static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
B
Balbir Singh 已提交
3615
{
3616
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3617
	unsigned long nr_pages;
3618 3619
	int ret;

3620
	buf = strstrip(buf);
3621
	ret = page_counter_memparse(buf, "-1", &nr_pages);
3622 3623
	if (ret)
		return ret;
3624

3625
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3626
	case RES_LIMIT:
3627 3628 3629 3630
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3631 3632
		switch (MEMFILE_TYPE(of_cft(of)->private)) {
		case _MEM:
3633
			ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3634
			break;
3635
		case _MEMSWAP:
3636
			ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3637
			break;
3638
		case _KMEM:
3639
			ret = memcg_update_kmem_max(memcg, nr_pages);
3640
			break;
V
Vladimir Davydov 已提交
3641
		case _TCP:
3642
			ret = memcg_update_tcp_max(memcg, nr_pages);
V
Vladimir Davydov 已提交
3643
			break;
3644
		}
3645
		break;
3646 3647 3648
	case RES_SOFT_LIMIT:
		memcg->soft_limit = nr_pages;
		ret = 0;
3649 3650
		break;
	}
3651
	return ret ?: nbytes;
B
Balbir Singh 已提交
3652 3653
}

3654 3655
static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
				size_t nbytes, loff_t off)
3656
{
3657
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3658
	struct page_counter *counter;
3659

3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
	switch (MEMFILE_TYPE(of_cft(of)->private)) {
	case _MEM:
		counter = &memcg->memory;
		break;
	case _MEMSWAP:
		counter = &memcg->memsw;
		break;
	case _KMEM:
		counter = &memcg->kmem;
		break;
V
Vladimir Davydov 已提交
3670
	case _TCP:
3671
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
3672
		break;
3673 3674 3675
	default:
		BUG();
	}
3676

3677
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3678
	case RES_MAX_USAGE:
3679
		page_counter_reset_watermark(counter);
3680 3681
		break;
	case RES_FAILCNT:
3682
		counter->failcnt = 0;
3683
		break;
3684 3685
	default:
		BUG();
3686
	}
3687

3688
	return nbytes;
3689 3690
}

3691
static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3692 3693
					struct cftype *cft)
{
3694
	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3695 3696
}

3697
#ifdef CONFIG_MMU
3698
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3699 3700
					struct cftype *cft, u64 val)
{
3701
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3702

3703
	if (val & ~MOVE_MASK)
3704
		return -EINVAL;
3705

3706
	/*
3707 3708 3709 3710
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
3711
	 */
3712
	memcg->move_charge_at_immigrate = val;
3713 3714
	return 0;
}
3715
#else
3716
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3717 3718 3719 3720 3721
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
3722

3723
#ifdef CONFIG_NUMA
3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740

#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
#define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)

static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
					   int nid, unsigned int lru_mask)
{
	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
	unsigned long nr = 0;
	enum lru_list lru;

	VM_BUG_ON((unsigned)nid >= nr_node_ids);

	for_each_lru(lru) {
		if (!(BIT(lru) & lru_mask))
			continue;
3741
		nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754
	}
	return nr;
}

static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
					     unsigned int lru_mask)
{
	unsigned long nr = 0;
	enum lru_list lru;

	for_each_lru(lru) {
		if (!(BIT(lru) & lru_mask))
			continue;
3755
		nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
3756 3757 3758 3759
	}
	return nr;
}

3760
static int memcg_numa_stat_show(struct seq_file *m, void *v)
3761
{
3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773
	struct numa_stat {
		const char *name;
		unsigned int lru_mask;
	};

	static const struct numa_stat stats[] = {
		{ "total", LRU_ALL },
		{ "file", LRU_ALL_FILE },
		{ "anon", LRU_ALL_ANON },
		{ "unevictable", BIT(LRU_UNEVICTABLE) },
	};
	const struct numa_stat *stat;
3774
	int nid;
3775
	unsigned long nr;
3776
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3777

3778 3779 3780 3781 3782 3783 3784 3785 3786
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
		seq_printf(m, "%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
							  stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3787 3788
	}

3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		struct mem_cgroup *iter;

		nr = 0;
		for_each_mem_cgroup_tree(iter, memcg)
			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = 0;
			for_each_mem_cgroup_tree(iter, memcg)
				nr += mem_cgroup_node_nr_lru_pages(
					iter, nid, stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3804 3805 3806 3807 3808 3809
	}

	return 0;
}
#endif /* CONFIG_NUMA */

3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831
static const unsigned int memcg1_stats[] = {
	MEMCG_CACHE,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	NR_SHMEM,
	NR_FILE_MAPPED,
	NR_FILE_DIRTY,
	NR_WRITEBACK,
	MEMCG_SWAP,
};

static const char *const memcg1_stat_names[] = {
	"cache",
	"rss",
	"rss_huge",
	"shmem",
	"mapped_file",
	"dirty",
	"writeback",
	"swap",
};

3832
/* Universal VM events cgroup1 shows, original sort order */
3833
static const unsigned int memcg1_events[] = {
3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
	PGPGIN,
	PGPGOUT,
	PGFAULT,
	PGMAJFAULT,
};

static const char *const memcg1_event_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

3847
static int memcg_stat_show(struct seq_file *m, void *v)
3848
{
3849
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3850
	unsigned long memory, memsw;
3851 3852
	struct mem_cgroup *mi;
	unsigned int i;
3853

3854
	BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3855 3856
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);

3857 3858
	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3859
			continue;
3860
		seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3861
			   memcg_page_state_local(memcg, memcg1_stats[i]) *
3862
			   PAGE_SIZE);
3863
	}
L
Lee Schermerhorn 已提交
3864

3865 3866
	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
		seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3867
			   memcg_events_local(memcg, memcg1_events[i]));
3868 3869 3870

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3871
			   memcg_page_state_local(memcg, NR_LRU_BASE + i) *
3872
			   PAGE_SIZE);
3873

K
KAMEZAWA Hiroyuki 已提交
3874
	/* Hierarchical information */
3875 3876
	memory = memsw = PAGE_COUNTER_MAX;
	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3877 3878
		memory = min(memory, mi->memory.max);
		memsw = min(memsw, mi->memsw.max);
3879
	}
3880 3881
	seq_printf(m, "hierarchical_memory_limit %llu\n",
		   (u64)memory * PAGE_SIZE);
3882
	if (do_memsw_account())
3883 3884
		seq_printf(m, "hierarchical_memsw_limit %llu\n",
			   (u64)memsw * PAGE_SIZE);
K
KOSAKI Motohiro 已提交
3885

3886
	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3887
		if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3888
			continue;
3889
		seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
3890 3891
			   (u64)memcg_page_state(memcg, memcg1_stats[i]) *
			   PAGE_SIZE);
3892 3893
	}

3894 3895
	for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
		seq_printf(m, "total_%s %llu\n", memcg1_event_names[i],
3896
			   (u64)memcg_events(memcg, memcg1_events[i]));
3897

3898 3899
	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i],
3900 3901
			   (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
			   PAGE_SIZE);
K
KAMEZAWA Hiroyuki 已提交
3902

K
KOSAKI Motohiro 已提交
3903 3904
#ifdef CONFIG_DEBUG_VM
	{
3905 3906
		pg_data_t *pgdat;
		struct mem_cgroup_per_node *mz;
3907
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
3908 3909 3910
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

3911 3912 3913
		for_each_online_pgdat(pgdat) {
			mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
			rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
3914

3915 3916 3917 3918 3919
			recent_rotated[0] += rstat->recent_rotated[0];
			recent_rotated[1] += rstat->recent_rotated[1];
			recent_scanned[0] += rstat->recent_scanned[0];
			recent_scanned[1] += rstat->recent_scanned[1];
		}
3920 3921 3922 3923
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
3924 3925 3926
	}
#endif

3927 3928 3929
	return 0;
}

3930 3931
static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
K
KOSAKI Motohiro 已提交
3932
{
3933
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3934

3935
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
3936 3937
}

3938 3939
static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
K
KOSAKI Motohiro 已提交
3940
{
3941
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3942

3943
	if (val > 100)
K
KOSAKI Motohiro 已提交
3944 3945
		return -EINVAL;

3946
	if (css->parent)
3947 3948 3949
		memcg->swappiness = val;
	else
		vm_swappiness = val;
3950

K
KOSAKI Motohiro 已提交
3951 3952 3953
	return 0;
}

3954 3955 3956
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
3957
	unsigned long usage;
3958 3959 3960 3961
	int i;

	rcu_read_lock();
	if (!swap)
3962
		t = rcu_dereference(memcg->thresholds.primary);
3963
	else
3964
		t = rcu_dereference(memcg->memsw_thresholds.primary);
3965 3966 3967 3968

	if (!t)
		goto unlock;

3969
	usage = mem_cgroup_usage(memcg, swap);
3970 3971

	/*
3972
	 * current_threshold points to threshold just below or equal to usage.
3973 3974 3975
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
3976
	i = t->current_threshold;
3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
4000
	t->current_threshold = i - 1;
4001 4002 4003 4004 4005 4006
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
4007 4008
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
4009
		if (do_memsw_account())
4010 4011 4012 4013
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
4014 4015 4016 4017 4018 4019 4020
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

4021 4022 4023 4024 4025 4026 4027
	if (_a->threshold > _b->threshold)
		return 1;

	if (_a->threshold < _b->threshold)
		return -1;

	return 0;
4028 4029
}

4030
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4031 4032 4033
{
	struct mem_cgroup_eventfd_list *ev;

4034 4035
	spin_lock(&memcg_oom_lock);

4036
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
4037
		eventfd_signal(ev->eventfd, 1);
4038 4039

	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4040 4041 4042
	return 0;
}

4043
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
4044
{
K
KAMEZAWA Hiroyuki 已提交
4045 4046
	struct mem_cgroup *iter;

4047
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
4048
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
4049 4050
}

4051
static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4052
	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
4053
{
4054 4055
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4056 4057
	unsigned long threshold;
	unsigned long usage;
4058
	int i, size, ret;
4059

4060
	ret = page_counter_memparse(args, "-1", &threshold);
4061 4062 4063 4064
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
4065

4066
	if (type == _MEM) {
4067
		thresholds = &memcg->thresholds;
4068
		usage = mem_cgroup_usage(memcg, false);
4069
	} else if (type == _MEMSWAP) {
4070
		thresholds = &memcg->memsw_thresholds;
4071
		usage = mem_cgroup_usage(memcg, true);
4072
	} else
4073 4074 4075
		BUG();

	/* Check if a threshold crossed before adding a new one */
4076
	if (thresholds->primary)
4077 4078
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

4079
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
4080 4081

	/* Allocate memory for new array of thresholds */
4082
	new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
4083
	if (!new) {
4084 4085 4086
		ret = -ENOMEM;
		goto unlock;
	}
4087
	new->size = size;
4088 4089

	/* Copy thresholds (if any) to new array */
4090 4091
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
4092
				sizeof(struct mem_cgroup_threshold));
4093 4094
	}

4095
	/* Add new threshold */
4096 4097
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
4098 4099

	/* Sort thresholds. Registering of new threshold isn't time-critical */
4100
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
4101 4102 4103
			compare_thresholds, NULL);

	/* Find current threshold */
4104
	new->current_threshold = -1;
4105
	for (i = 0; i < size; i++) {
4106
		if (new->entries[i].threshold <= usage) {
4107
			/*
4108 4109
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
4110 4111
			 * it here.
			 */
4112
			++new->current_threshold;
4113 4114
		} else
			break;
4115 4116
	}

4117 4118 4119 4120 4121
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
4122

4123
	/* To be sure that nobody uses thresholds */
4124 4125 4126 4127 4128 4129 4130 4131
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

4132
static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4133 4134
	struct eventfd_ctx *eventfd, const char *args)
{
4135
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
T
Tejun Heo 已提交
4136 4137
}

4138
static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4139 4140
	struct eventfd_ctx *eventfd, const char *args)
{
4141
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
T
Tejun Heo 已提交
4142 4143
}

4144
static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4145
	struct eventfd_ctx *eventfd, enum res_type type)
4146
{
4147 4148
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
4149
	unsigned long usage;
4150
	int i, j, size;
4151 4152

	mutex_lock(&memcg->thresholds_lock);
4153 4154

	if (type == _MEM) {
4155
		thresholds = &memcg->thresholds;
4156
		usage = mem_cgroup_usage(memcg, false);
4157
	} else if (type == _MEMSWAP) {
4158
		thresholds = &memcg->memsw_thresholds;
4159
		usage = mem_cgroup_usage(memcg, true);
4160
	} else
4161 4162
		BUG();

4163 4164 4165
	if (!thresholds->primary)
		goto unlock;

4166 4167 4168 4169
	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
4170 4171 4172
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
4173 4174 4175
			size++;
	}

4176
	new = thresholds->spare;
4177

4178 4179
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
4180 4181
		kfree(new);
		new = NULL;
4182
		goto swap_buffers;
4183 4184
	}

4185
	new->size = size;
4186 4187

	/* Copy thresholds and find current threshold */
4188 4189 4190
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
4191 4192
			continue;

4193
		new->entries[j] = thresholds->primary->entries[i];
4194
		if (new->entries[j].threshold <= usage) {
4195
			/*
4196
			 * new->current_threshold will not be used
4197 4198 4199
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
4200
			++new->current_threshold;
4201 4202 4203 4204
		}
		j++;
	}

4205
swap_buffers:
4206 4207
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
4208

4209
	rcu_assign_pointer(thresholds->primary, new);
4210

4211
	/* To be sure that nobody uses thresholds */
4212
	synchronize_rcu();
4213 4214 4215 4216 4217 4218

	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}
4219
unlock:
4220 4221
	mutex_unlock(&memcg->thresholds_lock);
}
4222

4223
static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4224 4225
	struct eventfd_ctx *eventfd)
{
4226
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
T
Tejun Heo 已提交
4227 4228
}

4229
static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4230 4231
	struct eventfd_ctx *eventfd)
{
4232
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
T
Tejun Heo 已提交
4233 4234
}

4235
static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4236
	struct eventfd_ctx *eventfd, const char *args)
K
KAMEZAWA Hiroyuki 已提交
4237 4238 4239 4240 4241 4242 4243
{
	struct mem_cgroup_eventfd_list *event;

	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

4244
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4245 4246 4247 4248 4249

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
4250
	if (memcg->under_oom)
K
KAMEZAWA Hiroyuki 已提交
4251
		eventfd_signal(eventfd, 1);
4252
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4253 4254 4255 4256

	return 0;
}

4257
static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
4258
	struct eventfd_ctx *eventfd)
K
KAMEZAWA Hiroyuki 已提交
4259 4260 4261
{
	struct mem_cgroup_eventfd_list *ev, *tmp;

4262
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4263

4264
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
4265 4266 4267 4268 4269 4270
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

4271
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
4272 4273
}

4274
static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
4275
{
4276
	struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
4277

4278
	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
4279
	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
R
Roman Gushchin 已提交
4280 4281
	seq_printf(sf, "oom_kill %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
4282 4283 4284
	return 0;
}

4285
static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
4286 4287
	struct cftype *cft, u64 val)
{
4288
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4289 4290

	/* cannot set to root cgroup and only 0 and 1 are allowed */
4291
	if (!css->parent || !((val == 0) || (val == 1)))
4292 4293
		return -EINVAL;

4294
	memcg->oom_kill_disable = val;
4295
	if (!val)
4296
		memcg_oom_recover(memcg);
4297

4298 4299 4300
	return 0;
}

4301 4302
#ifdef CONFIG_CGROUP_WRITEBACK

4303 4304
#include <trace/events/writeback.h>

T
Tejun Heo 已提交
4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return wb_domain_init(&memcg->cgwb_domain, gfp);
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
	wb_domain_exit(&memcg->cgwb_domain);
}

4315 4316 4317 4318 4319
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
	wb_domain_size_changed(&memcg->cgwb_domain);
}

T
Tejun Heo 已提交
4320 4321 4322 4323 4324 4325 4326 4327 4328 4329
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);

	if (!memcg->css.parent)
		return NULL;

	return &memcg->cgwb_domain;
}

4330 4331 4332 4333 4334 4335
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page().
 */
static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
{
4336
	long x = atomic_long_read(&memcg->vmstats[idx]);
4337 4338 4339
	int cpu;

	for_each_online_cpu(cpu)
4340
		x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
4341 4342 4343 4344 4345
	if (x < 0)
		x = 0;
	return x;
}

4346 4347 4348
/**
 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
 * @wb: bdi_writeback in question
4349 4350
 * @pfilepages: out parameter for number of file pages
 * @pheadroom: out parameter for number of allocatable pages according to memcg
4351 4352 4353
 * @pdirty: out parameter for number of dirty pages
 * @pwriteback: out parameter for number of pages under writeback
 *
4354 4355 4356
 * Determine the numbers of file, headroom, dirty, and writeback pages in
 * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
 * is a bit more involved.
4357
 *
4358 4359 4360 4361 4362
 * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
 * headroom is calculated as the lowest headroom of itself and the
 * ancestors.  Note that this doesn't consider the actual amount of
 * available memory in the system.  The caller should further cap
 * *@pheadroom accordingly.
4363
 */
4364 4365 4366
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback)
4367 4368 4369 4370
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
	struct mem_cgroup *parent;

4371
	*pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
4372 4373

	/* this should eventually include NR_UNSTABLE_NFS */
4374
	*pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
4375 4376
	*pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
			memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
4377
	*pheadroom = PAGE_COUNTER_MAX;
4378 4379

	while ((parent = parent_mem_cgroup(memcg))) {
4380
		unsigned long ceiling = min(memcg->memory.max, memcg->high);
4381 4382
		unsigned long used = page_counter_read(&memcg->memory);

4383
		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
4384 4385 4386 4387
		memcg = parent;
	}
}

4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441
/*
 * Foreign dirty flushing
 *
 * There's an inherent mismatch between memcg and writeback.  The former
 * trackes ownership per-page while the latter per-inode.  This was a
 * deliberate design decision because honoring per-page ownership in the
 * writeback path is complicated, may lead to higher CPU and IO overheads
 * and deemed unnecessary given that write-sharing an inode across
 * different cgroups isn't a common use-case.
 *
 * Combined with inode majority-writer ownership switching, this works well
 * enough in most cases but there are some pathological cases.  For
 * example, let's say there are two cgroups A and B which keep writing to
 * different but confined parts of the same inode.  B owns the inode and
 * A's memory is limited far below B's.  A's dirty ratio can rise enough to
 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
 * triggering background writeback.  A will be slowed down without a way to
 * make writeback of the dirty pages happen.
 *
 * Conditions like the above can lead to a cgroup getting repatedly and
 * severely throttled after making some progress after each
 * dirty_expire_interval while the underyling IO device is almost
 * completely idle.
 *
 * Solving this problem completely requires matching the ownership tracking
 * granularities between memcg and writeback in either direction.  However,
 * the more egregious behaviors can be avoided by simply remembering the
 * most recent foreign dirtying events and initiating remote flushes on
 * them when local writeback isn't enough to keep the memory clean enough.
 *
 * The following two functions implement such mechanism.  When a foreign
 * page - a page whose memcg and writeback ownerships don't match - is
 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
 * bdi_writeback on the page owning memcg.  When balance_dirty_pages()
 * decides that the memcg needs to sleep due to high dirty ratio, it calls
 * mem_cgroup_flush_foreign() which queues writeback on the recorded
 * foreign bdi_writebacks which haven't expired.  Both the numbers of
 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
 * limited to MEMCG_CGWB_FRN_CNT.
 *
 * The mechanism only remembers IDs and doesn't hold any object references.
 * As being wrong occasionally doesn't matter, updates and accesses to the
 * records are lockless and racy.
 */
void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
					     struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = page->mem_cgroup;
	struct memcg_cgwb_frn *frn;
	u64 now = get_jiffies_64();
	u64 oldest_at = now;
	int oldest = -1;
	int i;

4442 4443
	trace_track_foreign_dirty(page, wb);

4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503
	/*
	 * Pick the slot to use.  If there is already a slot for @wb, keep
	 * using it.  If not replace the oldest one which isn't being
	 * written out.
	 */
	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
		frn = &memcg->cgwb_frn[i];
		if (frn->bdi_id == wb->bdi->id &&
		    frn->memcg_id == wb->memcg_css->id)
			break;
		if (time_before64(frn->at, oldest_at) &&
		    atomic_read(&frn->done.cnt) == 1) {
			oldest = i;
			oldest_at = frn->at;
		}
	}

	if (i < MEMCG_CGWB_FRN_CNT) {
		/*
		 * Re-using an existing one.  Update timestamp lazily to
		 * avoid making the cacheline hot.  We want them to be
		 * reasonably up-to-date and significantly shorter than
		 * dirty_expire_interval as that's what expires the record.
		 * Use the shorter of 1s and dirty_expire_interval / 8.
		 */
		unsigned long update_intv =
			min_t(unsigned long, HZ,
			      msecs_to_jiffies(dirty_expire_interval * 10) / 8);

		if (time_before64(frn->at, now - update_intv))
			frn->at = now;
	} else if (oldest >= 0) {
		/* replace the oldest free one */
		frn = &memcg->cgwb_frn[oldest];
		frn->bdi_id = wb->bdi->id;
		frn->memcg_id = wb->memcg_css->id;
		frn->at = now;
	}
}

/* issue foreign writeback flushes for recorded foreign dirtying events */
void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
	unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
	u64 now = jiffies_64;
	int i;

	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
		struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];

		/*
		 * If the record is older than dirty_expire_interval,
		 * writeback on it has already started.  No need to kick it
		 * off again.  Also, don't start a new one if there's
		 * already one in flight.
		 */
		if (time_after64(frn->at, now - intv) &&
		    atomic_read(&frn->done.cnt) == 1) {
			frn->at = 0;
4504
			trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
4505 4506 4507 4508 4509 4510 4511
			cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
					       WB_REASON_FOREIGN_FLUSH,
					       &frn->done);
		}
	}
}

T
Tejun Heo 已提交
4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522
#else	/* CONFIG_CGROUP_WRITEBACK */

static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return 0;
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
}

4523 4524 4525 4526
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
}

4527 4528
#endif	/* CONFIG_CGROUP_WRITEBACK */

4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541
/*
 * DO NOT USE IN NEW FILES.
 *
 * "cgroup.event_control" implementation.
 *
 * This is way over-engineered.  It tries to support fully configurable
 * events for each user.  Such level of flexibility is completely
 * unnecessary especially in the light of the planned unified hierarchy.
 *
 * Please deprecate this and replace with something simpler if at all
 * possible.
 */

4542 4543 4544 4545 4546
/*
 * Unregister event and free resources.
 *
 * Gets called from workqueue.
 */
4547
static void memcg_event_remove(struct work_struct *work)
4548
{
4549 4550
	struct mem_cgroup_event *event =
		container_of(work, struct mem_cgroup_event, remove);
4551
	struct mem_cgroup *memcg = event->memcg;
4552 4553 4554

	remove_wait_queue(event->wqh, &event->wait);

4555
	event->unregister_event(memcg, event->eventfd);
4556 4557 4558 4559 4560 4561

	/* Notify userspace the event is going away. */
	eventfd_signal(event->eventfd, 1);

	eventfd_ctx_put(event->eventfd);
	kfree(event);
4562
	css_put(&memcg->css);
4563 4564 4565
}

/*
4566
 * Gets called on EPOLLHUP on eventfd when user closes it.
4567 4568 4569
 *
 * Called with wqh->lock held and interrupts disabled.
 */
4570
static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
4571
			    int sync, void *key)
4572
{
4573 4574
	struct mem_cgroup_event *event =
		container_of(wait, struct mem_cgroup_event, wait);
4575
	struct mem_cgroup *memcg = event->memcg;
A
Al Viro 已提交
4576
	__poll_t flags = key_to_poll(key);
4577

4578
	if (flags & EPOLLHUP) {
4579 4580 4581 4582 4583 4584 4585 4586 4587
		/*
		 * If the event has been detached at cgroup removal, we
		 * can simply return knowing the other side will cleanup
		 * for us.
		 *
		 * We can't race against event freeing since the other
		 * side will require wqh->lock via remove_wait_queue(),
		 * which we hold.
		 */
4588
		spin_lock(&memcg->event_list_lock);
4589 4590 4591 4592 4593 4594 4595 4596
		if (!list_empty(&event->list)) {
			list_del_init(&event->list);
			/*
			 * We are in atomic context, but cgroup_event_remove()
			 * may sleep, so we have to call it in workqueue.
			 */
			schedule_work(&event->remove);
		}
4597
		spin_unlock(&memcg->event_list_lock);
4598 4599 4600 4601 4602
	}

	return 0;
}

4603
static void memcg_event_ptable_queue_proc(struct file *file,
4604 4605
		wait_queue_head_t *wqh, poll_table *pt)
{
4606 4607
	struct mem_cgroup_event *event =
		container_of(pt, struct mem_cgroup_event, pt);
4608 4609 4610 4611 4612 4613

	event->wqh = wqh;
	add_wait_queue(wqh, &event->wait);
}

/*
4614 4615
 * DO NOT USE IN NEW FILES.
 *
4616 4617 4618 4619 4620
 * Parse input and register new cgroup event handler.
 *
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
4621 4622
static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
					 char *buf, size_t nbytes, loff_t off)
4623
{
4624
	struct cgroup_subsys_state *css = of_css(of);
4625
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4626
	struct mem_cgroup_event *event;
4627 4628 4629 4630
	struct cgroup_subsys_state *cfile_css;
	unsigned int efd, cfd;
	struct fd efile;
	struct fd cfile;
4631
	const char *name;
4632 4633 4634
	char *endp;
	int ret;

4635 4636 4637
	buf = strstrip(buf);

	efd = simple_strtoul(buf, &endp, 10);
4638 4639
	if (*endp != ' ')
		return -EINVAL;
4640
	buf = endp + 1;
4641

4642
	cfd = simple_strtoul(buf, &endp, 10);
4643 4644
	if ((*endp != ' ') && (*endp != '\0'))
		return -EINVAL;
4645
	buf = endp + 1;
4646 4647 4648 4649 4650

	event = kzalloc(sizeof(*event), GFP_KERNEL);
	if (!event)
		return -ENOMEM;

4651
	event->memcg = memcg;
4652
	INIT_LIST_HEAD(&event->list);
4653 4654 4655
	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
	INIT_WORK(&event->remove, memcg_event_remove);
4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680

	efile = fdget(efd);
	if (!efile.file) {
		ret = -EBADF;
		goto out_kfree;
	}

	event->eventfd = eventfd_ctx_fileget(efile.file);
	if (IS_ERR(event->eventfd)) {
		ret = PTR_ERR(event->eventfd);
		goto out_put_efile;
	}

	cfile = fdget(cfd);
	if (!cfile.file) {
		ret = -EBADF;
		goto out_put_eventfd;
	}

	/* the process need read permission on control file */
	/* AV: shouldn't we check that it's been opened for read instead? */
	ret = inode_permission(file_inode(cfile.file), MAY_READ);
	if (ret < 0)
		goto out_put_cfile;

4681 4682 4683 4684 4685
	/*
	 * Determine the event callbacks and set them in @event.  This used
	 * to be done via struct cftype but cgroup core no longer knows
	 * about these events.  The following is crude but the whole thing
	 * is for compatibility anyway.
4686 4687
	 *
	 * DO NOT ADD NEW FILES.
4688
	 */
A
Al Viro 已提交
4689
	name = cfile.file->f_path.dentry->d_name.name;
4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700

	if (!strcmp(name, "memory.usage_in_bytes")) {
		event->register_event = mem_cgroup_usage_register_event;
		event->unregister_event = mem_cgroup_usage_unregister_event;
	} else if (!strcmp(name, "memory.oom_control")) {
		event->register_event = mem_cgroup_oom_register_event;
		event->unregister_event = mem_cgroup_oom_unregister_event;
	} else if (!strcmp(name, "memory.pressure_level")) {
		event->register_event = vmpressure_register_event;
		event->unregister_event = vmpressure_unregister_event;
	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
T
Tejun Heo 已提交
4701 4702
		event->register_event = memsw_cgroup_usage_register_event;
		event->unregister_event = memsw_cgroup_usage_unregister_event;
4703 4704 4705 4706 4707
	} else {
		ret = -EINVAL;
		goto out_put_cfile;
	}

4708
	/*
4709 4710 4711
	 * Verify @cfile should belong to @css.  Also, remaining events are
	 * automatically removed on cgroup destruction but the removal is
	 * asynchronous, so take an extra ref on @css.
4712
	 */
A
Al Viro 已提交
4713
	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
4714
					       &memory_cgrp_subsys);
4715
	ret = -EINVAL;
4716
	if (IS_ERR(cfile_css))
4717
		goto out_put_cfile;
4718 4719
	if (cfile_css != css) {
		css_put(cfile_css);
4720
		goto out_put_cfile;
4721
	}
4722

4723
	ret = event->register_event(memcg, event->eventfd, buf);
4724 4725 4726
	if (ret)
		goto out_put_css;

4727
	vfs_poll(efile.file, &event->pt);
4728

4729 4730 4731
	spin_lock(&memcg->event_list_lock);
	list_add(&event->list, &memcg->event_list);
	spin_unlock(&memcg->event_list_lock);
4732 4733 4734 4735

	fdput(cfile);
	fdput(efile);

4736
	return nbytes;
4737 4738

out_put_css:
4739
	css_put(css);
4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751
out_put_cfile:
	fdput(cfile);
out_put_eventfd:
	eventfd_ctx_put(event->eventfd);
out_put_efile:
	fdput(efile);
out_kfree:
	kfree(event);

	return ret;
}

4752
static struct cftype mem_cgroup_legacy_files[] = {
B
Balbir Singh 已提交
4753
	{
4754
		.name = "usage_in_bytes",
4755
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4756
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4757
	},
4758 4759
	{
		.name = "max_usage_in_bytes",
4760
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4761
		.write = mem_cgroup_reset,
4762
		.read_u64 = mem_cgroup_read_u64,
4763
	},
B
Balbir Singh 已提交
4764
	{
4765
		.name = "limit_in_bytes",
4766
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4767
		.write = mem_cgroup_write,
4768
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4769
	},
4770 4771 4772
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4773
		.write = mem_cgroup_write,
4774
		.read_u64 = mem_cgroup_read_u64,
4775
	},
B
Balbir Singh 已提交
4776 4777
	{
		.name = "failcnt",
4778
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4779
		.write = mem_cgroup_reset,
4780
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
4781
	},
4782 4783
	{
		.name = "stat",
4784
		.seq_show = memcg_stat_show,
4785
	},
4786 4787
	{
		.name = "force_empty",
4788
		.write = mem_cgroup_force_empty_write,
4789
	},
4790 4791 4792 4793 4794
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
4795
	{
4796
		.name = "cgroup.event_control",		/* XXX: for compat */
4797
		.write = memcg_write_event_control,
4798
		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
4799
	},
K
KOSAKI Motohiro 已提交
4800 4801 4802 4803 4804
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
4805 4806 4807 4808 4809
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
4810 4811
	{
		.name = "oom_control",
4812
		.seq_show = mem_cgroup_oom_control_read,
4813
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
4814 4815
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
4816 4817 4818
	{
		.name = "pressure_level",
	},
4819 4820 4821
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
4822
		.seq_show = memcg_numa_stat_show,
4823 4824
	},
#endif
4825 4826 4827
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4828
		.write = mem_cgroup_write,
4829
		.read_u64 = mem_cgroup_read_u64,
4830 4831 4832 4833
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4834
		.read_u64 = mem_cgroup_read_u64,
4835 4836 4837 4838
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4839
		.write = mem_cgroup_reset,
4840
		.read_u64 = mem_cgroup_read_u64,
4841 4842 4843 4844
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4845
		.write = mem_cgroup_reset,
4846
		.read_u64 = mem_cgroup_read_u64,
4847
	},
Y
Yang Shi 已提交
4848
#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
4849 4850
	{
		.name = "kmem.slabinfo",
4851 4852 4853
		.seq_start = memcg_slab_start,
		.seq_next = memcg_slab_next,
		.seq_stop = memcg_slab_stop,
4854
		.seq_show = memcg_slab_show,
4855 4856
	},
#endif
V
Vladimir Davydov 已提交
4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879
	{
		.name = "kmem.tcp.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.failcnt",
		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
4880
	{ },	/* terminate */
4881
};
4882

4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908
/*
 * Private memory cgroup IDR
 *
 * Swap-out records and page cache shadow entries need to store memcg
 * references in constrained space, so we maintain an ID space that is
 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
 * memory-controlled cgroups to 64k.
 *
 * However, there usually are many references to the oflline CSS after
 * the cgroup has been destroyed, such as page cache or reclaimable
 * slab objects, that don't need to hang on to the ID. We want to keep
 * those dead CSS from occupying IDs, or we might quickly exhaust the
 * relatively small ID space and prevent the creation of new cgroups
 * even when there are much fewer than 64k cgroups - possibly none.
 *
 * Maintain a private 16-bit ID space for memcg, and allow the ID to
 * be freed and recycled when it's no longer needed, which is usually
 * when the CSS is offlined.
 *
 * The only exception to that are records of swapped out tmpfs/shmem
 * pages that need to be attributed to live ancestors on swapin. But
 * those references are manageable from userspace.
 */

static DEFINE_IDR(mem_cgroup_idr);

4909 4910 4911 4912 4913 4914 4915 4916
static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
{
	if (memcg->id.id > 0) {
		idr_remove(&mem_cgroup_idr, memcg->id.id);
		memcg->id.id = 0;
	}
}

4917
static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4918
{
4919
	refcount_add(n, &memcg->id.ref);
4920 4921
}

4922
static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4923
{
4924
	if (refcount_sub_and_test(n, &memcg->id.ref)) {
4925
		mem_cgroup_id_remove(memcg);
4926 4927 4928 4929 4930 4931

		/* Memcg ID pins CSS */
		css_put(&memcg->css);
	}
}

4932 4933 4934 4935 4936 4937 4938 4939 4940 4941
static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
{
	mem_cgroup_id_get_many(memcg, 1);
}

static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
{
	mem_cgroup_id_put_many(memcg, 1);
}

4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953
/**
 * mem_cgroup_from_id - look up a memcg from a memcg id
 * @id: the memcg id to look up
 *
 * Caller must hold rcu_read_lock().
 */
struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return idr_find(&mem_cgroup_idr, id);
}

4954
static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4955 4956
{
	struct mem_cgroup_per_node *pn;
4957
	int tmp = node;
4958 4959 4960 4961 4962 4963 4964 4965
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4966 4967
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4968
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4969 4970
	if (!pn)
		return 1;
4971

4972 4973 4974 4975 4976 4977
	pn->lruvec_stat_local = alloc_percpu(struct lruvec_stat);
	if (!pn->lruvec_stat_local) {
		kfree(pn);
		return 1;
	}

4978 4979
	pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
	if (!pn->lruvec_stat_cpu) {
4980
		free_percpu(pn->lruvec_stat_local);
4981 4982 4983 4984
		kfree(pn);
		return 1;
	}

4985 4986 4987 4988 4989
	lruvec_init(&pn->lruvec);
	pn->usage_in_excess = 0;
	pn->on_tree = false;
	pn->memcg = memcg;

4990
	memcg->nodeinfo[node] = pn;
4991 4992 4993
	return 0;
}

4994
static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4995
{
4996 4997
	struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];

M
Michal Hocko 已提交
4998 4999 5000
	if (!pn)
		return;

5001
	free_percpu(pn->lruvec_stat_cpu);
5002
	free_percpu(pn->lruvec_stat_local);
5003
	kfree(pn);
5004 5005
}

5006
static void __mem_cgroup_free(struct mem_cgroup *memcg)
5007
{
5008
	int node;
5009

5010
	/*
5011
	 * Flush percpu vmstats and vmevents to guarantee the value correctness
5012 5013
	 * on parent's and all ancestor levels.
	 */
5014
	memcg_flush_percpu_vmstats(memcg, false);
5015
	memcg_flush_percpu_vmevents(memcg);
5016
	for_each_node(node)
5017
		free_mem_cgroup_per_node_info(memcg, node);
5018
	free_percpu(memcg->vmstats_percpu);
5019
	free_percpu(memcg->vmstats_local);
5020
	kfree(memcg);
5021
}
5022

5023 5024 5025 5026 5027 5028
static void mem_cgroup_free(struct mem_cgroup *memcg)
{
	memcg_wb_domain_exit(memcg);
	__mem_cgroup_free(memcg);
}

5029
static struct mem_cgroup *mem_cgroup_alloc(void)
B
Balbir Singh 已提交
5030
{
5031
	struct mem_cgroup *memcg;
5032
	unsigned int size;
5033
	int node;
5034
	int __maybe_unused i;
B
Balbir Singh 已提交
5035

5036 5037 5038 5039
	size = sizeof(struct mem_cgroup);
	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);

	memcg = kzalloc(size, GFP_KERNEL);
5040
	if (!memcg)
5041 5042
		return NULL;

5043 5044 5045 5046 5047 5048
	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
				 1, MEM_CGROUP_ID_MAX,
				 GFP_KERNEL);
	if (memcg->id.id < 0)
		goto fail;

5049 5050 5051 5052
	memcg->vmstats_local = alloc_percpu(struct memcg_vmstats_percpu);
	if (!memcg->vmstats_local)
		goto fail;

5053 5054
	memcg->vmstats_percpu = alloc_percpu(struct memcg_vmstats_percpu);
	if (!memcg->vmstats_percpu)
5055
		goto fail;
5056

B
Bob Liu 已提交
5057
	for_each_node(node)
5058
		if (alloc_mem_cgroup_per_node_info(memcg, node))
5059
			goto fail;
5060

5061 5062
	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
		goto fail;
5063

5064
	INIT_WORK(&memcg->high_work, high_work_func);
5065 5066 5067 5068
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);
5069
	vmpressure_init(&memcg->vmpressure);
5070 5071
	INIT_LIST_HEAD(&memcg->event_list);
	spin_lock_init(&memcg->event_list_lock);
5072
	memcg->socket_pressure = jiffies;
5073
#ifdef CONFIG_MEMCG_KMEM
V
Vladimir Davydov 已提交
5074 5075
	memcg->kmemcg_id = -1;
#endif
5076 5077
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&memcg->cgwb_list);
5078 5079 5080
	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
		memcg->cgwb_frn[i].done =
			__WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
5081
#endif
5082
	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
5083 5084
	return memcg;
fail:
5085
	mem_cgroup_id_remove(memcg);
5086
	__mem_cgroup_free(memcg);
5087
	return NULL;
5088 5089
}

5090 5091
static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
5092
{
5093 5094 5095
	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
	struct mem_cgroup *memcg;
	long error = -ENOMEM;
5096

5097 5098 5099
	memcg = mem_cgroup_alloc();
	if (!memcg)
		return ERR_PTR(error);
5100

5101 5102 5103 5104 5105 5106 5107 5108
	memcg->high = PAGE_COUNTER_MAX;
	memcg->soft_limit = PAGE_COUNTER_MAX;
	if (parent) {
		memcg->swappiness = mem_cgroup_swappiness(parent);
		memcg->oom_kill_disable = parent->oom_kill_disable;
	}
	if (parent && parent->use_hierarchy) {
		memcg->use_hierarchy = true;
5109
		page_counter_init(&memcg->memory, &parent->memory);
5110
		page_counter_init(&memcg->swap, &parent->swap);
5111 5112
		page_counter_init(&memcg->memsw, &parent->memsw);
		page_counter_init(&memcg->kmem, &parent->kmem);
5113
		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
5114
	} else {
5115
		page_counter_init(&memcg->memory, NULL);
5116
		page_counter_init(&memcg->swap, NULL);
5117 5118
		page_counter_init(&memcg->memsw, NULL);
		page_counter_init(&memcg->kmem, NULL);
5119
		page_counter_init(&memcg->tcpmem, NULL);
5120 5121 5122 5123 5124
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
5125
		if (parent != root_mem_cgroup)
5126
			memory_cgrp_subsys.broken_hierarchy = true;
5127
	}
5128

5129 5130
	/* The following stuff does not apply to the root */
	if (!parent) {
5131 5132 5133
#ifdef CONFIG_MEMCG_KMEM
		INIT_LIST_HEAD(&memcg->kmem_caches);
#endif
5134 5135 5136 5137
		root_mem_cgroup = memcg;
		return &memcg->css;
	}

5138
	error = memcg_online_kmem(memcg);
5139 5140
	if (error)
		goto fail;
5141

5142
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5143
		static_branch_inc(&memcg_sockets_enabled_key);
5144

5145 5146
	return &memcg->css;
fail:
5147
	mem_cgroup_id_remove(memcg);
5148
	mem_cgroup_free(memcg);
5149
	return ERR_PTR(-ENOMEM);
5150 5151
}

5152
static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
5153
{
5154 5155
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

5156 5157 5158 5159 5160 5161 5162 5163 5164 5165
	/*
	 * A memcg must be visible for memcg_expand_shrinker_maps()
	 * by the time the maps are allocated. So, we allocate maps
	 * here, when for_each_mem_cgroup() can't skip it.
	 */
	if (memcg_alloc_shrinker_maps(memcg)) {
		mem_cgroup_id_remove(memcg);
		return -ENOMEM;
	}

5166
	/* Online state pins memcg ID, memcg ID pins CSS */
5167
	refcount_set(&memcg->id.ref, 1);
5168
	css_get(css);
5169
	return 0;
B
Balbir Singh 已提交
5170 5171
}

5172
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
5173
{
5174
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5175
	struct mem_cgroup_event *event, *tmp;
5176 5177 5178 5179 5180 5181

	/*
	 * Unregister events and notify userspace.
	 * Notify userspace about cgroup removing only after rmdir of cgroup
	 * directory to avoid race between userspace and kernelspace.
	 */
5182 5183
	spin_lock(&memcg->event_list_lock);
	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
5184 5185 5186
		list_del_init(&event->list);
		schedule_work(&event->remove);
	}
5187
	spin_unlock(&memcg->event_list_lock);
5188

R
Roman Gushchin 已提交
5189
	page_counter_set_min(&memcg->memory, 0);
5190
	page_counter_set_low(&memcg->memory, 0);
5191

5192
	memcg_offline_kmem(memcg);
5193
	wb_memcg_offline(memcg);
5194

5195 5196
	drain_all_stock(memcg);

5197
	mem_cgroup_id_put(memcg);
5198 5199
}

5200 5201 5202 5203 5204 5205 5206
static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	invalidate_reclaim_iterators(memcg);
}

5207
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
B
Balbir Singh 已提交
5208
{
5209
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5210
	int __maybe_unused i;
5211

5212 5213 5214 5215
#ifdef CONFIG_CGROUP_WRITEBACK
	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
		wb_wait_for_completion(&memcg->cgwb_frn[i].done);
#endif
5216
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
5217
		static_branch_dec(&memcg_sockets_enabled_key);
5218

5219
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
V
Vladimir Davydov 已提交
5220
		static_branch_dec(&memcg_sockets_enabled_key);
5221

5222 5223 5224
	vmpressure_cleanup(&memcg->vmpressure);
	cancel_work_sync(&memcg->high_work);
	mem_cgroup_remove_from_trees(memcg);
5225
	memcg_free_shrinker_maps(memcg);
5226
	memcg_free_kmem(memcg);
5227
	mem_cgroup_free(memcg);
B
Balbir Singh 已提交
5228 5229
}

5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246
/**
 * mem_cgroup_css_reset - reset the states of a mem_cgroup
 * @css: the target css
 *
 * Reset the states of the mem_cgroup associated with @css.  This is
 * invoked when the userland requests disabling on the default hierarchy
 * but the memcg is pinned through dependency.  The memcg should stop
 * applying policies and should revert to the vanilla state as it may be
 * made visible again.
 *
 * The current implementation only resets the essential configurations.
 * This needs to be expanded to cover all the visible parts.
 */
static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

5247 5248 5249 5250 5251
	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
R
Roman Gushchin 已提交
5252
	page_counter_set_min(&memcg->memory, 0);
5253
	page_counter_set_low(&memcg->memory, 0);
5254
	memcg->high = PAGE_COUNTER_MAX;
5255
	memcg->soft_limit = PAGE_COUNTER_MAX;
5256
	memcg_wb_domain_size_changed(memcg);
5257 5258
}

5259
#ifdef CONFIG_MMU
5260
/* Handlers for move charge at task migration. */
5261
static int mem_cgroup_do_precharge(unsigned long count)
5262
{
5263
	int ret;
5264

5265 5266
	/* Try a single bulk charge without reclaim first, kswapd may wake */
	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
5267
	if (!ret) {
5268 5269 5270
		mc.precharge += count;
		return ret;
	}
5271

5272
	/* Try charges one by one with reclaim, but do not retry */
5273
	while (count--) {
5274
		ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
5275 5276
		if (ret)
			return ret;
5277
		mc.precharge++;
5278
		cond_resched();
5279
	}
5280
	return 0;
5281 5282 5283 5284
}

union mc_target {
	struct page	*page;
5285
	swp_entry_t	ent;
5286 5287 5288
};

enum mc_target_type {
5289
	MC_TARGET_NONE = 0,
5290
	MC_TARGET_PAGE,
5291
	MC_TARGET_SWAP,
5292
	MC_TARGET_DEVICE,
5293 5294
};

D
Daisuke Nishimura 已提交
5295 5296
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
5297
{
5298
	struct page *page = vm_normal_page(vma, addr, ptent);
5299

D
Daisuke Nishimura 已提交
5300 5301 5302
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
5303
		if (!(mc.flags & MOVE_ANON))
D
Daisuke Nishimura 已提交
5304
			return NULL;
5305 5306 5307 5308
	} else {
		if (!(mc.flags & MOVE_FILE))
			return NULL;
	}
D
Daisuke Nishimura 已提交
5309 5310 5311 5312 5313 5314
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

5315
#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
D
Daisuke Nishimura 已提交
5316
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5317
			pte_t ptent, swp_entry_t *entry)
D
Daisuke Nishimura 已提交
5318 5319 5320 5321
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

5322
	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
D
Daisuke Nishimura 已提交
5323
		return NULL;
5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340

	/*
	 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
	 * a device and because they are not accessible by CPU they are store
	 * as special swap entry in the CPU page table.
	 */
	if (is_device_private_entry(ent)) {
		page = device_private_entry_to_page(ent);
		/*
		 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
		 * a refcount of 1 when free (unlike normal page)
		 */
		if (!page_ref_add_unless(page, 1, 1))
			return NULL;
		return page;
	}

5341 5342 5343 5344
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
5345
	page = find_get_page(swap_address_space(ent), swp_offset(ent));
5346
	if (do_memsw_account())
D
Daisuke Nishimura 已提交
5347 5348 5349 5350
		entry->val = ent.val;

	return page;
}
5351 5352
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5353
			pte_t ptent, swp_entry_t *entry)
5354 5355 5356 5357
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
5358

5359 5360 5361 5362 5363 5364 5365 5366 5367
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
5368
	if (!(mc.flags & MOVE_FILE))
5369 5370 5371
		return NULL;

	mapping = vma->vm_file->f_mapping;
5372
	pgoff = linear_page_index(vma, addr);
5373 5374

	/* page is moved even if it's not RSS of this task(page-faulted). */
5375 5376
#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
5377 5378
	if (shmem_mapping(mapping)) {
		page = find_get_entry(mapping, pgoff);
5379
		if (xa_is_value(page)) {
5380
			swp_entry_t swp = radix_to_swp_entry(page);
5381
			if (do_memsw_account())
5382
				*entry = swp;
5383 5384
			page = find_get_page(swap_address_space(swp),
					     swp_offset(swp));
5385 5386 5387 5388 5389
		}
	} else
		page = find_get_page(mapping, pgoff);
#else
	page = find_get_page(mapping, pgoff);
5390
#endif
5391 5392 5393
	return page;
}

5394 5395 5396
/**
 * mem_cgroup_move_account - move account of the page
 * @page: the page
5397
 * @compound: charge the page as compound or small page
5398 5399 5400
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
5401
 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
5402 5403 5404 5405 5406
 *
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
 */
static int mem_cgroup_move_account(struct page *page,
5407
				   bool compound,
5408 5409 5410 5411
				   struct mem_cgroup *from,
				   struct mem_cgroup *to)
{
	unsigned long flags;
5412
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5413
	int ret;
5414
	bool anon;
5415 5416 5417

	VM_BUG_ON(from == to);
	VM_BUG_ON_PAGE(PageLRU(page), page);
5418
	VM_BUG_ON(compound && !PageTransHuge(page));
5419 5420

	/*
5421
	 * Prevent mem_cgroup_migrate() from looking at
5422
	 * page->mem_cgroup of its source page while we change it.
5423
	 */
5424
	ret = -EBUSY;
5425 5426 5427 5428 5429 5430 5431
	if (!trylock_page(page))
		goto out;

	ret = -EINVAL;
	if (page->mem_cgroup != from)
		goto out_unlock;

5432 5433
	anon = PageAnon(page);

5434 5435
	spin_lock_irqsave(&from->move_lock, flags);

5436
	if (!anon && page_mapped(page)) {
5437 5438
		__mod_memcg_state(from, NR_FILE_MAPPED, -nr_pages);
		__mod_memcg_state(to, NR_FILE_MAPPED, nr_pages);
5439 5440
	}

5441 5442
	/*
	 * move_lock grabbed above and caller set from->moving_account, so
5443
	 * mod_memcg_page_state will serialize updates to PageDirty.
5444 5445 5446 5447 5448 5449
	 * So mapping should be stable for dirty pages.
	 */
	if (!anon && PageDirty(page)) {
		struct address_space *mapping = page_mapping(page);

		if (mapping_cap_account_dirty(mapping)) {
5450 5451
			__mod_memcg_state(from, NR_FILE_DIRTY, -nr_pages);
			__mod_memcg_state(to, NR_FILE_DIRTY, nr_pages);
5452 5453 5454
		}
	}

5455
	if (PageWriteback(page)) {
5456 5457
		__mod_memcg_state(from, NR_WRITEBACK, -nr_pages);
		__mod_memcg_state(to, NR_WRITEBACK, nr_pages);
5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472
	}

	/*
	 * It is safe to change page->mem_cgroup here because the page
	 * is referenced, charged, and isolated - we can't race with
	 * uncharging, charging, migration, or LRU putback.
	 */

	/* caller should have done css_get */
	page->mem_cgroup = to;
	spin_unlock_irqrestore(&from->move_lock, flags);

	ret = 0;

	local_irq_disable();
5473
	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
5474
	memcg_check_events(to, page);
5475
	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
5476 5477 5478 5479 5480 5481 5482 5483
	memcg_check_events(from, page);
	local_irq_enable();
out_unlock:
	unlock_page(page);
out:
	return ret;
}

5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498
/**
 * get_mctgt_type - get target type of moving charge
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
5499 5500
 *   3(MC_TARGET_DEVICE): like MC_TARGET_PAGE  but page is MEMORY_DEVICE_PRIVATE
 *     (so ZONE_DEVICE page and thus not on the lru).
5501 5502 5503
 *     For now we such page is charge like a regular page would be as for all
 *     intent and purposes it is just special memory taking the place of a
 *     regular page.
5504 5505
 *
 *     See Documentations/vm/hmm.txt and include/linux/hmm.h
5506 5507 5508 5509
 *
 * Called with pte lock held.
 */

5510
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
5511 5512 5513
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
5514
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
5515 5516 5517 5518 5519
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
5520
		page = mc_handle_swap_pte(vma, ptent, &ent);
5521
	else if (pte_none(ptent))
5522
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
5523 5524

	if (!page && !ent.val)
5525
		return ret;
5526 5527
	if (page) {
		/*
5528
		 * Do only loose check w/o serialization.
5529
		 * mem_cgroup_move_account() checks the page is valid or
5530
		 * not under LRU exclusion.
5531
		 */
5532
		if (page->mem_cgroup == mc.from) {
5533
			ret = MC_TARGET_PAGE;
5534
			if (is_device_private_page(page))
5535
				ret = MC_TARGET_DEVICE;
5536 5537 5538 5539 5540 5541
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
5542 5543 5544 5545 5546
	/*
	 * There is a swap entry and a page doesn't exist or isn't charged.
	 * But we cannot move a tail-page in a THP.
	 */
	if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
L
Li Zefan 已提交
5547
	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
5548 5549 5550
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
5551 5552 5553 5554
	}
	return ret;
}

5555 5556
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
5557 5558
 * We don't consider PMD mapped swapping or file mapped pages because THP does
 * not support them for now.
5559 5560 5561 5562 5563 5564 5565 5566
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	enum mc_target_type ret = MC_TARGET_NONE;

5567 5568 5569 5570 5571
	if (unlikely(is_swap_pmd(pmd))) {
		VM_BUG_ON(thp_migration_supported() &&
				  !is_pmd_migration_entry(pmd));
		return ret;
	}
5572
	page = pmd_page(pmd);
5573
	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
5574
	if (!(mc.flags & MOVE_ANON))
5575
		return ret;
5576
	if (page->mem_cgroup == mc.from) {
5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

5593 5594 5595 5596
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
5597
	struct vm_area_struct *vma = walk->vma;
5598 5599 5600
	pte_t *pte;
	spinlock_t *ptl;

5601 5602
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
5603 5604
		/*
		 * Note their can not be MC_TARGET_DEVICE for now as we do not
5605 5606
		 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
		 * this might change.
5607
		 */
5608 5609
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
5610
		spin_unlock(ptl);
5611
		return 0;
5612
	}
5613

5614 5615
	if (pmd_trans_unstable(pmd))
		return 0;
5616 5617
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
5618
		if (get_mctgt_type(vma, addr, *pte, NULL))
5619 5620 5621 5622
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

5623 5624 5625
	return 0;
}

5626 5627 5628 5629
static const struct mm_walk_ops precharge_walk_ops = {
	.pmd_entry	= mem_cgroup_count_precharge_pte_range,
};

5630 5631 5632 5633
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;

5634
	down_read(&mm->mmap_sem);
5635
	walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
5636
	up_read(&mm->mmap_sem);
5637 5638 5639 5640 5641 5642 5643 5644 5645

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
5646 5647 5648 5649 5650
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
5651 5652
}

5653 5654
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
5655
{
5656 5657 5658
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

5659
	/* we must uncharge all the leftover precharges from mc.to */
5660
	if (mc.precharge) {
5661
		cancel_charge(mc.to, mc.precharge);
5662 5663 5664 5665 5666 5667 5668
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
5669
		cancel_charge(mc.from, mc.moved_charge);
5670
		mc.moved_charge = 0;
5671
	}
5672 5673 5674
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
5675
		if (!mem_cgroup_is_root(mc.from))
5676
			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
5677

5678 5679
		mem_cgroup_id_put_many(mc.from, mc.moved_swap);

5680
		/*
5681 5682
		 * we charged both to->memory and to->memsw, so we
		 * should uncharge to->memory.
5683
		 */
5684
		if (!mem_cgroup_is_root(mc.to))
5685 5686
			page_counter_uncharge(&mc.to->memory, mc.moved_swap);

5687 5688
		mem_cgroup_id_get_many(mc.to, mc.moved_swap);
		css_put_many(&mc.to->css, mc.moved_swap);
5689

5690 5691
		mc.moved_swap = 0;
	}
5692 5693 5694 5695 5696 5697 5698
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
5699 5700
	struct mm_struct *mm = mc.mm;

5701 5702 5703 5704 5705 5706
	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
5707
	spin_lock(&mc.lock);
5708 5709
	mc.from = NULL;
	mc.to = NULL;
5710
	mc.mm = NULL;
5711
	spin_unlock(&mc.lock);
5712 5713

	mmput(mm);
5714 5715
}

5716
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5717
{
5718
	struct cgroup_subsys_state *css;
5719
	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
5720
	struct mem_cgroup *from;
5721
	struct task_struct *leader, *p;
5722
	struct mm_struct *mm;
5723
	unsigned long move_flags;
5724
	int ret = 0;
5725

5726 5727
	/* charge immigration isn't supported on the default hierarchy */
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5728 5729
		return 0;

5730 5731 5732 5733 5734 5735 5736
	/*
	 * Multi-process migrations only happen on the default hierarchy
	 * where charge immigration is not used.  Perform charge
	 * immigration if @tset contains a leader and whine if there are
	 * multiple.
	 */
	p = NULL;
5737
	cgroup_taskset_for_each_leader(leader, css, tset) {
5738 5739
		WARN_ON_ONCE(p);
		p = leader;
5740
		memcg = mem_cgroup_from_css(css);
5741 5742 5743 5744
	}
	if (!p)
		return 0;

5745 5746 5747 5748 5749 5750 5751 5752 5753
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
	if (!move_flags)
		return 0;

5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769
	from = mem_cgroup_from_task(p);

	VM_BUG_ON(from == memcg);

	mm = get_task_mm(p);
	if (!mm)
		return 0;
	/* We move charges only when we move a owner of the mm */
	if (mm->owner == p) {
		VM_BUG_ON(mc.from);
		VM_BUG_ON(mc.to);
		VM_BUG_ON(mc.precharge);
		VM_BUG_ON(mc.moved_charge);
		VM_BUG_ON(mc.moved_swap);

		spin_lock(&mc.lock);
5770
		mc.mm = mm;
5771 5772 5773 5774 5775 5776 5777 5778 5779
		mc.from = from;
		mc.to = memcg;
		mc.flags = move_flags;
		spin_unlock(&mc.lock);
		/* We set mc.moving_task later */

		ret = mem_cgroup_precharge_mc(mm);
		if (ret)
			mem_cgroup_clear_mc();
5780 5781
	} else {
		mmput(mm);
5782 5783 5784 5785
	}
	return ret;
}

5786
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5787
{
5788 5789
	if (mc.to)
		mem_cgroup_clear_mc();
5790 5791
}

5792 5793 5794
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
5795
{
5796
	int ret = 0;
5797
	struct vm_area_struct *vma = walk->vma;
5798 5799
	pte_t *pte;
	spinlock_t *ptl;
5800 5801 5802
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
5803

5804 5805
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
5806
		if (mc.precharge < HPAGE_PMD_NR) {
5807
			spin_unlock(ptl);
5808 5809 5810 5811 5812 5813
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
5814
				if (!mem_cgroup_move_account(page, true,
5815
							     mc.from, mc.to)) {
5816 5817 5818 5819 5820 5821
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
5822 5823 5824 5825 5826 5827 5828 5829
		} else if (target_type == MC_TARGET_DEVICE) {
			page = target.page;
			if (!mem_cgroup_move_account(page, true,
						     mc.from, mc.to)) {
				mc.precharge -= HPAGE_PMD_NR;
				mc.moved_charge += HPAGE_PMD_NR;
			}
			put_page(page);
5830
		}
5831
		spin_unlock(ptl);
5832
		return 0;
5833 5834
	}

5835 5836
	if (pmd_trans_unstable(pmd))
		return 0;
5837 5838 5839 5840
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
5841
		bool device = false;
5842
		swp_entry_t ent;
5843 5844 5845 5846

		if (!mc.precharge)
			break;

5847
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
5848 5849 5850
		case MC_TARGET_DEVICE:
			device = true;
			/* fall through */
5851 5852
		case MC_TARGET_PAGE:
			page = target.page;
5853 5854 5855 5856 5857 5858 5859 5860
			/*
			 * We can have a part of the split pmd here. Moving it
			 * can be done but it would be too convoluted so simply
			 * ignore such a partial THP and keep it in original
			 * memcg. There should be somebody mapping the head.
			 */
			if (PageTransCompound(page))
				goto put;
5861
			if (!device && isolate_lru_page(page))
5862
				goto put;
5863 5864
			if (!mem_cgroup_move_account(page, false,
						mc.from, mc.to)) {
5865
				mc.precharge--;
5866 5867
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
5868
			}
5869 5870
			if (!device)
				putback_lru_page(page);
5871
put:			/* get_mctgt_type() gets the page */
5872 5873
			put_page(page);
			break;
5874 5875
		case MC_TARGET_SWAP:
			ent = target.ent;
5876
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
5877
				mc.precharge--;
5878 5879 5880
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
5881
			break;
5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
5896
		ret = mem_cgroup_do_precharge(1);
5897 5898 5899 5900 5901 5902 5903
		if (!ret)
			goto retry;
	}

	return ret;
}

5904 5905 5906 5907
static const struct mm_walk_ops charge_walk_ops = {
	.pmd_entry	= mem_cgroup_move_charge_pte_range,
};

5908
static void mem_cgroup_move_charge(void)
5909 5910
{
	lru_add_drain_all();
5911
	/*
5912 5913 5914
	 * Signal lock_page_memcg() to take the memcg's move_lock
	 * while we're moving its pages to another memcg. Then wait
	 * for already started RCU-only updates to finish.
5915 5916 5917
	 */
	atomic_inc(&mc.from->moving_account);
	synchronize_rcu();
5918
retry:
5919
	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
5931 5932 5933 5934
	/*
	 * When we have consumed all precharges and failed in doing
	 * additional charge, the page walk just aborts.
	 */
5935 5936
	walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
			NULL);
5937

5938
	up_read(&mc.mm->mmap_sem);
5939
	atomic_dec(&mc.from->moving_account);
5940 5941
}

5942
static void mem_cgroup_move_task(void)
B
Balbir Singh 已提交
5943
{
5944 5945
	if (mc.to) {
		mem_cgroup_move_charge();
5946
		mem_cgroup_clear_mc();
5947
	}
B
Balbir Singh 已提交
5948
}
5949
#else	/* !CONFIG_MMU */
5950
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5951 5952 5953
{
	return 0;
}
5954
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5955 5956
{
}
5957
static void mem_cgroup_move_task(void)
5958 5959 5960
{
}
#endif
B
Balbir Singh 已提交
5961

5962 5963
/*
 * Cgroup retains root cgroups across [un]mount cycles making it necessary
5964 5965
 * to verify whether we're attached to the default hierarchy on each mount
 * attempt.
5966
 */
5967
static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5968 5969
{
	/*
5970
	 * use_hierarchy is forced on the default hierarchy.  cgroup core
5971 5972 5973
	 * guarantees that @root doesn't have any children, so turning it
	 * on for the root memcg is enough.
	 */
5974
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5975 5976 5977
		root_mem_cgroup->use_hierarchy = true;
	else
		root_mem_cgroup->use_hierarchy = false;
5978 5979
}

5980 5981 5982 5983 5984 5985 5986 5987 5988 5989
static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
{
	if (value == PAGE_COUNTER_MAX)
		seq_puts(m, "max\n");
	else
		seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);

	return 0;
}

5990 5991 5992
static u64 memory_current_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
{
5993 5994 5995
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5996 5997
}

R
Roman Gushchin 已提交
5998 5999
static int memory_min_show(struct seq_file *m, void *v)
{
6000 6001
	return seq_puts_memcg_tunable(m,
		READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
R
Roman Gushchin 已提交
6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020
}

static ssize_t memory_min_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long min;
	int err;

	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &min);
	if (err)
		return err;

	page_counter_set_min(&memcg->memory, min);

	return nbytes;
}

6021 6022
static int memory_low_show(struct seq_file *m, void *v)
{
6023 6024
	return seq_puts_memcg_tunable(m,
		READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
6025 6026 6027 6028 6029 6030 6031 6032 6033 6034
}

static ssize_t memory_low_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long low;
	int err;

	buf = strstrip(buf);
6035
	err = page_counter_memparse(buf, "max", &low);
6036 6037 6038
	if (err)
		return err;

6039
	page_counter_set_low(&memcg->memory, low);
6040 6041 6042 6043 6044 6045

	return nbytes;
}

static int memory_high_show(struct seq_file *m, void *v)
{
6046
	return seq_puts_memcg_tunable(m, READ_ONCE(mem_cgroup_from_seq(m)->high));
6047 6048 6049 6050 6051 6052
}

static ssize_t memory_high_write(struct kernfs_open_file *of,
				 char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6053
	unsigned long nr_pages;
6054 6055 6056 6057
	unsigned long high;
	int err;

	buf = strstrip(buf);
6058
	err = page_counter_memparse(buf, "max", &high);
6059 6060 6061 6062 6063
	if (err)
		return err;

	memcg->high = high;

6064 6065 6066 6067 6068
	nr_pages = page_counter_read(&memcg->memory);
	if (nr_pages > high)
		try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
					     GFP_KERNEL, true);

6069
	memcg_wb_domain_size_changed(memcg);
6070 6071 6072 6073 6074
	return nbytes;
}

static int memory_max_show(struct seq_file *m, void *v)
{
6075 6076
	return seq_puts_memcg_tunable(m,
		READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
6077 6078 6079 6080 6081 6082
}

static ssize_t memory_max_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6083 6084
	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
	bool drained = false;
6085 6086 6087 6088
	unsigned long max;
	int err;

	buf = strstrip(buf);
6089
	err = page_counter_memparse(buf, "max", &max);
6090 6091 6092
	if (err)
		return err;

6093
	xchg(&memcg->memory.max, max);
6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118

	for (;;) {
		unsigned long nr_pages = page_counter_read(&memcg->memory);

		if (nr_pages <= max)
			break;

		if (signal_pending(current)) {
			err = -EINTR;
			break;
		}

		if (!drained) {
			drain_all_stock(memcg);
			drained = true;
			continue;
		}

		if (nr_reclaims) {
			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
							  GFP_KERNEL, true))
				nr_reclaims--;
			continue;
		}

6119
		memcg_memory_event(memcg, MEMCG_OOM);
6120 6121 6122
		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
			break;
	}
6123

6124
	memcg_wb_domain_size_changed(memcg);
6125 6126 6127
	return nbytes;
}

6128 6129 6130 6131 6132 6133 6134 6135 6136 6137
static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
{
	seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
	seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
	seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
	seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
	seq_printf(m, "oom_kill %lu\n",
		   atomic_long_read(&events[MEMCG_OOM_KILL]));
}

6138 6139
static int memory_events_show(struct seq_file *m, void *v)
{
6140
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6141

6142 6143 6144 6145 6146 6147 6148
	__memory_events_show(m, memcg->memory_events);
	return 0;
}

static int memory_events_local_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6149

6150
	__memory_events_show(m, memcg->memory_events_local);
6151 6152 6153
	return 0;
}

6154 6155
static int memory_stat_show(struct seq_file *m, void *v)
{
6156
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6157
	char *buf;
6158

6159 6160 6161 6162 6163
	buf = memory_stat_format(memcg);
	if (!buf)
		return -ENOMEM;
	seq_puts(m, buf);
	kfree(buf);
6164 6165 6166
	return 0;
}

6167 6168
static int memory_oom_group_show(struct seq_file *m, void *v)
{
6169
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197

	seq_printf(m, "%d\n", memcg->oom_group);

	return 0;
}

static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
				      char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	int ret, oom_group;

	buf = strstrip(buf);
	if (!buf)
		return -EINVAL;

	ret = kstrtoint(buf, 0, &oom_group);
	if (ret)
		return ret;

	if (oom_group != 0 && oom_group != 1)
		return -EINVAL;

	memcg->oom_group = oom_group;

	return nbytes;
}

6198 6199 6200
static struct cftype memory_files[] = {
	{
		.name = "current",
6201
		.flags = CFTYPE_NOT_ON_ROOT,
6202 6203
		.read_u64 = memory_current_read,
	},
R
Roman Gushchin 已提交
6204 6205 6206 6207 6208 6209
	{
		.name = "min",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_min_show,
		.write = memory_min_write,
	},
6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230
	{
		.name = "low",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_low_show,
		.write = memory_low_write,
	},
	{
		.name = "high",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_high_show,
		.write = memory_high_write,
	},
	{
		.name = "max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_max_show,
		.write = memory_max_write,
	},
	{
		.name = "events",
		.flags = CFTYPE_NOT_ON_ROOT,
6231
		.file_offset = offsetof(struct mem_cgroup, events_file),
6232 6233
		.seq_show = memory_events_show,
	},
6234 6235 6236 6237 6238 6239
	{
		.name = "events.local",
		.flags = CFTYPE_NOT_ON_ROOT,
		.file_offset = offsetof(struct mem_cgroup, events_local_file),
		.seq_show = memory_events_local_show,
	},
6240 6241 6242 6243 6244
	{
		.name = "stat",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_stat_show,
	},
6245 6246 6247 6248 6249 6250
	{
		.name = "oom.group",
		.flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
		.seq_show = memory_oom_group_show,
		.write = memory_oom_group_write,
	},
6251 6252 6253
	{ }	/* terminate */
};

6254
struct cgroup_subsys memory_cgrp_subsys = {
6255
	.css_alloc = mem_cgroup_css_alloc,
6256
	.css_online = mem_cgroup_css_online,
6257
	.css_offline = mem_cgroup_css_offline,
6258
	.css_released = mem_cgroup_css_released,
6259
	.css_free = mem_cgroup_css_free,
6260
	.css_reset = mem_cgroup_css_reset,
6261 6262
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
6263
	.post_attach = mem_cgroup_move_task,
6264
	.bind = mem_cgroup_bind,
6265 6266
	.dfl_cftypes = memory_files,
	.legacy_cftypes = mem_cgroup_legacy_files,
6267
	.early_init = 0,
B
Balbir Singh 已提交
6268
};
6269

6270
/**
R
Roman Gushchin 已提交
6271
 * mem_cgroup_protected - check if memory consumption is in the normal range
6272
 * @root: the top ancestor of the sub-tree being checked
6273 6274
 * @memcg: the memory cgroup to check
 *
6275 6276
 * WARNING: This function is not stateless! It can only be used as part
 *          of a top-down tree iteration, not for isolated queries.
6277
 *
R
Roman Gushchin 已提交
6278 6279 6280 6281 6282
 * Returns one of the following:
 *   MEMCG_PROT_NONE: cgroup memory is not protected
 *   MEMCG_PROT_LOW: cgroup memory is protected as long there is
 *     an unprotected supply of reclaimable memory from other cgroups.
 *   MEMCG_PROT_MIN: cgroup memory is protected
6283
 *
R
Roman Gushchin 已提交
6284
 * @root is exclusive; it is never protected when looked at directly
6285
 *
R
Roman Gushchin 已提交
6286 6287 6288
 * To provide a proper hierarchical behavior, effective memory.min/low values
 * are used. Below is the description of how effective memory.low is calculated.
 * Effective memory.min values is calculated in the same way.
6289
 *
6290 6291 6292 6293 6294 6295 6296
 * Effective memory.low is always equal or less than the original memory.low.
 * If there is no memory.low overcommittment (which is always true for
 * top-level memory cgroups), these two values are equal.
 * Otherwise, it's a part of parent's effective memory.low,
 * calculated as a cgroup's memory.low usage divided by sum of sibling's
 * memory.low usages, where memory.low usage is the size of actually
 * protected memory.
6297
 *
6298 6299 6300
 *                                             low_usage
 * elow = min( memory.low, parent->elow * ------------------ ),
 *                                        siblings_low_usage
6301
 *
6302 6303
 *             | memory.current, if memory.current < memory.low
 * low_usage = |
6304
 *	       | 0, otherwise.
6305
 *
6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332
 *
 * Such definition of the effective memory.low provides the expected
 * hierarchical behavior: parent's memory.low value is limiting
 * children, unprotected memory is reclaimed first and cgroups,
 * which are not using their guarantee do not affect actual memory
 * distribution.
 *
 * For example, if there are memcgs A, A/B, A/C, A/D and A/E:
 *
 *     A      A/memory.low = 2G, A/memory.current = 6G
 *    //\\
 *   BC  DE   B/memory.low = 3G  B/memory.current = 2G
 *            C/memory.low = 1G  C/memory.current = 2G
 *            D/memory.low = 0   D/memory.current = 2G
 *            E/memory.low = 10G E/memory.current = 0
 *
 * and the memory pressure is applied, the following memory distribution
 * is expected (approximately):
 *
 *     A/memory.current = 2G
 *
 *     B/memory.current = 1.3G
 *     C/memory.current = 0.6G
 *     D/memory.current = 0
 *     E/memory.current = 0
 *
 * These calculations require constant tracking of the actual low usages
R
Roman Gushchin 已提交
6333 6334
 * (see propagate_protected_usage()), as well as recursive calculation of
 * effective memory.low values. But as we do call mem_cgroup_protected()
6335 6336 6337 6338
 * path for each memory cgroup top-down from the reclaim,
 * it's possible to optimize this part, and save calculated elow
 * for next usage. This part is intentionally racy, but it's ok,
 * as memory.low is a best-effort mechanism.
6339
 */
R
Roman Gushchin 已提交
6340 6341
enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
						struct mem_cgroup *memcg)
6342
{
6343
	struct mem_cgroup *parent;
R
Roman Gushchin 已提交
6344 6345 6346
	unsigned long emin, parent_emin;
	unsigned long elow, parent_elow;
	unsigned long usage;
6347

6348
	if (mem_cgroup_disabled())
R
Roman Gushchin 已提交
6349
		return MEMCG_PROT_NONE;
6350

6351 6352 6353
	if (!root)
		root = root_mem_cgroup;
	if (memcg == root)
R
Roman Gushchin 已提交
6354
		return MEMCG_PROT_NONE;
6355

6356
	usage = page_counter_read(&memcg->memory);
R
Roman Gushchin 已提交
6357 6358 6359 6360 6361
	if (!usage)
		return MEMCG_PROT_NONE;

	emin = memcg->memory.min;
	elow = memcg->memory.low;
6362

R
Roman Gushchin 已提交
6363
	parent = parent_mem_cgroup(memcg);
6364 6365 6366 6367
	/* No parent means a non-hierarchical mode on v1 memcg */
	if (!parent)
		return MEMCG_PROT_NONE;

6368 6369 6370
	if (parent == root)
		goto exit;

R
Roman Gushchin 已提交
6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384
	parent_emin = READ_ONCE(parent->memory.emin);
	emin = min(emin, parent_emin);
	if (emin && parent_emin) {
		unsigned long min_usage, siblings_min_usage;

		min_usage = min(usage, memcg->memory.min);
		siblings_min_usage = atomic_long_read(
			&parent->memory.children_min_usage);

		if (min_usage && siblings_min_usage)
			emin = min(emin, parent_emin * min_usage /
				   siblings_min_usage);
	}

6385 6386
	parent_elow = READ_ONCE(parent->memory.elow);
	elow = min(elow, parent_elow);
R
Roman Gushchin 已提交
6387 6388
	if (elow && parent_elow) {
		unsigned long low_usage, siblings_low_usage;
6389

R
Roman Gushchin 已提交
6390 6391 6392
		low_usage = min(usage, memcg->memory.low);
		siblings_low_usage = atomic_long_read(
			&parent->memory.children_low_usage);
6393

R
Roman Gushchin 已提交
6394 6395 6396 6397
		if (low_usage && siblings_low_usage)
			elow = min(elow, parent_elow * low_usage /
				   siblings_low_usage);
	}
6398 6399

exit:
R
Roman Gushchin 已提交
6400
	memcg->memory.emin = emin;
6401
	memcg->memory.elow = elow;
R
Roman Gushchin 已提交
6402 6403 6404 6405 6406 6407 6408

	if (usage <= emin)
		return MEMCG_PROT_MIN;
	else if (usage <= elow)
		return MEMCG_PROT_LOW;
	else
		return MEMCG_PROT_NONE;
6409 6410
}

6411 6412 6413 6414 6415 6416
/**
 * mem_cgroup_try_charge - try charging a page
 * @page: page to charge
 * @mm: mm context of the victim
 * @gfp_mask: reclaim mode
 * @memcgp: charged memcg return
6417
 * @compound: charge the page as compound or small page
6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429
 *
 * Try to charge @page to the memcg that @mm belongs to, reclaiming
 * pages according to @gfp_mask if necessary.
 *
 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
 * Otherwise, an error code is returned.
 *
 * After page->mapping has been set up, the caller must finalize the
 * charge with mem_cgroup_commit_charge().  Or abort the transaction
 * with mem_cgroup_cancel_charge() in case page instantiation fails.
 */
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
6430 6431
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound)
6432 6433
{
	struct mem_cgroup *memcg = NULL;
6434
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447
	int ret = 0;

	if (mem_cgroup_disabled())
		goto out;

	if (PageSwapCache(page)) {
		/*
		 * Every swap fault against a single page tries to charge the
		 * page, bail as early as possible.  shmem_unuse() encounters
		 * already charged pages, too.  The USED bit is protected by
		 * the page lock, which serializes swap cache removal, which
		 * in turn serializes uncharging.
		 */
6448
		VM_BUG_ON_PAGE(!PageLocked(page), page);
6449
		if (compound_head(page)->mem_cgroup)
6450
			goto out;
6451

6452
		if (do_swap_account) {
6453 6454 6455 6456 6457 6458 6459 6460 6461
			swp_entry_t ent = { .val = page_private(page), };
			unsigned short id = lookup_swap_cgroup_id(ent);

			rcu_read_lock();
			memcg = mem_cgroup_from_id(id);
			if (memcg && !css_tryget_online(&memcg->css))
				memcg = NULL;
			rcu_read_unlock();
		}
6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474
	}

	if (!memcg)
		memcg = get_mem_cgroup_from_mm(mm);

	ret = try_charge(memcg, gfp_mask, nr_pages);

	css_put(&memcg->css);
out:
	*memcgp = memcg;
	return ret;
}

6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487
int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound)
{
	struct mem_cgroup *memcg;
	int ret;

	ret = mem_cgroup_try_charge(page, mm, gfp_mask, memcgp, compound);
	memcg = *memcgp;
	mem_cgroup_throttle_swaprate(memcg, page_to_nid(page), gfp_mask);
	return ret;
}

6488 6489 6490 6491 6492
/**
 * mem_cgroup_commit_charge - commit a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 * @lrucare: page might be on LRU already
6493
 * @compound: charge the page as compound or small page
6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505
 *
 * Finalize a charge transaction started by mem_cgroup_try_charge(),
 * after page->mapping has been set up.  This must happen atomically
 * as part of the page instantiation, i.e. under the page table lock
 * for anonymous pages, under the page lock for page and swap cache.
 *
 * In addition, the page must not be on the LRU during the commit, to
 * prevent racing with task migration.  If it might be, use @lrucare.
 *
 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
 */
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
6506
			      bool lrucare, bool compound)
6507
{
6508
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522

	VM_BUG_ON_PAGE(!page->mapping, page);
	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

6523 6524 6525
	commit_charge(page, memcg, lrucare);

	local_irq_disable();
6526
	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
6527 6528
	memcg_check_events(memcg, page);
	local_irq_enable();
6529

6530
	if (do_memsw_account() && PageSwapCache(page)) {
6531 6532 6533 6534 6535 6536
		swp_entry_t entry = { .val = page_private(page) };
		/*
		 * The swap entry might not get freed for a long time,
		 * let's not wait for it.  The page already received a
		 * memory+swap charge, drop the swap entry duplicate.
		 */
6537
		mem_cgroup_uncharge_swap(entry, nr_pages);
6538 6539 6540 6541 6542 6543 6544
	}
}

/**
 * mem_cgroup_cancel_charge - cancel a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
6545
 * @compound: charge the page as compound or small page
6546 6547 6548
 *
 * Cancel a charge transaction started by mem_cgroup_try_charge().
 */
6549 6550
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound)
6551
{
6552
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

	cancel_charge(memcg, nr_pages);
}

6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578
struct uncharge_gather {
	struct mem_cgroup *memcg;
	unsigned long pgpgout;
	unsigned long nr_anon;
	unsigned long nr_file;
	unsigned long nr_kmem;
	unsigned long nr_huge;
	unsigned long nr_shmem;
	struct page *dummy_page;
};

static inline void uncharge_gather_clear(struct uncharge_gather *ug)
6579
{
6580 6581 6582 6583 6584 6585
	memset(ug, 0, sizeof(*ug));
}

static void uncharge_batch(const struct uncharge_gather *ug)
{
	unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
6586 6587
	unsigned long flags;

6588 6589
	if (!mem_cgroup_is_root(ug->memcg)) {
		page_counter_uncharge(&ug->memcg->memory, nr_pages);
6590
		if (do_memsw_account())
6591 6592 6593 6594
			page_counter_uncharge(&ug->memcg->memsw, nr_pages);
		if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
			page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
		memcg_oom_recover(ug->memcg);
6595
	}
6596 6597

	local_irq_save(flags);
6598 6599 6600 6601 6602
	__mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
	__mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
	__mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
	__mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
	__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
6603
	__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, nr_pages);
6604
	memcg_check_events(ug->memcg, ug->dummy_page);
6605
	local_irq_restore(flags);
6606

6607 6608 6609 6610 6611 6612 6613
	if (!mem_cgroup_is_root(ug->memcg))
		css_put_many(&ug->memcg->css, nr_pages);
}

static void uncharge_page(struct page *page, struct uncharge_gather *ug)
{
	VM_BUG_ON_PAGE(PageLRU(page), page);
6614 6615
	VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) &&
			!PageHWPoison(page) , page);
6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637

	if (!page->mem_cgroup)
		return;

	/*
	 * Nobody should be changing or seriously looking at
	 * page->mem_cgroup at this point, we have fully
	 * exclusive access to the page.
	 */

	if (ug->memcg != page->mem_cgroup) {
		if (ug->memcg) {
			uncharge_batch(ug);
			uncharge_gather_clear(ug);
		}
		ug->memcg = page->mem_cgroup;
	}

	if (!PageKmemcg(page)) {
		unsigned int nr_pages = 1;

		if (PageTransHuge(page)) {
6638
			nr_pages = compound_nr(page);
6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649
			ug->nr_huge += nr_pages;
		}
		if (PageAnon(page))
			ug->nr_anon += nr_pages;
		else {
			ug->nr_file += nr_pages;
			if (PageSwapBacked(page))
				ug->nr_shmem += nr_pages;
		}
		ug->pgpgout++;
	} else {
6650
		ug->nr_kmem += compound_nr(page);
6651 6652 6653 6654 6655
		__ClearPageKmemcg(page);
	}

	ug->dummy_page = page;
	page->mem_cgroup = NULL;
6656 6657 6658 6659
}

static void uncharge_list(struct list_head *page_list)
{
6660
	struct uncharge_gather ug;
6661
	struct list_head *next;
6662 6663

	uncharge_gather_clear(&ug);
6664

6665 6666 6667 6668
	/*
	 * Note that the list can be a single page->lru; hence the
	 * do-while loop instead of a simple list_for_each_entry().
	 */
6669 6670
	next = page_list->next;
	do {
6671 6672
		struct page *page;

6673 6674 6675
		page = list_entry(next, struct page, lru);
		next = page->lru.next;

6676
		uncharge_page(page, &ug);
6677 6678
	} while (next != page_list);

6679 6680
	if (ug.memcg)
		uncharge_batch(&ug);
6681 6682
}

6683 6684 6685 6686 6687 6688 6689 6690 6691
/**
 * mem_cgroup_uncharge - uncharge a page
 * @page: page to uncharge
 *
 * Uncharge a page previously charged with mem_cgroup_try_charge() and
 * mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge(struct page *page)
{
6692 6693
	struct uncharge_gather ug;

6694 6695 6696
	if (mem_cgroup_disabled())
		return;

6697
	/* Don't touch page->lru of any random page, pre-check: */
6698
	if (!page->mem_cgroup)
6699 6700
		return;

6701 6702 6703
	uncharge_gather_clear(&ug);
	uncharge_page(page, &ug);
	uncharge_batch(&ug);
6704
}
6705

6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716
/**
 * mem_cgroup_uncharge_list - uncharge a list of page
 * @page_list: list of pages to uncharge
 *
 * Uncharge a list of pages previously charged with
 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge_list(struct list_head *page_list)
{
	if (mem_cgroup_disabled())
		return;
6717

6718 6719
	if (!list_empty(page_list))
		uncharge_list(page_list);
6720 6721 6722
}

/**
6723 6724 6725
 * mem_cgroup_migrate - charge a page's replacement
 * @oldpage: currently circulating page
 * @newpage: replacement page
6726
 *
6727 6728
 * Charge @newpage as a replacement page for @oldpage. @oldpage will
 * be uncharged upon free.
6729 6730 6731
 *
 * Both pages must be locked, @newpage->mapping must be set up.
 */
6732
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
6733
{
6734
	struct mem_cgroup *memcg;
6735 6736
	unsigned int nr_pages;
	bool compound;
6737
	unsigned long flags;
6738 6739 6740 6741

	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
6742 6743
	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
		       newpage);
6744 6745 6746 6747 6748

	if (mem_cgroup_disabled())
		return;

	/* Page cache replacement: new page already charged? */
6749
	if (newpage->mem_cgroup)
6750 6751
		return;

6752
	/* Swapcache readahead pages can get replaced before being charged */
6753
	memcg = oldpage->mem_cgroup;
6754
	if (!memcg)
6755 6756
		return;

6757 6758 6759 6760 6761 6762 6763 6764
	/* Force-charge the new page. The old one will be freed soon */
	compound = PageTransHuge(newpage);
	nr_pages = compound ? hpage_nr_pages(newpage) : 1;

	page_counter_charge(&memcg->memory, nr_pages);
	if (do_memsw_account())
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);
6765

6766
	commit_charge(newpage, memcg, false);
6767

6768
	local_irq_save(flags);
6769 6770
	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
	memcg_check_events(memcg, newpage);
6771
	local_irq_restore(flags);
6772 6773
}

6774
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
6775 6776
EXPORT_SYMBOL(memcg_sockets_enabled_key);

6777
void mem_cgroup_sk_alloc(struct sock *sk)
6778 6779 6780
{
	struct mem_cgroup *memcg;

6781 6782 6783
	if (!mem_cgroup_sockets_enabled)
		return;

6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797
	/*
	 * Socket cloning can throw us here with sk_memcg already
	 * filled. It won't however, necessarily happen from
	 * process context. So the test for root memcg given
	 * the current task's memcg won't help us in this case.
	 *
	 * Respecting the original socket's memcg is a better
	 * decision in this case.
	 */
	if (sk->sk_memcg) {
		css_get(&sk->sk_memcg->css);
		return;
	}

6798 6799
	rcu_read_lock();
	memcg = mem_cgroup_from_task(current);
6800 6801
	if (memcg == root_mem_cgroup)
		goto out;
6802
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
6803 6804
		goto out;
	if (css_tryget_online(&memcg->css))
6805
		sk->sk_memcg = memcg;
6806
out:
6807 6808 6809
	rcu_read_unlock();
}

6810
void mem_cgroup_sk_free(struct sock *sk)
6811
{
6812 6813
	if (sk->sk_memcg)
		css_put(&sk->sk_memcg->css);
6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825
}

/**
 * mem_cgroup_charge_skmem - charge socket memory
 * @memcg: memcg to charge
 * @nr_pages: number of pages to charge
 *
 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
 * @memcg's configured limit, %false if the charge had to be forced.
 */
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
6826
	gfp_t gfp_mask = GFP_KERNEL;
6827

6828
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6829
		struct page_counter *fail;
6830

6831 6832
		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
			memcg->tcpmem_pressure = 0;
6833 6834
			return true;
		}
6835 6836
		page_counter_charge(&memcg->tcpmem, nr_pages);
		memcg->tcpmem_pressure = 1;
6837
		return false;
6838
	}
6839

6840 6841 6842 6843
	/* Don't block in the packet receive path */
	if (in_softirq())
		gfp_mask = GFP_NOWAIT;

6844
	mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
6845

6846 6847 6848 6849
	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
		return true;

	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
6850 6851 6852 6853 6854
	return false;
}

/**
 * mem_cgroup_uncharge_skmem - uncharge socket memory
M
Mike Rapoport 已提交
6855 6856
 * @memcg: memcg to uncharge
 * @nr_pages: number of pages to uncharge
6857 6858 6859
 */
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
6860
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
6861
		page_counter_uncharge(&memcg->tcpmem, nr_pages);
6862 6863
		return;
	}
6864

6865
	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
6866

6867
	refill_stock(memcg, nr_pages);
6868 6869
}

6870 6871 6872 6873 6874 6875 6876 6877 6878
static int __init cgroup_memory(char *s)
{
	char *token;

	while ((token = strsep(&s, ",")) != NULL) {
		if (!*token)
			continue;
		if (!strcmp(token, "nosocket"))
			cgroup_memory_nosocket = true;
6879 6880
		if (!strcmp(token, "nokmem"))
			cgroup_memory_nokmem = true;
6881 6882 6883 6884
	}
	return 0;
}
__setup("cgroup.memory=", cgroup_memory);
6885

6886
/*
6887 6888
 * subsys_initcall() for memory controller.
 *
6889 6890 6891 6892
 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
 * basically everything that doesn't depend on a specific mem_cgroup structure
 * should be initialized from here.
6893 6894 6895
 */
static int __init mem_cgroup_init(void)
{
6896 6897
	int cpu, node;

6898
#ifdef CONFIG_MEMCG_KMEM
6899 6900
	/*
	 * Kmem cache creation is mostly done with the slab_mutex held,
6901 6902 6903
	 * so use a workqueue with limited concurrency to avoid stalling
	 * all worker threads in case lots of cgroups are created and
	 * destroyed simultaneously.
6904
	 */
6905 6906
	memcg_kmem_cache_wq = alloc_workqueue("memcg_kmem_cache", 0, 1);
	BUG_ON(!memcg_kmem_cache_wq);
6907 6908
#endif

6909 6910
	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
				  memcg_hotplug_cpu_dead);
6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921

	for_each_possible_cpu(cpu)
		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
			  drain_local_stock);

	for_each_node(node) {
		struct mem_cgroup_tree_per_node *rtpn;

		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
				    node_online(node) ? node : NUMA_NO_NODE);

6922
		rtpn->rb_root = RB_ROOT;
6923
		rtpn->rb_rightmost = NULL;
6924
		spin_lock_init(&rtpn->lock);
6925 6926 6927
		soft_limit_tree.rb_tree_per_node[node] = rtpn;
	}

6928 6929 6930
	return 0;
}
subsys_initcall(mem_cgroup_init);
6931 6932

#ifdef CONFIG_MEMCG_SWAP
6933 6934
static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
{
6935
	while (!refcount_inc_not_zero(&memcg->id.ref)) {
6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950
		/*
		 * The root cgroup cannot be destroyed, so it's refcount must
		 * always be >= 1.
		 */
		if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
			VM_BUG_ON(1);
			break;
		}
		memcg = parent_mem_cgroup(memcg);
		if (!memcg)
			memcg = root_mem_cgroup;
	}
	return memcg;
}

6951 6952 6953 6954 6955 6956 6957 6958 6959
/**
 * mem_cgroup_swapout - transfer a memsw charge to swap
 * @page: page whose memsw charge to transfer
 * @entry: swap entry to move the charge to
 *
 * Transfer the memsw charge of @page to @entry.
 */
void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
{
6960
	struct mem_cgroup *memcg, *swap_memcg;
6961
	unsigned int nr_entries;
6962 6963 6964 6965 6966
	unsigned short oldid;

	VM_BUG_ON_PAGE(PageLRU(page), page);
	VM_BUG_ON_PAGE(page_count(page), page);

6967
	if (!do_memsw_account())
6968 6969 6970 6971 6972 6973 6974 6975
		return;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return;

6976 6977 6978 6979 6980 6981
	/*
	 * In case the memcg owning these pages has been offlined and doesn't
	 * have an ID allocated to it anymore, charge the closest online
	 * ancestor for the swap instead and transfer the memory+swap charge.
	 */
	swap_memcg = mem_cgroup_id_get_online(memcg);
6982 6983 6984 6985 6986 6987
	nr_entries = hpage_nr_pages(page);
	/* Get references for the tail pages, too */
	if (nr_entries > 1)
		mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
				   nr_entries);
6988
	VM_BUG_ON_PAGE(oldid, page);
6989
	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
6990 6991 6992 6993

	page->mem_cgroup = NULL;

	if (!mem_cgroup_is_root(memcg))
6994
		page_counter_uncharge(&memcg->memory, nr_entries);
6995

6996 6997
	if (memcg != swap_memcg) {
		if (!mem_cgroup_is_root(swap_memcg))
6998 6999
			page_counter_charge(&swap_memcg->memsw, nr_entries);
		page_counter_uncharge(&memcg->memsw, nr_entries);
7000 7001
	}

7002 7003
	/*
	 * Interrupts should be disabled here because the caller holds the
M
Matthew Wilcox 已提交
7004
	 * i_pages lock which is taken with interrupts-off. It is
7005
	 * important here to have the interrupts disabled because it is the
M
Matthew Wilcox 已提交
7006
	 * only synchronisation we have for updating the per-CPU variables.
7007 7008
	 */
	VM_BUG_ON(!irqs_disabled());
7009 7010
	mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
				     -nr_entries);
7011
	memcg_check_events(memcg, page);
7012 7013

	if (!mem_cgroup_is_root(memcg))
7014
		css_put_many(&memcg->css, nr_entries);
7015 7016
}

7017 7018
/**
 * mem_cgroup_try_charge_swap - try charging swap space for a page
7019 7020 7021
 * @page: page being added to swap
 * @entry: swap entry to charge
 *
7022
 * Try to charge @page's memcg for the swap space at @entry.
7023 7024 7025 7026 7027
 *
 * Returns 0 on success, -ENOMEM on failure.
 */
int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
{
7028
	unsigned int nr_pages = hpage_nr_pages(page);
7029
	struct page_counter *counter;
7030
	struct mem_cgroup *memcg;
7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041
	unsigned short oldid;

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
		return 0;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return 0;

7042 7043
	if (!entry.val) {
		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7044
		return 0;
7045
	}
7046

7047 7048
	memcg = mem_cgroup_id_get_online(memcg);

7049
	if (!mem_cgroup_is_root(memcg) &&
7050
	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
7051 7052
		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
7053
		mem_cgroup_id_put(memcg);
7054
		return -ENOMEM;
7055
	}
7056

7057 7058 7059 7060
	/* Get references for the tail pages, too */
	if (nr_pages > 1)
		mem_cgroup_id_get_many(memcg, nr_pages - 1);
	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
7061
	VM_BUG_ON_PAGE(oldid, page);
7062
	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
7063 7064 7065 7066

	return 0;
}

7067
/**
7068
 * mem_cgroup_uncharge_swap - uncharge swap space
7069
 * @entry: swap entry to uncharge
7070
 * @nr_pages: the amount of swap space to uncharge
7071
 */
7072
void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
7073 7074 7075 7076
{
	struct mem_cgroup *memcg;
	unsigned short id;

7077
	if (!do_swap_account)
7078 7079
		return;

7080
	id = swap_cgroup_record(entry, 0, nr_pages);
7081
	rcu_read_lock();
7082
	memcg = mem_cgroup_from_id(id);
7083
	if (memcg) {
7084 7085
		if (!mem_cgroup_is_root(memcg)) {
			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
7086
				page_counter_uncharge(&memcg->swap, nr_pages);
7087
			else
7088
				page_counter_uncharge(&memcg->memsw, nr_pages);
7089
		}
7090
		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
7091
		mem_cgroup_id_put_many(memcg, nr_pages);
7092 7093 7094 7095
	}
	rcu_read_unlock();
}

7096 7097 7098 7099 7100 7101 7102 7103
long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
	long nr_swap_pages = get_nr_swap_pages();

	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return nr_swap_pages;
	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
		nr_swap_pages = min_t(long, nr_swap_pages,
7104
				      READ_ONCE(memcg->swap.max) -
7105 7106 7107 7108
				      page_counter_read(&memcg->swap));
	return nr_swap_pages;
}

7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124
bool mem_cgroup_swap_full(struct page *page)
{
	struct mem_cgroup *memcg;

	VM_BUG_ON_PAGE(!PageLocked(page), page);

	if (vm_swap_full())
		return true;
	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return false;

	memcg = page->mem_cgroup;
	if (!memcg)
		return false;

	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7125
		if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.max)
7126 7127 7128 7129 7130
			return true;

	return false;
}

7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147
/* for remember boot option*/
#ifdef CONFIG_MEMCG_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata;
#endif

static int __init enable_swap_account(char *s)
{
	if (!strcmp(s, "1"))
		really_do_swap_account = 1;
	else if (!strcmp(s, "0"))
		really_do_swap_account = 0;
	return 1;
}
__setup("swapaccount=", enable_swap_account);

7148 7149 7150 7151 7152 7153 7154 7155 7156 7157
static u64 swap_current_read(struct cgroup_subsys_state *css,
			     struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
}

static int swap_max_show(struct seq_file *m, void *v)
{
7158 7159
	return seq_puts_memcg_tunable(m,
		READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173
}

static ssize_t swap_max_write(struct kernfs_open_file *of,
			      char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long max;
	int err;

	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &max);
	if (err)
		return err;

7174
	xchg(&memcg->swap.max, max);
7175 7176 7177 7178

	return nbytes;
}

7179 7180
static int swap_events_show(struct seq_file *m, void *v)
{
7181
	struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
7182 7183 7184 7185 7186 7187 7188 7189 7190

	seq_printf(m, "max %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
	seq_printf(m, "fail %lu\n",
		   atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));

	return 0;
}

7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202
static struct cftype swap_files[] = {
	{
		.name = "swap.current",
		.flags = CFTYPE_NOT_ON_ROOT,
		.read_u64 = swap_current_read,
	},
	{
		.name = "swap.max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = swap_max_show,
		.write = swap_max_write,
	},
7203 7204 7205 7206 7207 7208
	{
		.name = "swap.events",
		.flags = CFTYPE_NOT_ON_ROOT,
		.file_offset = offsetof(struct mem_cgroup, swap_events_file),
		.seq_show = swap_events_show,
	},
7209 7210 7211
	{ }	/* terminate */
};

7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{ },	/* terminate */
};

static int __init mem_cgroup_swap_init(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
7243 7244
		WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
					       swap_files));
7245 7246 7247 7248 7249 7250 7251 7252
		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
						  memsw_cgroup_files));
	}
	return 0;
}
subsys_initcall(mem_cgroup_swap_init);

#endif /* CONFIG_MEMCG_SWAP */