memcontrol.c 151.4 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
13 14 15 16
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
17 18 19 20 21 22
 * Native page reclaim
 * Charge lifetime sanitation
 * Lockless page tracking & accounting
 * Unified hierarchy configuration model
 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
 *
B
Balbir Singh 已提交
23 24 25 26 27 28 29 30 31 32 33
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

34
#include <linux/page_counter.h>
B
Balbir Singh 已提交
35 36
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
37
#include <linux/mm.h>
38
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
39
#include <linux/pagemap.h>
40
#include <linux/smp.h>
41
#include <linux/page-flags.h>
42
#include <linux/backing-dev.h>
43 44
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
45
#include <linux/limits.h>
46
#include <linux/export.h>
47
#include <linux/mutex.h>
48
#include <linux/rbtree.h>
49
#include <linux/slab.h>
50
#include <linux/swap.h>
51
#include <linux/swapops.h>
52
#include <linux/spinlock.h>
53
#include <linux/eventfd.h>
54
#include <linux/poll.h>
55
#include <linux/sort.h>
56
#include <linux/fs.h>
57
#include <linux/seq_file.h>
58
#include <linux/vmpressure.h>
59
#include <linux/mm_inline.h>
60
#include <linux/swap_cgroup.h>
61
#include <linux/cpu.h>
62
#include <linux/oom.h>
63
#include <linux/lockdep.h>
64
#include <linux/file.h>
65
#include <linux/tracehook.h>
K
KAMEZAWA Hiroyuki 已提交
66
#include "internal.h"
G
Glauber Costa 已提交
67
#include <net/sock.h>
M
Michal Hocko 已提交
68
#include <net/ip.h>
69
#include "slab.h"
B
Balbir Singh 已提交
70

71 72
#include <asm/uaccess.h>

73 74
#include <trace/events/vmscan.h>

75 76
struct cgroup_subsys memory_cgrp_subsys __read_mostly;
EXPORT_SYMBOL(memory_cgrp_subsys);
77

78 79
struct mem_cgroup *root_mem_cgroup __read_mostly;

80
#define MEM_CGROUP_RECLAIM_RETRIES	5
B
Balbir Singh 已提交
81

82 83 84
/* Socket memory accounting disabled? */
static bool cgroup_memory_nosocket;

85 86 87
/* Kernel memory accounting disabled? */
static bool cgroup_memory_nokmem;

88
/* Whether the swap controller is active */
A
Andrew Morton 已提交
89
#ifdef CONFIG_MEMCG_SWAP
90 91
int do_swap_account __read_mostly;
#else
92
#define do_swap_account		0
93 94
#endif

95 96 97 98 99 100
/* Whether legacy memory+swap accounting is active */
static bool do_memsw_account(void)
{
	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
}

101 102 103
static const char * const mem_cgroup_stat_names[] = {
	"cache",
	"rss",
104
	"rss_huge",
105
	"mapped_file",
106
	"dirty",
107
	"writeback",
108 109 110 111 112 113 114 115 116 117
	"swap",
};

static const char * const mem_cgroup_events_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

118 119 120 121 122 123 124 125
static const char * const mem_cgroup_lru_names[] = {
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

126 127 128
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
129

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

K
KAMEZAWA Hiroyuki 已提交
150 151 152 153 154
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
155

156 157 158
/*
 * cgroup_event represents events which userspace want to receive.
 */
159
struct mem_cgroup_event {
160
	/*
161
	 * memcg which the event belongs to.
162
	 */
163
	struct mem_cgroup *memcg;
164 165 166 167 168 169 170 171
	/*
	 * eventfd to signal userspace about the event.
	 */
	struct eventfd_ctx *eventfd;
	/*
	 * Each of these stored in a list by the cgroup.
	 */
	struct list_head list;
172 173 174 175 176
	/*
	 * register_event() callback will be used to add new userspace
	 * waiter for changes related to this event.  Use eventfd_signal()
	 * on eventfd to send notification to userspace.
	 */
177
	int (*register_event)(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
178
			      struct eventfd_ctx *eventfd, const char *args);
179 180 181 182 183
	/*
	 * unregister_event() callback will be called when userspace closes
	 * the eventfd or on cgroup removing.  This callback must be set,
	 * if you want provide notification functionality.
	 */
184
	void (*unregister_event)(struct mem_cgroup *memcg,
185
				 struct eventfd_ctx *eventfd);
186 187 188 189 190 191 192 193 194 195
	/*
	 * All fields below needed to unregister event when
	 * userspace closes eventfd.
	 */
	poll_table pt;
	wait_queue_head_t *wqh;
	wait_queue_t wait;
	struct work_struct remove;
};

196 197
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
198

199 200
/* Stuffs for move charges at task migration. */
/*
201
 * Types of charges to be moved.
202
 */
203 204 205
#define MOVE_ANON	0x1U
#define MOVE_FILE	0x2U
#define MOVE_MASK	(MOVE_ANON | MOVE_FILE)
206

207 208
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
209
	spinlock_t	  lock; /* for from, to */
210
	struct mm_struct  *mm;
211 212
	struct mem_cgroup *from;
	struct mem_cgroup *to;
213
	unsigned long flags;
214
	unsigned long precharge;
215
	unsigned long moved_charge;
216
	unsigned long moved_swap;
217 218 219
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
220
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
221 222
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
223

224 225 226 227
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
228
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
229
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
230

231 232
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
233
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
234
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
235
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
236 237 238
	NR_CHARGE_TYPE,
};

239
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
240 241 242 243
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
244
	_KMEM,
V
Vladimir Davydov 已提交
245
	_TCP,
G
Glauber Costa 已提交
246 247
};

248 249
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
250
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
251 252
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
253

254 255 256 257 258 259 260 261 262 263 264 265 266
/* Some nice accessors for the vmpressure. */
struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
{
	if (!memcg)
		memcg = root_mem_cgroup;
	return &memcg->vmpressure;
}

struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
{
	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}

267 268 269 270 271
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

272
#ifndef CONFIG_SLOB
273
/*
274
 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
L
Li Zefan 已提交
275 276 277 278 279
 * The main reason for not using cgroup id for this:
 *  this works better in sparse environments, where we have a lot of memcgs,
 *  but only a few kmem-limited. Or also, if we have, for instance, 200
 *  memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *  200 entry array for that.
280
 *
281 282
 * The current size of the caches array is stored in memcg_nr_cache_ids. It
 * will double each time we have to increase it.
283
 */
284 285
static DEFINE_IDA(memcg_cache_ida);
int memcg_nr_cache_ids;
286

287 288 289 290 291 292 293 294 295 296 297 298 299
/* Protects memcg_nr_cache_ids */
static DECLARE_RWSEM(memcg_cache_ids_sem);

void memcg_get_cache_ids(void)
{
	down_read(&memcg_cache_ids_sem);
}

void memcg_put_cache_ids(void)
{
	up_read(&memcg_cache_ids_sem);
}

300 301 302 303 304 305
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
L
Li Zefan 已提交
306
 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
307 308
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
L
Li Zefan 已提交
309
 * cgrp_id space is not getting any smaller, and we don't have to necessarily
310 311 312
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
L
Li Zefan 已提交
313
#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
314

315 316 317 318 319 320
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
321
DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
322
EXPORT_SYMBOL(memcg_kmem_enabled_key);
323

324
#endif /* !CONFIG_SLOB */
325

326
static struct mem_cgroup_per_zone *
327
mem_cgroup_zone_zoneinfo(struct mem_cgroup *memcg, struct zone *zone)
328
{
329 330 331
	int nid = zone_to_nid(zone);
	int zid = zone_idx(zone);

332
	return &memcg->nodeinfo[nid]->zoneinfo[zid];
333 334
}

335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
/**
 * mem_cgroup_css_from_page - css of the memcg associated with a page
 * @page: page of interest
 *
 * If memcg is bound to the default hierarchy, css of the memcg associated
 * with @page is returned.  The returned css remains associated with @page
 * until it is released.
 *
 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
 * is returned.
 */
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
{
	struct mem_cgroup *memcg;

	memcg = page->mem_cgroup;

352
	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
353 354 355 356 357
		memcg = root_mem_cgroup;

	return &memcg->css;
}

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
/**
 * page_cgroup_ino - return inode number of the memcg a page is charged to
 * @page: the page
 *
 * Look up the closest online ancestor of the memory cgroup @page is charged to
 * and return its inode number or 0 if @page is not charged to any cgroup. It
 * is safe to call this function without holding a reference to @page.
 *
 * Note, this function is inherently racy, because there is nothing to prevent
 * the cgroup inode from getting torn down and potentially reallocated a moment
 * after page_cgroup_ino() returns, so it only should be used by callers that
 * do not care (such as procfs interfaces).
 */
ino_t page_cgroup_ino(struct page *page)
{
	struct mem_cgroup *memcg;
	unsigned long ino = 0;

	rcu_read_lock();
	memcg = READ_ONCE(page->mem_cgroup);
	while (memcg && !(memcg->css.flags & CSS_ONLINE))
		memcg = parent_mem_cgroup(memcg);
	if (memcg)
		ino = cgroup_ino(memcg->css.cgroup);
	rcu_read_unlock();
	return ino;
}

386
static struct mem_cgroup_per_zone *
387
mem_cgroup_page_zoneinfo(struct mem_cgroup *memcg, struct page *page)
388
{
389 390
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
391

392
	return &memcg->nodeinfo[nid]->zoneinfo[zid];
393 394
}

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

410 411
static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_zone *mz,
					 struct mem_cgroup_tree_per_zone *mctz,
412
					 unsigned long new_usage_in_excess)
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
}

442 443
static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
					 struct mem_cgroup_tree_per_zone *mctz)
444 445 446 447 448 449 450
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

451 452
static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,
				       struct mem_cgroup_tree_per_zone *mctz)
453
{
454 455 456
	unsigned long flags;

	spin_lock_irqsave(&mctz->lock, flags);
457
	__mem_cgroup_remove_exceeded(mz, mctz);
458
	spin_unlock_irqrestore(&mctz->lock, flags);
459 460
}

461 462 463
static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
{
	unsigned long nr_pages = page_counter_read(&memcg->memory);
464
	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
465 466 467 468 469 470 471
	unsigned long excess = 0;

	if (nr_pages > soft_limit)
		excess = nr_pages - soft_limit;

	return excess;
}
472 473 474

static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
{
475
	unsigned long excess;
476 477 478
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

479
	mctz = soft_limit_tree_from_page(page);
480 481 482 483 484
	/*
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
	 */
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
485
		mz = mem_cgroup_page_zoneinfo(memcg, page);
486
		excess = soft_limit_excess(memcg);
487 488 489 490 491
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
		if (excess || mz->on_tree) {
492 493 494
			unsigned long flags;

			spin_lock_irqsave(&mctz->lock, flags);
495 496
			/* if on-tree, remove it */
			if (mz->on_tree)
497
				__mem_cgroup_remove_exceeded(mz, mctz);
498 499 500 501
			/*
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
			 */
502
			__mem_cgroup_insert_exceeded(mz, mctz, excess);
503
			spin_unlock_irqrestore(&mctz->lock, flags);
504 505 506 507 508 509 510
		}
	}
}

static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
{
	struct mem_cgroup_tree_per_zone *mctz;
511 512
	struct mem_cgroup_per_zone *mz;
	int nid, zid;
513

514 515 516 517
	for_each_node(nid) {
		for (zid = 0; zid < MAX_NR_ZONES; zid++) {
			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
			mctz = soft_limit_tree_node_zone(nid, zid);
518
			mem_cgroup_remove_exceeded(mz, mctz);
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
		}
	}
}

static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
	struct mem_cgroup_per_zone *mz;

retry:
	mz = NULL;
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
541
	__mem_cgroup_remove_exceeded(mz, mctz);
542
	if (!soft_limit_excess(mz->memcg) ||
543
	    !css_tryget_online(&mz->memcg->css))
544 545 546 547 548 549 550 551 552 553
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

554
	spin_lock_irq(&mctz->lock);
555
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
556
	spin_unlock_irq(&mctz->lock);
557 558 559
	return mz;
}

560
/*
561 562
 * Return page count for single (non recursive) @memcg.
 *
563 564 565 566 567
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
568
 * a periodic synchronization of counter in memcg's counter.
569 570 571 572 573 574 575 576 577
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
578
 * common workload, threshold and synchronization as vmstat[] should be
579 580
 * implemented.
 */
581 582
static unsigned long
mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
583
{
584
	long val = 0;
585 586
	int cpu;

587
	/* Per-cpu values can be negative, use a signed accumulator */
588
	for_each_possible_cpu(cpu)
589
		val += per_cpu(memcg->stat->count[idx], cpu);
590 591 592 593 594 595
	/*
	 * Summing races with updates, so val may be negative.  Avoid exposing
	 * transient negative values.
	 */
	if (val < 0)
		val = 0;
596 597 598
	return val;
}

599
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
600 601 602 603 604
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

605
	for_each_possible_cpu(cpu)
606
		val += per_cpu(memcg->stat->events[idx], cpu);
607 608 609
	return val;
}

610
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
611
					 struct page *page,
612
					 bool compound, int nr_pages)
613
{
614 615 616 617
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
618
	if (PageAnon(page))
619
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
620
				nr_pages);
621
	else
622
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
623
				nr_pages);
624

625 626
	if (compound) {
		VM_BUG_ON_PAGE(!PageTransHuge(page), page);
627 628
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
				nr_pages);
629
	}
630

631 632
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
633
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
634
	else {
635
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
636 637
		nr_pages = -nr_pages; /* for event */
	}
638

639
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
640 641
}

642 643
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
					   int nid, unsigned int lru_mask)
644
{
645
	unsigned long nr = 0;
646 647
	int zid;

648
	VM_BUG_ON((unsigned)nid >= nr_node_ids);
649

650 651 652 653 654 655 656 657 658 659 660 661
	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		struct mem_cgroup_per_zone *mz;
		enum lru_list lru;

		for_each_lru(lru) {
			if (!(BIT(lru) & lru_mask))
				continue;
			mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
			nr += mz->lru_size[lru];
		}
	}
	return nr;
662
}
663

664
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
665
			unsigned int lru_mask)
666
{
667
	unsigned long nr = 0;
668
	int nid;
669

670
	for_each_node_state(nid, N_MEMORY)
671 672
		nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
	return nr;
673 674
}

675 676
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
677 678 679
{
	unsigned long val, next;

680
	val = __this_cpu_read(memcg->stat->nr_page_events);
681
	next = __this_cpu_read(memcg->stat->targets[target]);
682
	/* from time_after() in jiffies.h */
683 684 685 686 687
	if ((long)next - (long)val < 0) {
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
688 689 690
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
691 692 693 694 695 696 697 698
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
		__this_cpu_write(memcg->stat->targets[target], next);
		return true;
699
	}
700
	return false;
701 702 703 704 705 706
}

/*
 * Check events in order.
 *
 */
707
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
708 709
{
	/* threshold event is triggered in finer grain than soft limit */
710 711
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
712
		bool do_softlimit;
713
		bool do_numainfo __maybe_unused;
714

715 716
		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
717 718 719 720
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
721
		mem_cgroup_threshold(memcg);
722 723
		if (unlikely(do_softlimit))
			mem_cgroup_update_tree(memcg, page);
724
#if MAX_NUMNODES > 1
725
		if (unlikely(do_numainfo))
726
			atomic_inc(&memcg->numainfo_events);
727
#endif
728
	}
729 730
}

731
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
732
{
733 734 735 736 737 738 739 740
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

741
	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
742
}
M
Michal Hocko 已提交
743
EXPORT_SYMBOL(mem_cgroup_from_task);
744

745
static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
746
{
747
	struct mem_cgroup *memcg = NULL;
748

749 750
	rcu_read_lock();
	do {
751 752 753 754 755 756
		/*
		 * Page cache insertions can happen withou an
		 * actual mm context, e.g. during disk probing
		 * on boot, loopback IO, acct() writes etc.
		 */
		if (unlikely(!mm))
757
			memcg = root_mem_cgroup;
758 759 760 761 762
		else {
			memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
			if (unlikely(!memcg))
				memcg = root_mem_cgroup;
		}
763
	} while (!css_tryget_online(&memcg->css));
764
	rcu_read_unlock();
765
	return memcg;
766 767
}

768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
 * Reclaimers can specify a zone and a priority level in @reclaim to
 * divide up the memcgs in the hierarchy among all concurrent
 * reclaimers operating on the same zone and priority.
 */
785
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
786
				   struct mem_cgroup *prev,
787
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
788
{
M
Michal Hocko 已提交
789
	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
790
	struct cgroup_subsys_state *css = NULL;
791
	struct mem_cgroup *memcg = NULL;
792
	struct mem_cgroup *pos = NULL;
793

794 795
	if (mem_cgroup_disabled())
		return NULL;
796

797 798
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
799

800
	if (prev && !reclaim)
801
		pos = prev;
K
KAMEZAWA Hiroyuki 已提交
802

803 804
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
805
			goto out;
806
		return root;
807
	}
K
KAMEZAWA Hiroyuki 已提交
808

809
	rcu_read_lock();
M
Michal Hocko 已提交
810

811 812 813 814 815 816 817 818 819
	if (reclaim) {
		struct mem_cgroup_per_zone *mz;

		mz = mem_cgroup_zone_zoneinfo(root, reclaim->zone);
		iter = &mz->iter[reclaim->priority];

		if (prev && reclaim->generation != iter->generation)
			goto out_unlock;

820
		while (1) {
821
			pos = READ_ONCE(iter->position);
822 823
			if (!pos || css_tryget(&pos->css))
				break;
824
			/*
825 826 827 828 829 830
			 * css reference reached zero, so iter->position will
			 * be cleared by ->css_released. However, we should not
			 * rely on this happening soon, because ->css_released
			 * is called from a work queue, and by busy-waiting we
			 * might block it. So we clear iter->position right
			 * away.
831
			 */
832 833
			(void)cmpxchg(&iter->position, pos, NULL);
		}
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
	}

	if (pos)
		css = &pos->css;

	for (;;) {
		css = css_next_descendant_pre(css, &root->css);
		if (!css) {
			/*
			 * Reclaimers share the hierarchy walk, and a
			 * new one might jump in right at the end of
			 * the hierarchy - make sure they see at least
			 * one group and restart from the beginning.
			 */
			if (!prev)
				continue;
			break;
851
		}
K
KAMEZAWA Hiroyuki 已提交
852

853 854 855 856 857 858
		/*
		 * Verify the css and acquire a reference.  The root
		 * is provided by the caller, so we know it's alive
		 * and kicking, and don't take an extra reference.
		 */
		memcg = mem_cgroup_from_css(css);
K
KAMEZAWA Hiroyuki 已提交
859

860 861
		if (css == &root->css)
			break;
K
KAMEZAWA Hiroyuki 已提交
862

863 864
		if (css_tryget(css))
			break;
865

866
		memcg = NULL;
867
	}
868 869 870

	if (reclaim) {
		/*
871 872 873
		 * The position could have already been updated by a competing
		 * thread, so check that the value hasn't changed since we read
		 * it to avoid reclaiming from the same cgroup twice.
874
		 */
875 876
		(void)cmpxchg(&iter->position, pos, memcg);

877 878 879 880 881 882 883
		if (pos)
			css_put(&pos->css);

		if (!memcg)
			iter->generation++;
		else if (!prev)
			reclaim->generation = iter->generation;
884
	}
885

886 887
out_unlock:
	rcu_read_unlock();
888
out:
889 890 891
	if (prev && prev != root)
		css_put(&prev->css);

892
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
893
}
K
KAMEZAWA Hiroyuki 已提交
894

895 896 897 898 899 900 901
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
902 903 904 905 906 907
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
908

909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
{
	struct mem_cgroup *memcg = dead_memcg;
	struct mem_cgroup_reclaim_iter *iter;
	struct mem_cgroup_per_zone *mz;
	int nid, zid;
	int i;

	while ((memcg = parent_mem_cgroup(memcg))) {
		for_each_node(nid) {
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
				mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
				for (i = 0; i <= DEF_PRIORITY; i++) {
					iter = &mz->iter[i];
					cmpxchg(&iter->position,
						dead_memcg, NULL);
				}
			}
		}
	}
}

931 932 933 934 935 936
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
937
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
938
	     iter != NULL;				\
939
	     iter = mem_cgroup_iter(root, iter, NULL))
940

941
#define for_each_mem_cgroup(iter)			\
942
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
943
	     iter != NULL;				\
944
	     iter = mem_cgroup_iter(NULL, iter, NULL))
K
KAMEZAWA Hiroyuki 已提交
945

946 947 948
/**
 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
 * @zone: zone of the wanted lruvec
949
 * @memcg: memcg of the wanted lruvec
950 951 952 953 954 955 956 957 958
 *
 * Returns the lru list vector holding pages for the given @zone and
 * @mem.  This can be the global zone lruvec, if the memory controller
 * is disabled.
 */
struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
				      struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_zone *mz;
959
	struct lruvec *lruvec;
960

961 962 963 964
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
965

966
	mz = mem_cgroup_zone_zoneinfo(memcg, zone);
967 968 969 970 971 972 973 974 975 976
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
977 978 979
}

/**
980
 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
981
 * @page: the page
982
 * @zone: zone of the page
983 984 985 986
 *
 * This function is only safe when following the LRU page isolation
 * and putback protocol: the LRU lock must be held, and the page must
 * either be PageLRU() or the caller must have isolated/allocated it.
987
 */
988
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
K
KAMEZAWA Hiroyuki 已提交
989 990
{
	struct mem_cgroup_per_zone *mz;
991
	struct mem_cgroup *memcg;
992
	struct lruvec *lruvec;
993

994 995 996 997
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
998

999
	memcg = page->mem_cgroup;
1000
	/*
1001
	 * Swapcache readahead pages are added to the LRU - and
1002
	 * possibly migrated - before they are charged.
1003
	 */
1004 1005
	if (!memcg)
		memcg = root_mem_cgroup;
1006

1007
	mz = mem_cgroup_page_zoneinfo(memcg, page);
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1018
}
1019

1020
/**
1021 1022 1023 1024
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
 * @nr_pages: positive when adding or negative when removing
1025
 *
1026 1027 1028
 * This function must be called under lru_lock, just before a page is added
 * to or just after a page is removed from an lru list (that ordering being
 * so as to allow it to check that lru_size 0 is consistent with list_empty).
1029
 */
1030 1031
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
				int nr_pages)
1032 1033
{
	struct mem_cgroup_per_zone *mz;
1034
	unsigned long *lru_size;
1035 1036
	long size;
	bool empty;
1037

1038 1039
	__update_lru_size(lruvec, lru, nr_pages);

1040 1041 1042
	if (mem_cgroup_disabled())
		return;

1043 1044
	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	lru_size = mz->lru_size + lru;
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	empty = list_empty(lruvec->lists + lru);

	if (nr_pages < 0)
		*lru_size += nr_pages;

	size = *lru_size;
	if (WARN_ONCE(size < 0 || empty != !size,
		"%s(%p, %d, %d): lru_size %ld but %sempty\n",
		__func__, lruvec, lru, nr_pages, size, empty ? "" : "not ")) {
		VM_BUG_ON(1);
		*lru_size = 0;
	}

	if (nr_pages > 0)
		*lru_size += nr_pages;
K
KAMEZAWA Hiroyuki 已提交
1060
}
1061

1062
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1063
{
1064
	struct mem_cgroup *task_memcg;
1065
	struct task_struct *p;
1066
	bool ret;
1067

1068
	p = find_lock_task_mm(task);
1069
	if (p) {
1070
		task_memcg = get_mem_cgroup_from_mm(p->mm);
1071 1072 1073 1074 1075 1076 1077
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
1078
		rcu_read_lock();
1079 1080
		task_memcg = mem_cgroup_from_task(task);
		css_get(&task_memcg->css);
1081
		rcu_read_unlock();
1082
	}
1083 1084
	ret = mem_cgroup_is_descendant(task_memcg, memcg);
	css_put(&task_memcg->css);
1085 1086 1087
	return ret;
}

1088
/**
1089
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1090
 * @memcg: the memory cgroup
1091
 *
1092
 * Returns the maximum amount of memory @mem can be charged with, in
1093
 * pages.
1094
 */
1095
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1096
{
1097 1098 1099
	unsigned long margin = 0;
	unsigned long count;
	unsigned long limit;
1100

1101
	count = page_counter_read(&memcg->memory);
1102
	limit = READ_ONCE(memcg->memory.limit);
1103 1104 1105
	if (count < limit)
		margin = limit - count;

1106
	if (do_memsw_account()) {
1107
		count = page_counter_read(&memcg->memsw);
1108
		limit = READ_ONCE(memcg->memsw.limit);
1109 1110
		if (count <= limit)
			margin = min(margin, limit - count);
1111 1112
		else
			margin = 0;
1113 1114 1115
	}

	return margin;
1116 1117
}

1118
/*
Q
Qiang Huang 已提交
1119
 * A routine for checking "mem" is under move_account() or not.
1120
 *
Q
Qiang Huang 已提交
1121 1122 1123
 * Checking a cgroup is mc.from or mc.to or under hierarchy of
 * moving cgroups. This is for waiting at high-memory pressure
 * caused by "move".
1124
 */
1125
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1126
{
1127 1128
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1129
	bool ret = false;
1130 1131 1132 1133 1134 1135 1136 1137 1138
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1139

1140 1141
	ret = mem_cgroup_is_descendant(from, memcg) ||
		mem_cgroup_is_descendant(to, memcg);
1142 1143
unlock:
	spin_unlock(&mc.lock);
1144 1145 1146
	return ret;
}

1147
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1148 1149
{
	if (mc.moving_task && current != mc.moving_task) {
1150
		if (mem_cgroup_under_move(memcg)) {
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1163
#define K(x) ((x) << (PAGE_SHIFT-10))
1164
/**
1165
 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1166 1167 1168 1169 1170 1171 1172 1173
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
1174 1175
	struct mem_cgroup *iter;
	unsigned int i;
1176 1177 1178

	rcu_read_lock();

1179 1180 1181 1182 1183 1184 1185 1186
	if (p) {
		pr_info("Task in ");
		pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
		pr_cont(" killed as a result of limit of ");
	} else {
		pr_info("Memory limit reached of cgroup ");
	}

T
Tejun Heo 已提交
1187
	pr_cont_cgroup_path(memcg->css.cgroup);
1188
	pr_cont("\n");
1189 1190 1191

	rcu_read_unlock();

1192 1193 1194 1195 1196 1197 1198 1199 1200
	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memory)),
		K((u64)memcg->memory.limit), memcg->memory.failcnt);
	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->memsw)),
		K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
		K((u64)page_counter_read(&memcg->kmem)),
		K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1201 1202

	for_each_mem_cgroup_tree(iter, memcg) {
T
Tejun Heo 已提交
1203 1204
		pr_info("Memory cgroup stats for ");
		pr_cont_cgroup_path(iter->css.cgroup);
1205 1206 1207
		pr_cont(":");

		for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1208
			if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1209
				continue;
1210
			pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
1211 1212 1213 1214 1215 1216 1217 1218 1219
				K(mem_cgroup_read_stat(iter, i)));
		}

		for (i = 0; i < NR_LRU_LISTS; i++)
			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));

		pr_cont("\n");
	}
1220 1221
}

1222 1223 1224 1225
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1226
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1227 1228
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1229 1230
	struct mem_cgroup *iter;

1231
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1232
		num++;
1233 1234 1235
	return num;
}

D
David Rientjes 已提交
1236 1237 1238
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1239
static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1240
{
1241
	unsigned long limit;
1242

1243
	limit = memcg->memory.limit;
1244
	if (mem_cgroup_swappiness(memcg)) {
1245
		unsigned long memsw_limit;
1246
		unsigned long swap_limit;
1247

1248
		memsw_limit = memcg->memsw.limit;
1249 1250 1251
		swap_limit = memcg->swap.limit;
		swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
		limit = min(limit + swap_limit, memsw_limit);
1252 1253
	}
	return limit;
D
David Rientjes 已提交
1254 1255
}

1256
static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1257
				     int order)
1258
{
1259 1260 1261 1262 1263 1264
	struct oom_control oc = {
		.zonelist = NULL,
		.nodemask = NULL,
		.gfp_mask = gfp_mask,
		.order = order,
	};
1265 1266 1267 1268 1269 1270
	struct mem_cgroup *iter;
	unsigned long chosen_points = 0;
	unsigned long totalpages;
	unsigned int points = 0;
	struct task_struct *chosen = NULL;

1271 1272
	mutex_lock(&oom_lock);

1273
	/*
1274 1275 1276
	 * If current has a pending SIGKILL or is exiting, then automatically
	 * select it.  The goal is to allow it to allocate so that it may
	 * quickly exit and free its memory.
1277
	 */
1278
	if (fatal_signal_pending(current) || task_will_free_mem(current)) {
1279
		mark_oom_victim(current);
1280
		try_oom_reaper(current);
1281
		goto unlock;
1282 1283
	}

1284
	check_panic_on_oom(&oc, CONSTRAINT_MEMCG, memcg);
1285
	totalpages = mem_cgroup_get_limit(memcg) ? : 1;
1286
	for_each_mem_cgroup_tree(iter, memcg) {
1287
		struct css_task_iter it;
1288 1289
		struct task_struct *task;

1290 1291
		css_task_iter_start(&iter->css, &it);
		while ((task = css_task_iter_next(&it))) {
1292
			switch (oom_scan_process_thread(&oc, task, totalpages)) {
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
			case OOM_SCAN_SELECT:
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = ULONG_MAX;
				get_task_struct(chosen);
				/* fall through */
			case OOM_SCAN_CONTINUE:
				continue;
			case OOM_SCAN_ABORT:
1303
				css_task_iter_end(&it);
1304 1305 1306
				mem_cgroup_iter_break(memcg, iter);
				if (chosen)
					put_task_struct(chosen);
1307 1308
				/* Set a dummy value to return "true". */
				chosen = (void *) 1;
1309
				goto unlock;
1310 1311 1312 1313
			case OOM_SCAN_OK:
				break;
			};
			points = oom_badness(task, memcg, NULL, totalpages);
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
			if (!points || points < chosen_points)
				continue;
			/* Prefer thread group leaders for display purposes */
			if (points == chosen_points &&
			    thread_group_leader(chosen))
				continue;

			if (chosen)
				put_task_struct(chosen);
			chosen = task;
			chosen_points = points;
			get_task_struct(chosen);
1326
		}
1327
		css_task_iter_end(&it);
1328 1329
	}

1330 1331
	if (chosen) {
		points = chosen_points * 1000 / totalpages;
1332 1333
		oom_kill_process(&oc, chosen, points, totalpages, memcg,
				 "Memory cgroup out of memory");
1334 1335 1336
	}
unlock:
	mutex_unlock(&oom_lock);
1337
	return chosen;
1338 1339
}

1340 1341
#if MAX_NUMNODES > 1

1342 1343
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1344
 * @memcg: the target memcg
1345 1346 1347 1348 1349 1350 1351
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1352
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1353 1354
		int nid, bool noswap)
{
1355
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1356 1357 1358
		return true;
	if (noswap || !total_swap_pages)
		return false;
1359
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1360 1361 1362 1363
		return true;
	return false;

}
1364 1365 1366 1367 1368 1369 1370

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1371
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1372 1373
{
	int nid;
1374 1375 1376 1377
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1378
	if (!atomic_read(&memcg->numainfo_events))
1379
		return;
1380
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1381 1382 1383
		return;

	/* make a nodemask where this memcg uses memory from */
1384
	memcg->scan_nodes = node_states[N_MEMORY];
1385

1386
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1387

1388 1389
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1390
	}
1391

1392 1393
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1408
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1409 1410 1411
{
	int node;

1412 1413
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1414

1415
	node = next_node_in(node, memcg->scan_nodes);
1416
	/*
1417 1418 1419
	 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
	 * last time it really checked all the LRUs due to rate limiting.
	 * Fallback to the current node in that case for simplicity.
1420 1421 1422 1423
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1424
	memcg->last_scanned_node = node;
1425 1426 1427
	return node;
}
#else
1428
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1429 1430 1431 1432 1433
{
	return 0;
}
#endif

1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
				   struct zone *zone,
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
{
	struct mem_cgroup *victim = NULL;
	int total = 0;
	int loop = 0;
	unsigned long excess;
	unsigned long nr_scanned;
	struct mem_cgroup_reclaim_cookie reclaim = {
		.zone = zone,
		.priority = 0,
	};

1449
	excess = soft_limit_excess(root_memcg);
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477

	while (1) {
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
		if (!victim) {
			loop++;
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
				if (!total)
					break;
				/*
				 * We want to do more targeted reclaim.
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
					break;
			}
			continue;
		}
		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
						     zone, &nr_scanned);
		*total_scanned += nr_scanned;
1478
		if (!soft_limit_excess(root_memcg))
1479
			break;
1480
	}
1481 1482
	mem_cgroup_iter_break(root_memcg, victim);
	return total;
1483 1484
}

1485 1486 1487 1488 1489 1490
#ifdef CONFIG_LOCKDEP
static struct lockdep_map memcg_oom_lock_dep_map = {
	.name = "memcg_oom_lock",
};
#endif

1491 1492
static DEFINE_SPINLOCK(memcg_oom_lock);

K
KAMEZAWA Hiroyuki 已提交
1493 1494 1495 1496
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
 */
1497
static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1498
{
1499
	struct mem_cgroup *iter, *failed = NULL;
1500

1501 1502
	spin_lock(&memcg_oom_lock);

1503
	for_each_mem_cgroup_tree(iter, memcg) {
1504
		if (iter->oom_lock) {
1505 1506 1507 1508 1509
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1510 1511
			mem_cgroup_iter_break(memcg, iter);
			break;
1512 1513
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1514
	}
K
KAMEZAWA Hiroyuki 已提交
1515

1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
	if (failed) {
		/*
		 * OK, we failed to lock the whole subtree so we have
		 * to clean up what we set up to the failing subtree
		 */
		for_each_mem_cgroup_tree(iter, memcg) {
			if (iter == failed) {
				mem_cgroup_iter_break(memcg, iter);
				break;
			}
			iter->oom_lock = false;
1527
		}
1528 1529
	} else
		mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1530 1531 1532 1533

	spin_unlock(&memcg_oom_lock);

	return !failed;
1534
}
1535

1536
static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1537
{
K
KAMEZAWA Hiroyuki 已提交
1538 1539
	struct mem_cgroup *iter;

1540
	spin_lock(&memcg_oom_lock);
1541
	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1542
	for_each_mem_cgroup_tree(iter, memcg)
1543
		iter->oom_lock = false;
1544
	spin_unlock(&memcg_oom_lock);
1545 1546
}

1547
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1548 1549 1550
{
	struct mem_cgroup *iter;

1551
	spin_lock(&memcg_oom_lock);
1552
	for_each_mem_cgroup_tree(iter, memcg)
1553 1554
		iter->under_oom++;
	spin_unlock(&memcg_oom_lock);
1555 1556
}

1557
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1558 1559 1560
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
1561 1562
	/*
	 * When a new child is created while the hierarchy is under oom,
1563
	 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
K
KAMEZAWA Hiroyuki 已提交
1564
	 */
1565
	spin_lock(&memcg_oom_lock);
1566
	for_each_mem_cgroup_tree(iter, memcg)
1567 1568 1569
		if (iter->under_oom > 0)
			iter->under_oom--;
	spin_unlock(&memcg_oom_lock);
1570 1571
}

K
KAMEZAWA Hiroyuki 已提交
1572 1573
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
1574
struct oom_wait_info {
1575
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
1576 1577 1578 1579 1580 1581
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
1582 1583
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
1584 1585 1586
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1587
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
1588

1589 1590
	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
	    !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
K
KAMEZAWA Hiroyuki 已提交
1591 1592 1593 1594
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

1595
static void memcg_oom_recover(struct mem_cgroup *memcg)
1596
{
1597 1598 1599 1600 1601 1602 1603 1604 1605
	/*
	 * For the following lockless ->under_oom test, the only required
	 * guarantee is that it must see the state asserted by an OOM when
	 * this function is called as a result of userland actions
	 * triggered by the notification of the OOM.  This is trivially
	 * achieved by invoking mem_cgroup_mark_under_oom() before
	 * triggering notification.
	 */
	if (memcg && memcg->under_oom)
1606
		__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1607 1608
}

1609
static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1610
{
1611
	if (!current->memcg_may_oom)
1612
		return;
K
KAMEZAWA Hiroyuki 已提交
1613
	/*
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
	 * We are in the middle of the charge context here, so we
	 * don't want to block when potentially sitting on a callstack
	 * that holds all kinds of filesystem and mm locks.
	 *
	 * Also, the caller may handle a failed allocation gracefully
	 * (like optional page cache readahead) and so an OOM killer
	 * invocation might not even be necessary.
	 *
	 * That's why we don't do anything here except remember the
	 * OOM context and then deal with it at the end of the page
	 * fault when the stack is unwound, the locks are released,
	 * and when we know whether the fault was overall successful.
K
KAMEZAWA Hiroyuki 已提交
1626
	 */
1627
	css_get(&memcg->css);
T
Tejun Heo 已提交
1628 1629 1630
	current->memcg_in_oom = memcg;
	current->memcg_oom_gfp_mask = mask;
	current->memcg_oom_order = order;
1631 1632 1633 1634
}

/**
 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1635
 * @handle: actually kill/wait or just clean up the OOM state
1636
 *
1637 1638
 * This has to be called at the end of a page fault if the memcg OOM
 * handler was enabled.
1639
 *
1640
 * Memcg supports userspace OOM handling where failed allocations must
1641 1642 1643 1644
 * sleep on a waitqueue until the userspace task resolves the
 * situation.  Sleeping directly in the charge context with all kinds
 * of locks held is not a good idea, instead we remember an OOM state
 * in the task and mem_cgroup_oom_synchronize() has to be called at
1645
 * the end of the page fault to complete the OOM handling.
1646 1647
 *
 * Returns %true if an ongoing memcg OOM situation was detected and
1648
 * completed, %false otherwise.
1649
 */
1650
bool mem_cgroup_oom_synchronize(bool handle)
1651
{
T
Tejun Heo 已提交
1652
	struct mem_cgroup *memcg = current->memcg_in_oom;
1653
	struct oom_wait_info owait;
1654
	bool locked;
1655 1656 1657

	/* OOM is global, do not handle */
	if (!memcg)
1658
		return false;
1659

1660
	if (!handle || oom_killer_disabled)
1661
		goto cleanup;
1662 1663 1664 1665 1666 1667

	owait.memcg = memcg;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
K
KAMEZAWA Hiroyuki 已提交
1668

1669
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	if (locked && !memcg->oom_kill_disable) {
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
T
Tejun Heo 已提交
1680 1681
		mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
					 current->memcg_oom_order);
1682
	} else {
1683
		schedule();
1684 1685 1686 1687 1688
		mem_cgroup_unmark_under_oom(memcg);
		finish_wait(&memcg_oom_waitq, &owait.wait);
	}

	if (locked) {
1689 1690 1691 1692 1693 1694 1695 1696
		mem_cgroup_oom_unlock(memcg);
		/*
		 * There is no guarantee that an OOM-lock contender
		 * sees the wakeups triggered by the OOM kill
		 * uncharges.  Wake any sleepers explicitely.
		 */
		memcg_oom_recover(memcg);
	}
1697
cleanup:
T
Tejun Heo 已提交
1698
	current->memcg_in_oom = NULL;
1699
	css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
1700
	return true;
1701 1702
}

1703
/**
1704 1705
 * lock_page_memcg - lock a page->mem_cgroup binding
 * @page: the page
1706
 *
1707 1708
 * This function protects unlocked LRU pages from being moved to
 * another cgroup and stabilizes their page->mem_cgroup binding.
1709
 */
J
Johannes Weiner 已提交
1710
void lock_page_memcg(struct page *page)
1711 1712
{
	struct mem_cgroup *memcg;
1713
	unsigned long flags;
1714

1715 1716 1717 1718 1719
	/*
	 * The RCU lock is held throughout the transaction.  The fast
	 * path can get away without acquiring the memcg->move_lock
	 * because page moving starts with an RCU grace period.
	 */
1720 1721 1722
	rcu_read_lock();

	if (mem_cgroup_disabled())
J
Johannes Weiner 已提交
1723
		return;
1724
again:
1725
	memcg = page->mem_cgroup;
1726
	if (unlikely(!memcg))
J
Johannes Weiner 已提交
1727
		return;
1728

Q
Qiang Huang 已提交
1729
	if (atomic_read(&memcg->moving_account) <= 0)
J
Johannes Weiner 已提交
1730
		return;
1731

1732
	spin_lock_irqsave(&memcg->move_lock, flags);
1733
	if (memcg != page->mem_cgroup) {
1734
		spin_unlock_irqrestore(&memcg->move_lock, flags);
1735 1736
		goto again;
	}
1737 1738 1739 1740

	/*
	 * When charge migration first begins, we can have locked and
	 * unlocked page stat updates happening concurrently.  Track
1741
	 * the task who has the lock for unlock_page_memcg().
1742 1743 1744
	 */
	memcg->move_lock_task = current;
	memcg->move_lock_flags = flags;
1745

J
Johannes Weiner 已提交
1746
	return;
1747
}
1748
EXPORT_SYMBOL(lock_page_memcg);
1749

1750
/**
1751
 * unlock_page_memcg - unlock a page->mem_cgroup binding
J
Johannes Weiner 已提交
1752
 * @page: the page
1753
 */
J
Johannes Weiner 已提交
1754
void unlock_page_memcg(struct page *page)
1755
{
J
Johannes Weiner 已提交
1756 1757
	struct mem_cgroup *memcg = page->mem_cgroup;

1758 1759 1760 1761 1762 1763 1764 1765
	if (memcg && memcg->move_lock_task == current) {
		unsigned long flags = memcg->move_lock_flags;

		memcg->move_lock_task = NULL;
		memcg->move_lock_flags = 0;

		spin_unlock_irqrestore(&memcg->move_lock, flags);
	}
1766

1767
	rcu_read_unlock();
1768
}
1769
EXPORT_SYMBOL(unlock_page_memcg);
1770

1771 1772 1773 1774
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
1775
#define CHARGE_BATCH	32U
1776 1777
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
1778
	unsigned int nr_pages;
1779
	struct work_struct work;
1780
	unsigned long flags;
1781
#define FLUSHING_CACHED_CHARGE	0
1782 1783
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1784
static DEFINE_MUTEX(percpu_charge_mutex);
1785

1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
1796
 */
1797
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1798 1799
{
	struct memcg_stock_pcp *stock;
1800
	bool ret = false;
1801

1802
	if (nr_pages > CHARGE_BATCH)
1803
		return ret;
1804

1805
	stock = &get_cpu_var(memcg_stock);
1806
	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1807
		stock->nr_pages -= nr_pages;
1808 1809
		ret = true;
	}
1810 1811 1812 1813 1814
	put_cpu_var(memcg_stock);
	return ret;
}

/*
1815
 * Returns stocks cached in percpu and reset cached information.
1816 1817 1818 1819 1820
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

1821
	if (stock->nr_pages) {
1822
		page_counter_uncharge(&old->memory, stock->nr_pages);
1823
		if (do_memsw_account())
1824
			page_counter_uncharge(&old->memsw, stock->nr_pages);
1825
		css_put_many(&old->css, stock->nr_pages);
1826
		stock->nr_pages = 0;
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
1837
	struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock);
1838
	drain_stock(stock);
1839
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1840 1841 1842
}

/*
1843
 * Cache charges(val) to local per_cpu area.
1844
 * This will be consumed by consume_stock() function, later.
1845
 */
1846
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1847 1848 1849
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

1850
	if (stock->cached != memcg) { /* reset if necessary */
1851
		drain_stock(stock);
1852
		stock->cached = memcg;
1853
	}
1854
	stock->nr_pages += nr_pages;
1855 1856 1857 1858
	put_cpu_var(memcg_stock);
}

/*
1859
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
1860
 * of the hierarchy under it.
1861
 */
1862
static void drain_all_stock(struct mem_cgroup *root_memcg)
1863
{
1864
	int cpu, curcpu;
1865

1866 1867 1868
	/* If someone's already draining, avoid adding running more workers. */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
1869 1870
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
1871
	curcpu = get_cpu();
1872 1873
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1874
		struct mem_cgroup *memcg;
1875

1876 1877
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
1878
			continue;
1879
		if (!mem_cgroup_is_descendant(memcg, root_memcg))
1880
			continue;
1881 1882 1883 1884 1885 1886
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
1887
	}
1888
	put_cpu();
A
Andrew Morton 已提交
1889
	put_online_cpus();
1890
	mutex_unlock(&percpu_charge_mutex);
1891 1892
}

1893
static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
1894 1895 1896 1897 1898 1899
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;

1900
	if (action == CPU_ONLINE)
1901 1902
		return NOTIFY_OK;

1903
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
1904
		return NOTIFY_OK;
1905

1906 1907 1908 1909 1910
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
static void reclaim_high(struct mem_cgroup *memcg,
			 unsigned int nr_pages,
			 gfp_t gfp_mask)
{
	do {
		if (page_counter_read(&memcg->memory) <= memcg->high)
			continue;
		mem_cgroup_events(memcg, MEMCG_HIGH, 1);
		try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
	} while ((memcg = parent_mem_cgroup(memcg)));
}

static void high_work_func(struct work_struct *work)
{
	struct mem_cgroup *memcg;

	memcg = container_of(work, struct mem_cgroup, high_work);
	reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
}

1931 1932 1933 1934 1935 1936 1937
/*
 * Scheduled by try_charge() to be executed from the userland return path
 * and reclaims memory over the high limit.
 */
void mem_cgroup_handle_over_high(void)
{
	unsigned int nr_pages = current->memcg_nr_pages_over_high;
1938
	struct mem_cgroup *memcg;
1939 1940 1941 1942

	if (likely(!nr_pages))
		return;

1943 1944
	memcg = get_mem_cgroup_from_mm(current->mm);
	reclaim_high(memcg, nr_pages, GFP_KERNEL);
1945 1946 1947 1948
	css_put(&memcg->css);
	current->memcg_nr_pages_over_high = 0;
}

1949 1950
static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
		      unsigned int nr_pages)
1951
{
1952
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
1953
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1954
	struct mem_cgroup *mem_over_limit;
1955
	struct page_counter *counter;
1956
	unsigned long nr_reclaimed;
1957 1958
	bool may_swap = true;
	bool drained = false;
1959

1960
	if (mem_cgroup_is_root(memcg))
1961
		return 0;
1962
retry:
1963
	if (consume_stock(memcg, nr_pages))
1964
		return 0;
1965

1966
	if (!do_memsw_account() ||
1967 1968
	    page_counter_try_charge(&memcg->memsw, batch, &counter)) {
		if (page_counter_try_charge(&memcg->memory, batch, &counter))
1969
			goto done_restock;
1970
		if (do_memsw_account())
1971 1972
			page_counter_uncharge(&memcg->memsw, batch);
		mem_over_limit = mem_cgroup_from_counter(counter, memory);
1973
	} else {
1974
		mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1975
		may_swap = false;
1976
	}
1977

1978 1979 1980 1981
	if (batch > nr_pages) {
		batch = nr_pages;
		goto retry;
	}
1982

1983 1984 1985 1986 1987 1988 1989 1990 1991
	/*
	 * Unlike in global OOM situations, memcg is not in a physical
	 * memory shortage.  Allow dying and OOM-killed tasks to
	 * bypass the last charges so that they can exit quickly and
	 * free their memory.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE) ||
		     fatal_signal_pending(current) ||
		     current->flags & PF_EXITING))
1992
		goto force;
1993 1994 1995 1996

	if (unlikely(task_in_memcg_oom(current)))
		goto nomem;

1997
	if (!gfpflags_allow_blocking(gfp_mask))
1998
		goto nomem;
1999

2000 2001
	mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);

2002 2003
	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
						    gfp_mask, may_swap);
2004

2005
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2006
		goto retry;
2007

2008
	if (!drained) {
2009
		drain_all_stock(mem_over_limit);
2010 2011 2012 2013
		drained = true;
		goto retry;
	}

2014 2015
	if (gfp_mask & __GFP_NORETRY)
		goto nomem;
2016 2017 2018 2019 2020 2021 2022 2023 2024
	/*
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
	 */
2025
	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2026 2027 2028 2029 2030 2031 2032 2033
		goto retry;
	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		goto retry;

2034 2035 2036
	if (nr_retries--)
		goto retry;

2037
	if (gfp_mask & __GFP_NOFAIL)
2038
		goto force;
2039

2040
	if (fatal_signal_pending(current))
2041
		goto force;
2042

2043 2044
	mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);

2045 2046
	mem_cgroup_oom(mem_over_limit, gfp_mask,
		       get_order(nr_pages * PAGE_SIZE));
2047
nomem:
2048
	if (!(gfp_mask & __GFP_NOFAIL))
2049
		return -ENOMEM;
2050 2051 2052 2053 2054 2055 2056
force:
	/*
	 * The allocation either can't fail or will lead to more memory
	 * being freed very soon.  Allow memory usage go over the limit
	 * temporarily by force charging it.
	 */
	page_counter_charge(&memcg->memory, nr_pages);
2057
	if (do_memsw_account())
2058 2059 2060 2061
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);

	return 0;
2062 2063

done_restock:
2064
	css_get_many(&memcg->css, batch);
2065 2066
	if (batch > nr_pages)
		refill_stock(memcg, batch - nr_pages);
2067

2068
	/*
2069 2070
	 * If the hierarchy is above the normal consumption range, schedule
	 * reclaim on returning to userland.  We can perform reclaim here
2071
	 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2072 2073 2074 2075
	 * GFP_KERNEL can consistently be used during reclaim.  @memcg is
	 * not recorded as it most likely matches current's and won't
	 * change in the meantime.  As high limit is checked again before
	 * reclaim, the cost of mismatch is negligible.
2076 2077
	 */
	do {
2078
		if (page_counter_read(&memcg->memory) > memcg->high) {
2079 2080 2081 2082 2083
			/* Don't bother a random interrupted task */
			if (in_interrupt()) {
				schedule_work(&memcg->high_work);
				break;
			}
V
Vladimir Davydov 已提交
2084
			current->memcg_nr_pages_over_high += batch;
2085 2086 2087
			set_notify_resume(current);
			break;
		}
2088
	} while ((memcg = parent_mem_cgroup(memcg)));
2089 2090

	return 0;
2091
}
2092

2093
static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2094
{
2095 2096 2097
	if (mem_cgroup_is_root(memcg))
		return;

2098
	page_counter_uncharge(&memcg->memory, nr_pages);
2099
	if (do_memsw_account())
2100
		page_counter_uncharge(&memcg->memsw, nr_pages);
2101

2102
	css_put_many(&memcg->css, nr_pages);
2103 2104
}

2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
static void lock_page_lru(struct page *page, int *isolated)
{
	struct zone *zone = page_zone(page);

	spin_lock_irq(&zone->lru_lock);
	if (PageLRU(page)) {
		struct lruvec *lruvec;

		lruvec = mem_cgroup_page_lruvec(page, zone);
		ClearPageLRU(page);
		del_page_from_lru_list(page, lruvec, page_lru(page));
		*isolated = 1;
	} else
		*isolated = 0;
}

static void unlock_page_lru(struct page *page, int isolated)
{
	struct zone *zone = page_zone(page);

	if (isolated) {
		struct lruvec *lruvec;

		lruvec = mem_cgroup_page_lruvec(page, zone);
		VM_BUG_ON_PAGE(PageLRU(page), page);
		SetPageLRU(page);
		add_page_to_lru_list(page, lruvec, page_lru(page));
	}
	spin_unlock_irq(&zone->lru_lock);
}

2136
static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2137
			  bool lrucare)
2138
{
2139
	int isolated;
2140

2141
	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2142 2143 2144 2145 2146

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
2147 2148
	if (lrucare)
		lock_page_lru(page, &isolated);
2149

2150 2151
	/*
	 * Nobody should be changing or seriously looking at
2152
	 * page->mem_cgroup at this point:
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
	 *
	 * - the page is uncharged
	 *
	 * - the page is off-LRU
	 *
	 * - an anonymous fault has exclusive page access, except for
	 *   a locked page table
	 *
	 * - a page cache insertion, a swapin fault, or a migration
	 *   have the page locked
	 */
2164
	page->mem_cgroup = memcg;
2165

2166 2167
	if (lrucare)
		unlock_page_lru(page, isolated);
2168
}
2169

2170
#ifndef CONFIG_SLOB
2171
static int memcg_alloc_cache_id(void)
2172
{
2173 2174 2175
	int id, size;
	int err;

2176
	id = ida_simple_get(&memcg_cache_ida,
2177 2178 2179
			    0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (id < 0)
		return id;
2180

2181
	if (id < memcg_nr_cache_ids)
2182 2183 2184 2185 2186 2187
		return id;

	/*
	 * There's no space for the new id in memcg_caches arrays,
	 * so we have to grow them.
	 */
2188
	down_write(&memcg_cache_ids_sem);
2189 2190

	size = 2 * (id + 1);
2191 2192 2193 2194 2195
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

2196
	err = memcg_update_all_caches(size);
2197 2198
	if (!err)
		err = memcg_update_all_list_lrus(size);
2199 2200 2201 2202 2203
	if (!err)
		memcg_nr_cache_ids = size;

	up_write(&memcg_cache_ids_sem);

2204
	if (err) {
2205
		ida_simple_remove(&memcg_cache_ida, id);
2206 2207 2208 2209 2210 2211 2212
		return err;
	}
	return id;
}

static void memcg_free_cache_id(int id)
{
2213
	ida_simple_remove(&memcg_cache_ida, id);
2214 2215
}

2216
struct memcg_kmem_cache_create_work {
2217 2218 2219 2220 2221
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

2222
static void memcg_kmem_cache_create_func(struct work_struct *w)
2223
{
2224 2225
	struct memcg_kmem_cache_create_work *cw =
		container_of(w, struct memcg_kmem_cache_create_work, work);
2226 2227
	struct mem_cgroup *memcg = cw->memcg;
	struct kmem_cache *cachep = cw->cachep;
2228

2229
	memcg_create_kmem_cache(memcg, cachep);
2230

2231
	css_put(&memcg->css);
2232 2233 2234 2235 2236 2237
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 */
2238 2239
static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					       struct kmem_cache *cachep)
2240
{
2241
	struct memcg_kmem_cache_create_work *cw;
2242

2243
	cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2244
	if (!cw)
2245
		return;
2246 2247

	css_get(&memcg->css);
2248 2249 2250

	cw->memcg = memcg;
	cw->cachep = cachep;
2251
	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2252 2253 2254 2255

	schedule_work(&cw->work);
}

2256 2257
static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
					     struct kmem_cache *cachep)
2258 2259 2260 2261
{
	/*
	 * We need to stop accounting when we kmalloc, because if the
	 * corresponding kmalloc cache is not yet created, the first allocation
2262
	 * in __memcg_schedule_kmem_cache_create will recurse.
2263 2264 2265 2266 2267 2268 2269
	 *
	 * However, it is better to enclose the whole function. Depending on
	 * the debugging options enabled, INIT_WORK(), for instance, can
	 * trigger an allocation. This too, will make us recurse. Because at
	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
	 * the safest choice is to do it like this, wrapping the whole function.
	 */
2270
	current->memcg_kmem_skip_account = 1;
2271
	__memcg_schedule_kmem_cache_create(memcg, cachep);
2272
	current->memcg_kmem_skip_account = 0;
2273
}
2274

2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
/*
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
 * If the cache does not exist yet, if we are the first user of it,
 * we either create it immediately, if possible, or create it asynchronously
 * in a workqueue.
 * In the latter case, we will let the current allocation go through with
 * the original cache.
 *
 * Can't be called in interrupt context or from kernel threads.
 * This function needs to be called with rcu_read_lock() held.
 */
V
Vladimir Davydov 已提交
2288
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
2289 2290
{
	struct mem_cgroup *memcg;
2291
	struct kmem_cache *memcg_cachep;
2292
	int kmemcg_id;
2293

2294
	VM_BUG_ON(!is_root_cache(cachep));
2295

V
Vladimir Davydov 已提交
2296 2297 2298 2299 2300 2301
	if (cachep->flags & SLAB_ACCOUNT)
		gfp |= __GFP_ACCOUNT;

	if (!(gfp & __GFP_ACCOUNT))
		return cachep;

2302
	if (current->memcg_kmem_skip_account)
2303 2304
		return cachep;

2305
	memcg = get_mem_cgroup_from_mm(current->mm);
2306
	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2307
	if (kmemcg_id < 0)
2308
		goto out;
2309

2310
	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2311 2312
	if (likely(memcg_cachep))
		return memcg_cachep;
2313 2314 2315 2316 2317 2318 2319 2320 2321

	/*
	 * If we are in a safe context (can wait, and not in interrupt
	 * context), we could be be predictable and return right away.
	 * This would guarantee that the allocation being performed
	 * already belongs in the new cache.
	 *
	 * However, there are some clashes that can arrive from locking.
	 * For instance, because we acquire the slab_mutex while doing
2322 2323 2324
	 * memcg_create_kmem_cache, this means no further allocation
	 * could happen with the slab_mutex held. So it's better to
	 * defer everything.
2325
	 */
2326
	memcg_schedule_kmem_cache_create(memcg, cachep);
2327
out:
2328
	css_put(&memcg->css);
2329
	return cachep;
2330 2331
}

2332 2333 2334
void __memcg_kmem_put_cache(struct kmem_cache *cachep)
{
	if (!is_root_cache(cachep))
2335
		css_put(&cachep->memcg_params.memcg->css);
2336 2337
}

2338 2339
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			      struct mem_cgroup *memcg)
2340
{
2341 2342
	unsigned int nr_pages = 1 << order;
	struct page_counter *counter;
2343 2344
	int ret;

2345
	ret = try_charge(memcg, gfp, nr_pages);
2346
	if (ret)
2347
		return ret;
2348 2349 2350 2351 2352

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
	    !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
		cancel_charge(memcg, nr_pages);
		return -ENOMEM;
2353 2354
	}

2355
	page->mem_cgroup = memcg;
2356

2357
	return 0;
2358 2359
}

2360
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2361
{
2362
	struct mem_cgroup *memcg;
2363
	int ret = 0;
2364

2365
	memcg = get_mem_cgroup_from_mm(current->mm);
2366
	if (!mem_cgroup_is_root(memcg))
2367
		ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
2368
	css_put(&memcg->css);
2369
	return ret;
2370 2371
}

2372
void __memcg_kmem_uncharge(struct page *page, int order)
2373
{
2374
	struct mem_cgroup *memcg = page->mem_cgroup;
2375
	unsigned int nr_pages = 1 << order;
2376 2377 2378 2379

	if (!memcg)
		return;

2380
	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2381

2382 2383 2384
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
		page_counter_uncharge(&memcg->kmem, nr_pages);

2385
	page_counter_uncharge(&memcg->memory, nr_pages);
2386
	if (do_memsw_account())
2387
		page_counter_uncharge(&memcg->memsw, nr_pages);
2388

2389
	page->mem_cgroup = NULL;
2390
	css_put_many(&memcg->css, nr_pages);
2391
}
2392
#endif /* !CONFIG_SLOB */
2393

2394 2395 2396 2397
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

/*
 * Because tail pages are not marked as "used", set it. We're under
2398
 * zone->lru_lock and migration entries setup in all page mappings.
2399
 */
2400
void mem_cgroup_split_huge_fixup(struct page *head)
2401
{
2402
	int i;
2403

2404 2405
	if (mem_cgroup_disabled())
		return;
2406

2407
	for (i = 1; i < HPAGE_PMD_NR; i++)
2408
		head[i].mem_cgroup = head->mem_cgroup;
2409

2410
	__this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2411
		       HPAGE_PMD_NR);
2412
}
2413
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2414

A
Andrew Morton 已提交
2415
#ifdef CONFIG_MEMCG_SWAP
2416 2417
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
					 bool charge)
K
KAMEZAWA Hiroyuki 已提交
2418
{
2419 2420
	int val = (charge) ? 1 : -1;
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
K
KAMEZAWA Hiroyuki 已提交
2421
}
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
2434
 * The caller must have charged to @to, IOW, called page_counter_charge() about
2435 2436 2437
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
2438
				struct mem_cgroup *from, struct mem_cgroup *to)
2439 2440 2441
{
	unsigned short old_id, new_id;

L
Li Zefan 已提交
2442 2443
	old_id = mem_cgroup_id(from);
	new_id = mem_cgroup_id(to);
2444 2445 2446

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
2447
		mem_cgroup_swap_statistics(to, true);
2448 2449 2450 2451 2452 2453
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2454
				struct mem_cgroup *from, struct mem_cgroup *to)
2455 2456 2457
{
	return -EINVAL;
}
2458
#endif
K
KAMEZAWA Hiroyuki 已提交
2459

2460
static DEFINE_MUTEX(memcg_limit_mutex);
2461

2462
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2463
				   unsigned long limit)
2464
{
2465 2466 2467
	unsigned long curusage;
	unsigned long oldusage;
	bool enlarge = false;
2468
	int retry_count;
2469
	int ret;
2470 2471 2472 2473 2474 2475

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
2476 2477
	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
		      mem_cgroup_count_children(memcg);
2478

2479
	oldusage = page_counter_read(&memcg->memory);
2480

2481
	do {
2482 2483 2484 2485
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2486 2487 2488 2489

		mutex_lock(&memcg_limit_mutex);
		if (limit > memcg->memsw.limit) {
			mutex_unlock(&memcg_limit_mutex);
2490
			ret = -EINVAL;
2491 2492
			break;
		}
2493 2494 2495 2496
		if (limit > memcg->memory.limit)
			enlarge = true;
		ret = page_counter_limit(&memcg->memory, limit);
		mutex_unlock(&memcg_limit_mutex);
2497 2498 2499 2500

		if (!ret)
			break;

2501 2502
		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);

2503
		curusage = page_counter_read(&memcg->memory);
2504
		/* Usage is reduced ? */
A
Andrew Morton 已提交
2505
		if (curusage >= oldusage)
2506 2507 2508
			retry_count--;
		else
			oldusage = curusage;
2509 2510
	} while (retry_count);

2511 2512
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
2513

2514 2515 2516
	return ret;
}

L
Li Zefan 已提交
2517
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2518
					 unsigned long limit)
2519
{
2520 2521 2522
	unsigned long curusage;
	unsigned long oldusage;
	bool enlarge = false;
2523
	int retry_count;
2524
	int ret;
2525

2526
	/* see mem_cgroup_resize_res_limit */
2527 2528 2529 2530 2531 2532
	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
		      mem_cgroup_count_children(memcg);

	oldusage = page_counter_read(&memcg->memsw);

	do {
2533 2534 2535 2536
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
2537 2538 2539 2540

		mutex_lock(&memcg_limit_mutex);
		if (limit < memcg->memory.limit) {
			mutex_unlock(&memcg_limit_mutex);
2541 2542 2543
			ret = -EINVAL;
			break;
		}
2544 2545 2546 2547
		if (limit > memcg->memsw.limit)
			enlarge = true;
		ret = page_counter_limit(&memcg->memsw, limit);
		mutex_unlock(&memcg_limit_mutex);
2548 2549 2550 2551

		if (!ret)
			break;

2552 2553
		try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);

2554
		curusage = page_counter_read(&memcg->memsw);
2555
		/* Usage is reduced ? */
2556
		if (curusage >= oldusage)
2557
			retry_count--;
2558 2559
		else
			oldusage = curusage;
2560 2561
	} while (retry_count);

2562 2563
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
2564

2565 2566 2567
	return ret;
}

2568 2569 2570 2571 2572 2573 2574 2575 2576
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
2577
	unsigned long excess;
2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601
	unsigned long nr_scanned;

	if (order > 0)
		return 0;

	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

		nr_scanned = 0;
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
						    gfp_mask, &nr_scanned);
		nr_reclaimed += reclaimed;
		*total_scanned += nr_scanned;
2602
		spin_lock_irq(&mctz->lock);
2603
		__mem_cgroup_remove_exceeded(mz, mctz);
2604 2605 2606 2607 2608 2609

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
2610 2611 2612
		if (!reclaimed)
			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);

2613
		excess = soft_limit_excess(mz->memcg);
2614 2615 2616 2617 2618 2619 2620 2621 2622
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
		/* If excess == 0, no tree ops */
2623
		__mem_cgroup_insert_exceeded(mz, mctz, excess);
2624
		spin_unlock_irq(&mctz->lock);
2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641
		css_put(&mz->memcg->css);
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
		css_put(&next_mz->memcg->css);
	return nr_reclaimed;
}

2642 2643 2644 2645 2646 2647
/*
 * Test whether @memcg has children, dead or alive.  Note that this
 * function doesn't care whether @memcg has use_hierarchy enabled and
 * returns %true if there are child csses according to the cgroup
 * hierarchy.  Testing use_hierarchy is the caller's responsiblity.
 */
2648 2649
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
2650 2651 2652 2653 2654 2655
	bool ret;

	rcu_read_lock();
	ret = css_next_child(NULL, &memcg->css);
	rcu_read_unlock();
	return ret;
2656 2657
}

2658
/*
2659
 * Reclaims as many pages from the given memcg as possible.
2660 2661 2662 2663 2664 2665 2666
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;

2667 2668
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
2669
	/* try to free all pages in this cgroup */
2670
	while (nr_retries && page_counter_read(&memcg->memory)) {
2671
		int progress;
2672

2673 2674 2675
		if (signal_pending(current))
			return -EINTR;

2676 2677
		progress = try_to_free_mem_cgroup_pages(memcg, 1,
							GFP_KERNEL, true);
2678
		if (!progress) {
2679
			nr_retries--;
2680
			/* maybe some writeback is necessary */
2681
			congestion_wait(BLK_RW_ASYNC, HZ/10);
2682
		}
2683 2684

	}
2685 2686

	return 0;
2687 2688
}

2689 2690 2691
static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
					    char *buf, size_t nbytes,
					    loff_t off)
2692
{
2693
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2694

2695 2696
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
2697
	return mem_cgroup_force_empty(memcg) ?: nbytes;
2698 2699
}

2700 2701
static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
				     struct cftype *cft)
2702
{
2703
	return mem_cgroup_from_css(css)->use_hierarchy;
2704 2705
}

2706 2707
static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, u64 val)
2708 2709
{
	int retval = 0;
2710
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
2711
	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2712

2713
	if (memcg->use_hierarchy == val)
2714
		return 0;
2715

2716
	/*
2717
	 * If parent's use_hierarchy is set, we can't make any modifications
2718 2719 2720 2721 2722 2723
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
2724
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2725
				(val == 1 || val == 0)) {
2726
		if (!memcg_has_children(memcg))
2727
			memcg->use_hierarchy = val;
2728 2729 2730 2731
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
2732

2733 2734 2735
	return retval;
}

2736
static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2737 2738
{
	struct mem_cgroup *iter;
2739
	int i;
2740

2741
	memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2742

2743 2744 2745 2746
	for_each_mem_cgroup_tree(iter, memcg) {
		for (i = 0; i < MEMCG_NR_STAT; i++)
			stat[i] += mem_cgroup_read_stat(iter, i);
	}
2747 2748
}

2749
static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2750 2751
{
	struct mem_cgroup *iter;
2752
	int i;
2753

2754
	memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2755

2756 2757 2758 2759
	for_each_mem_cgroup_tree(iter, memcg) {
		for (i = 0; i < MEMCG_NR_EVENTS; i++)
			events[i] += mem_cgroup_read_events(iter, i);
	}
2760 2761
}

2762
static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2763
{
2764
	unsigned long val = 0;
2765

2766
	if (mem_cgroup_is_root(memcg)) {
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777
		struct mem_cgroup *iter;

		for_each_mem_cgroup_tree(iter, memcg) {
			val += mem_cgroup_read_stat(iter,
					MEM_CGROUP_STAT_CACHE);
			val += mem_cgroup_read_stat(iter,
					MEM_CGROUP_STAT_RSS);
			if (swap)
				val += mem_cgroup_read_stat(iter,
						MEM_CGROUP_STAT_SWAP);
		}
2778
	} else {
2779
		if (!swap)
2780
			val = page_counter_read(&memcg->memory);
2781
		else
2782
			val = page_counter_read(&memcg->memsw);
2783
	}
2784
	return val;
2785 2786
}

2787 2788 2789 2790 2791 2792 2793
enum {
	RES_USAGE,
	RES_LIMIT,
	RES_MAX_USAGE,
	RES_FAILCNT,
	RES_SOFT_LIMIT,
};
2794

2795
static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2796
			       struct cftype *cft)
B
Balbir Singh 已提交
2797
{
2798
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2799
	struct page_counter *counter;
2800

2801
	switch (MEMFILE_TYPE(cft->private)) {
2802
	case _MEM:
2803 2804
		counter = &memcg->memory;
		break;
2805
	case _MEMSWAP:
2806 2807
		counter = &memcg->memsw;
		break;
2808
	case _KMEM:
2809
		counter = &memcg->kmem;
2810
		break;
V
Vladimir Davydov 已提交
2811
	case _TCP:
2812
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
2813
		break;
2814 2815 2816
	default:
		BUG();
	}
2817 2818 2819 2820

	switch (MEMFILE_ATTR(cft->private)) {
	case RES_USAGE:
		if (counter == &memcg->memory)
2821
			return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2822
		if (counter == &memcg->memsw)
2823
			return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835
		return (u64)page_counter_read(counter) * PAGE_SIZE;
	case RES_LIMIT:
		return (u64)counter->limit * PAGE_SIZE;
	case RES_MAX_USAGE:
		return (u64)counter->watermark * PAGE_SIZE;
	case RES_FAILCNT:
		return counter->failcnt;
	case RES_SOFT_LIMIT:
		return (u64)memcg->soft_limit * PAGE_SIZE;
	default:
		BUG();
	}
B
Balbir Singh 已提交
2836
}
2837

2838
#ifndef CONFIG_SLOB
2839
static int memcg_online_kmem(struct mem_cgroup *memcg)
2840 2841 2842
{
	int memcg_id;

2843 2844 2845
	if (cgroup_memory_nokmem)
		return 0;

2846
	BUG_ON(memcg->kmemcg_id >= 0);
2847
	BUG_ON(memcg->kmem_state);
2848

2849
	memcg_id = memcg_alloc_cache_id();
2850 2851
	if (memcg_id < 0)
		return memcg_id;
2852

2853
	static_branch_inc(&memcg_kmem_enabled_key);
2854
	/*
2855
	 * A memory cgroup is considered kmem-online as soon as it gets
V
Vladimir Davydov 已提交
2856
	 * kmemcg_id. Setting the id after enabling static branching will
2857 2858 2859
	 * guarantee no one starts accounting before all call sites are
	 * patched.
	 */
V
Vladimir Davydov 已提交
2860
	memcg->kmemcg_id = memcg_id;
2861
	memcg->kmem_state = KMEM_ONLINE;
2862 2863

	return 0;
2864 2865
}

2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
	struct cgroup_subsys_state *css;
	struct mem_cgroup *parent, *child;
	int kmemcg_id;

	if (memcg->kmem_state != KMEM_ONLINE)
		return;
	/*
	 * Clear the online state before clearing memcg_caches array
	 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
	 * guarantees that no cache will be created for this cgroup
	 * after we are done (see memcg_create_kmem_cache()).
	 */
	memcg->kmem_state = KMEM_ALLOCATED;

	memcg_deactivate_kmem_caches(memcg);

	kmemcg_id = memcg->kmemcg_id;
	BUG_ON(kmemcg_id < 0);

	parent = parent_mem_cgroup(memcg);
	if (!parent)
		parent = root_mem_cgroup;

	/*
	 * Change kmemcg_id of this cgroup and all its descendants to the
	 * parent's id, and then move all entries from this cgroup's list_lrus
	 * to ones of the parent. After we have finished, all list_lrus
	 * corresponding to this cgroup are guaranteed to remain empty. The
	 * ordering is imposed by list_lru_node->lock taken by
	 * memcg_drain_all_list_lrus().
	 */
2899
	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2900 2901 2902 2903 2904 2905 2906
	css_for_each_descendant_pre(css, &memcg->css) {
		child = mem_cgroup_from_css(css);
		BUG_ON(child->kmemcg_id != kmemcg_id);
		child->kmemcg_id = parent->kmemcg_id;
		if (!memcg->use_hierarchy)
			break;
	}
2907 2908
	rcu_read_unlock();

2909 2910 2911 2912 2913 2914 2915
	memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);

	memcg_free_cache_id(kmemcg_id);
}

static void memcg_free_kmem(struct mem_cgroup *memcg)
{
2916 2917 2918 2919
	/* css_alloc() failed, offlining didn't happen */
	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
		memcg_offline_kmem(memcg);

2920 2921 2922 2923 2924 2925
	if (memcg->kmem_state == KMEM_ALLOCATED) {
		memcg_destroy_kmem_caches(memcg);
		static_branch_dec(&memcg_kmem_enabled_key);
		WARN_ON(page_counter_read(&memcg->kmem));
	}
}
2926
#else
2927
static int memcg_online_kmem(struct mem_cgroup *memcg)
2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938
{
	return 0;
}
static void memcg_offline_kmem(struct mem_cgroup *memcg)
{
}
static void memcg_free_kmem(struct mem_cgroup *memcg)
{
}
#endif /* !CONFIG_SLOB */

2939
static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2940
				   unsigned long limit)
2941
{
2942
	int ret;
2943 2944 2945 2946 2947

	mutex_lock(&memcg_limit_mutex);
	ret = page_counter_limit(&memcg->kmem, limit);
	mutex_unlock(&memcg_limit_mutex);
	return ret;
2948
}
2949

V
Vladimir Davydov 已提交
2950 2951 2952 2953 2954 2955
static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
{
	int ret;

	mutex_lock(&memcg_limit_mutex);

2956
	ret = page_counter_limit(&memcg->tcpmem, limit);
V
Vladimir Davydov 已提交
2957 2958 2959
	if (ret)
		goto out;

2960
	if (!memcg->tcpmem_active) {
V
Vladimir Davydov 已提交
2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977
		/*
		 * The active flag needs to be written after the static_key
		 * update. This is what guarantees that the socket activation
		 * function is the last one to run. See sock_update_memcg() for
		 * details, and note that we don't mark any socket as belonging
		 * to this memcg until that flag is up.
		 *
		 * We need to do this, because static_keys will span multiple
		 * sites, but we can't control their order. If we mark a socket
		 * as accounted, but the accounting functions are not patched in
		 * yet, we'll lose accounting.
		 *
		 * We never race with the readers in sock_update_memcg(),
		 * because when this value change, the code to process it is not
		 * patched in yet.
		 */
		static_branch_inc(&memcg_sockets_enabled_key);
2978
		memcg->tcpmem_active = true;
V
Vladimir Davydov 已提交
2979 2980 2981 2982 2983 2984
	}
out:
	mutex_unlock(&memcg_limit_mutex);
	return ret;
}

2985 2986 2987 2988
/*
 * The user of this function is...
 * RES_LIMIT.
 */
2989 2990
static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
B
Balbir Singh 已提交
2991
{
2992
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2993
	unsigned long nr_pages;
2994 2995
	int ret;

2996
	buf = strstrip(buf);
2997
	ret = page_counter_memparse(buf, "-1", &nr_pages);
2998 2999
	if (ret)
		return ret;
3000

3001
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3002
	case RES_LIMIT:
3003 3004 3005 3006
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
3007 3008 3009
		switch (MEMFILE_TYPE(of_cft(of)->private)) {
		case _MEM:
			ret = mem_cgroup_resize_limit(memcg, nr_pages);
3010
			break;
3011 3012
		case _MEMSWAP:
			ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3013
			break;
3014 3015 3016
		case _KMEM:
			ret = memcg_update_kmem_limit(memcg, nr_pages);
			break;
V
Vladimir Davydov 已提交
3017 3018 3019
		case _TCP:
			ret = memcg_update_tcp_limit(memcg, nr_pages);
			break;
3020
		}
3021
		break;
3022 3023 3024
	case RES_SOFT_LIMIT:
		memcg->soft_limit = nr_pages;
		ret = 0;
3025 3026
		break;
	}
3027
	return ret ?: nbytes;
B
Balbir Singh 已提交
3028 3029
}

3030 3031
static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
				size_t nbytes, loff_t off)
3032
{
3033
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3034
	struct page_counter *counter;
3035

3036 3037 3038 3039 3040 3041 3042 3043 3044 3045
	switch (MEMFILE_TYPE(of_cft(of)->private)) {
	case _MEM:
		counter = &memcg->memory;
		break;
	case _MEMSWAP:
		counter = &memcg->memsw;
		break;
	case _KMEM:
		counter = &memcg->kmem;
		break;
V
Vladimir Davydov 已提交
3046
	case _TCP:
3047
		counter = &memcg->tcpmem;
V
Vladimir Davydov 已提交
3048
		break;
3049 3050 3051
	default:
		BUG();
	}
3052

3053
	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3054
	case RES_MAX_USAGE:
3055
		page_counter_reset_watermark(counter);
3056 3057
		break;
	case RES_FAILCNT:
3058
		counter->failcnt = 0;
3059
		break;
3060 3061
	default:
		BUG();
3062
	}
3063

3064
	return nbytes;
3065 3066
}

3067
static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3068 3069
					struct cftype *cft)
{
3070
	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3071 3072
}

3073
#ifdef CONFIG_MMU
3074
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3075 3076
					struct cftype *cft, u64 val)
{
3077
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3078

3079
	if (val & ~MOVE_MASK)
3080
		return -EINVAL;
3081

3082
	/*
3083 3084 3085 3086
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
3087
	 */
3088
	memcg->move_charge_at_immigrate = val;
3089 3090
	return 0;
}
3091
#else
3092
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3093 3094 3095 3096 3097
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
3098

3099
#ifdef CONFIG_NUMA
3100
static int memcg_numa_stat_show(struct seq_file *m, void *v)
3101
{
3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113
	struct numa_stat {
		const char *name;
		unsigned int lru_mask;
	};

	static const struct numa_stat stats[] = {
		{ "total", LRU_ALL },
		{ "file", LRU_ALL_FILE },
		{ "anon", LRU_ALL_ANON },
		{ "unevictable", BIT(LRU_UNEVICTABLE) },
	};
	const struct numa_stat *stat;
3114
	int nid;
3115
	unsigned long nr;
3116
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3117

3118 3119 3120 3121 3122 3123 3124 3125 3126
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
		seq_printf(m, "%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
							  stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3127 3128
	}

3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143
	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
		struct mem_cgroup *iter;

		nr = 0;
		for_each_mem_cgroup_tree(iter, memcg)
			nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
		seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
		for_each_node_state(nid, N_MEMORY) {
			nr = 0;
			for_each_mem_cgroup_tree(iter, memcg)
				nr += mem_cgroup_node_nr_lru_pages(
					iter, nid, stat->lru_mask);
			seq_printf(m, " N%d=%lu", nid, nr);
		}
		seq_putc(m, '\n');
3144 3145 3146 3147 3148 3149
	}

	return 0;
}
#endif /* CONFIG_NUMA */

3150
static int memcg_stat_show(struct seq_file *m, void *v)
3151
{
3152
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3153
	unsigned long memory, memsw;
3154 3155
	struct mem_cgroup *mi;
	unsigned int i;
3156

3157 3158 3159 3160
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
		     MEM_CGROUP_STAT_NSTATS);
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
		     MEM_CGROUP_EVENTS_NSTATS);
3161 3162
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);

3163
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3164
		if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3165
			continue;
3166
		seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
3167
			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3168
	}
L
Lee Schermerhorn 已提交
3169

3170 3171 3172 3173 3174 3175 3176 3177
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
			   mem_cgroup_read_events(memcg, i));

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
3178
	/* Hierarchical information */
3179 3180 3181 3182
	memory = memsw = PAGE_COUNTER_MAX;
	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
		memory = min(memory, mi->memory.limit);
		memsw = min(memsw, mi->memsw.limit);
3183
	}
3184 3185
	seq_printf(m, "hierarchical_memory_limit %llu\n",
		   (u64)memory * PAGE_SIZE);
3186
	if (do_memsw_account())
3187 3188
		seq_printf(m, "hierarchical_memsw_limit %llu\n",
			   (u64)memsw * PAGE_SIZE);
K
KOSAKI Motohiro 已提交
3189

3190
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3191
		unsigned long long val = 0;
3192

3193
		if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3194
			continue;
3195 3196
		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
3197
		seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214
	}

	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_events(mi, i);
		seq_printf(m, "total_%s %llu\n",
			   mem_cgroup_events_names[i], val);
	}

	for (i = 0; i < NR_LRU_LISTS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3215
	}
K
KAMEZAWA Hiroyuki 已提交
3216

K
KOSAKI Motohiro 已提交
3217 3218 3219 3220
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
3221
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
3222 3223 3224 3225 3226
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3227
				mz = &memcg->nodeinfo[nid]->zoneinfo[zid];
3228
				rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
3229

3230 3231 3232 3233
				recent_rotated[0] += rstat->recent_rotated[0];
				recent_rotated[1] += rstat->recent_rotated[1];
				recent_scanned[0] += rstat->recent_scanned[0];
				recent_scanned[1] += rstat->recent_scanned[1];
K
KOSAKI Motohiro 已提交
3234
			}
3235 3236 3237 3238
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
3239 3240 3241
	}
#endif

3242 3243 3244
	return 0;
}

3245 3246
static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
K
KOSAKI Motohiro 已提交
3247
{
3248
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3249

3250
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
3251 3252
}

3253 3254
static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
K
KOSAKI Motohiro 已提交
3255
{
3256
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
3257

3258
	if (val > 100)
K
KOSAKI Motohiro 已提交
3259 3260
		return -EINVAL;

3261
	if (css->parent)
3262 3263 3264
		memcg->swappiness = val;
	else
		vm_swappiness = val;
3265

K
KOSAKI Motohiro 已提交
3266 3267 3268
	return 0;
}

3269 3270 3271
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
3272
	unsigned long usage;
3273 3274 3275 3276
	int i;

	rcu_read_lock();
	if (!swap)
3277
		t = rcu_dereference(memcg->thresholds.primary);
3278
	else
3279
		t = rcu_dereference(memcg->memsw_thresholds.primary);
3280 3281 3282 3283

	if (!t)
		goto unlock;

3284
	usage = mem_cgroup_usage(memcg, swap);
3285 3286

	/*
3287
	 * current_threshold points to threshold just below or equal to usage.
3288 3289 3290
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
3291
	i = t->current_threshold;
3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
3315
	t->current_threshold = i - 1;
3316 3317 3318 3319 3320 3321
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
3322 3323
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
3324
		if (do_memsw_account())
3325 3326 3327 3328
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
3329 3330 3331 3332 3333 3334 3335
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

3336 3337 3338 3339 3340 3341 3342
	if (_a->threshold > _b->threshold)
		return 1;

	if (_a->threshold < _b->threshold)
		return -1;

	return 0;
3343 3344
}

3345
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3346 3347 3348
{
	struct mem_cgroup_eventfd_list *ev;

3349 3350
	spin_lock(&memcg_oom_lock);

3351
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
3352
		eventfd_signal(ev->eventfd, 1);
3353 3354

	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3355 3356 3357
	return 0;
}

3358
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
3359
{
K
KAMEZAWA Hiroyuki 已提交
3360 3361
	struct mem_cgroup *iter;

3362
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
3363
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
3364 3365
}

3366
static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3367
	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3368
{
3369 3370
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3371 3372
	unsigned long threshold;
	unsigned long usage;
3373
	int i, size, ret;
3374

3375
	ret = page_counter_memparse(args, "-1", &threshold);
3376 3377 3378 3379
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
3380

3381
	if (type == _MEM) {
3382
		thresholds = &memcg->thresholds;
3383
		usage = mem_cgroup_usage(memcg, false);
3384
	} else if (type == _MEMSWAP) {
3385
		thresholds = &memcg->memsw_thresholds;
3386
		usage = mem_cgroup_usage(memcg, true);
3387
	} else
3388 3389 3390
		BUG();

	/* Check if a threshold crossed before adding a new one */
3391
	if (thresholds->primary)
3392 3393
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

3394
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3395 3396

	/* Allocate memory for new array of thresholds */
3397
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3398
			GFP_KERNEL);
3399
	if (!new) {
3400 3401 3402
		ret = -ENOMEM;
		goto unlock;
	}
3403
	new->size = size;
3404 3405

	/* Copy thresholds (if any) to new array */
3406 3407
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3408
				sizeof(struct mem_cgroup_threshold));
3409 3410
	}

3411
	/* Add new threshold */
3412 3413
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
3414 3415

	/* Sort thresholds. Registering of new threshold isn't time-critical */
3416
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3417 3418 3419
			compare_thresholds, NULL);

	/* Find current threshold */
3420
	new->current_threshold = -1;
3421
	for (i = 0; i < size; i++) {
3422
		if (new->entries[i].threshold <= usage) {
3423
			/*
3424 3425
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
3426 3427
			 * it here.
			 */
3428
			++new->current_threshold;
3429 3430
		} else
			break;
3431 3432
	}

3433 3434 3435 3436 3437
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
3438

3439
	/* To be sure that nobody uses thresholds */
3440 3441 3442 3443 3444 3445 3446 3447
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

3448
static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3449 3450
	struct eventfd_ctx *eventfd, const char *args)
{
3451
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
T
Tejun Heo 已提交
3452 3453
}

3454
static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3455 3456
	struct eventfd_ctx *eventfd, const char *args)
{
3457
	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
T
Tejun Heo 已提交
3458 3459
}

3460
static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3461
	struct eventfd_ctx *eventfd, enum res_type type)
3462
{
3463 3464
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
3465
	unsigned long usage;
3466
	int i, j, size;
3467 3468

	mutex_lock(&memcg->thresholds_lock);
3469 3470

	if (type == _MEM) {
3471
		thresholds = &memcg->thresholds;
3472
		usage = mem_cgroup_usage(memcg, false);
3473
	} else if (type == _MEMSWAP) {
3474
		thresholds = &memcg->memsw_thresholds;
3475
		usage = mem_cgroup_usage(memcg, true);
3476
	} else
3477 3478
		BUG();

3479 3480 3481
	if (!thresholds->primary)
		goto unlock;

3482 3483 3484 3485
	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
3486 3487 3488
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
3489 3490 3491
			size++;
	}

3492
	new = thresholds->spare;
3493

3494 3495
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
3496 3497
		kfree(new);
		new = NULL;
3498
		goto swap_buffers;
3499 3500
	}

3501
	new->size = size;
3502 3503

	/* Copy thresholds and find current threshold */
3504 3505 3506
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
3507 3508
			continue;

3509
		new->entries[j] = thresholds->primary->entries[i];
3510
		if (new->entries[j].threshold <= usage) {
3511
			/*
3512
			 * new->current_threshold will not be used
3513 3514 3515
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
3516
			++new->current_threshold;
3517 3518 3519 3520
		}
		j++;
	}

3521
swap_buffers:
3522 3523
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
3524

3525
	rcu_assign_pointer(thresholds->primary, new);
3526

3527
	/* To be sure that nobody uses thresholds */
3528
	synchronize_rcu();
3529 3530 3531 3532 3533 3534

	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}
3535
unlock:
3536 3537
	mutex_unlock(&memcg->thresholds_lock);
}
3538

3539
static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3540 3541
	struct eventfd_ctx *eventfd)
{
3542
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
T
Tejun Heo 已提交
3543 3544
}

3545
static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3546 3547
	struct eventfd_ctx *eventfd)
{
3548
	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
T
Tejun Heo 已提交
3549 3550
}

3551
static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3552
	struct eventfd_ctx *eventfd, const char *args)
K
KAMEZAWA Hiroyuki 已提交
3553 3554 3555 3556 3557 3558 3559
{
	struct mem_cgroup_eventfd_list *event;

	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

3560
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3561 3562 3563 3564 3565

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
3566
	if (memcg->under_oom)
K
KAMEZAWA Hiroyuki 已提交
3567
		eventfd_signal(eventfd, 1);
3568
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3569 3570 3571 3572

	return 0;
}

3573
static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
T
Tejun Heo 已提交
3574
	struct eventfd_ctx *eventfd)
K
KAMEZAWA Hiroyuki 已提交
3575 3576 3577
{
	struct mem_cgroup_eventfd_list *ev, *tmp;

3578
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3579

3580
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
3581 3582 3583 3584 3585 3586
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

3587
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
3588 3589
}

3590
static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3591
{
3592
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3593

3594
	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3595
	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3596 3597 3598
	return 0;
}

3599
static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3600 3601
	struct cftype *cft, u64 val)
{
3602
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3603 3604

	/* cannot set to root cgroup and only 0 and 1 are allowed */
3605
	if (!css->parent || !((val == 0) || (val == 1)))
3606 3607
		return -EINVAL;

3608
	memcg->oom_kill_disable = val;
3609
	if (!val)
3610
		memcg_oom_recover(memcg);
3611

3612 3613 3614
	return 0;
}

3615 3616 3617 3618 3619 3620 3621
#ifdef CONFIG_CGROUP_WRITEBACK

struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
{
	return &memcg->cgwb_list;
}

T
Tejun Heo 已提交
3622 3623 3624 3625 3626 3627 3628 3629 3630 3631
static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return wb_domain_init(&memcg->cgwb_domain, gfp);
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
	wb_domain_exit(&memcg->cgwb_domain);
}

3632 3633 3634 3635 3636
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
	wb_domain_size_changed(&memcg->cgwb_domain);
}

T
Tejun Heo 已提交
3637 3638 3639 3640 3641 3642 3643 3644 3645 3646
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);

	if (!memcg->css.parent)
		return NULL;

	return &memcg->cgwb_domain;
}

3647 3648 3649
/**
 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
 * @wb: bdi_writeback in question
3650 3651
 * @pfilepages: out parameter for number of file pages
 * @pheadroom: out parameter for number of allocatable pages according to memcg
3652 3653 3654
 * @pdirty: out parameter for number of dirty pages
 * @pwriteback: out parameter for number of pages under writeback
 *
3655 3656 3657
 * Determine the numbers of file, headroom, dirty, and writeback pages in
 * @wb's memcg.  File, dirty and writeback are self-explanatory.  Headroom
 * is a bit more involved.
3658
 *
3659 3660 3661 3662 3663
 * A memcg's headroom is "min(max, high) - used".  In the hierarchy, the
 * headroom is calculated as the lowest headroom of itself and the
 * ancestors.  Note that this doesn't consider the actual amount of
 * available memory in the system.  The caller should further cap
 * *@pheadroom accordingly.
3664
 */
3665 3666 3667
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback)
3668 3669 3670 3671 3672 3673 3674 3675
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
	struct mem_cgroup *parent;

	*pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY);

	/* this should eventually include NR_UNSTABLE_NFS */
	*pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK);
3676 3677 3678
	*pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
						     (1 << LRU_ACTIVE_FILE));
	*pheadroom = PAGE_COUNTER_MAX;
3679 3680 3681 3682 3683

	while ((parent = parent_mem_cgroup(memcg))) {
		unsigned long ceiling = min(memcg->memory.limit, memcg->high);
		unsigned long used = page_counter_read(&memcg->memory);

3684
		*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3685 3686 3687 3688
		memcg = parent;
	}
}

T
Tejun Heo 已提交
3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699
#else	/* CONFIG_CGROUP_WRITEBACK */

static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
{
	return 0;
}

static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
{
}

3700 3701 3702 3703
static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
{
}

3704 3705
#endif	/* CONFIG_CGROUP_WRITEBACK */

3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718
/*
 * DO NOT USE IN NEW FILES.
 *
 * "cgroup.event_control" implementation.
 *
 * This is way over-engineered.  It tries to support fully configurable
 * events for each user.  Such level of flexibility is completely
 * unnecessary especially in the light of the planned unified hierarchy.
 *
 * Please deprecate this and replace with something simpler if at all
 * possible.
 */

3719 3720 3721 3722 3723
/*
 * Unregister event and free resources.
 *
 * Gets called from workqueue.
 */
3724
static void memcg_event_remove(struct work_struct *work)
3725
{
3726 3727
	struct mem_cgroup_event *event =
		container_of(work, struct mem_cgroup_event, remove);
3728
	struct mem_cgroup *memcg = event->memcg;
3729 3730 3731

	remove_wait_queue(event->wqh, &event->wait);

3732
	event->unregister_event(memcg, event->eventfd);
3733 3734 3735 3736 3737 3738

	/* Notify userspace the event is going away. */
	eventfd_signal(event->eventfd, 1);

	eventfd_ctx_put(event->eventfd);
	kfree(event);
3739
	css_put(&memcg->css);
3740 3741 3742 3743 3744 3745 3746
}

/*
 * Gets called on POLLHUP on eventfd when user closes it.
 *
 * Called with wqh->lock held and interrupts disabled.
 */
3747 3748
static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
			    int sync, void *key)
3749
{
3750 3751
	struct mem_cgroup_event *event =
		container_of(wait, struct mem_cgroup_event, wait);
3752
	struct mem_cgroup *memcg = event->memcg;
3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764
	unsigned long flags = (unsigned long)key;

	if (flags & POLLHUP) {
		/*
		 * If the event has been detached at cgroup removal, we
		 * can simply return knowing the other side will cleanup
		 * for us.
		 *
		 * We can't race against event freeing since the other
		 * side will require wqh->lock via remove_wait_queue(),
		 * which we hold.
		 */
3765
		spin_lock(&memcg->event_list_lock);
3766 3767 3768 3769 3770 3771 3772 3773
		if (!list_empty(&event->list)) {
			list_del_init(&event->list);
			/*
			 * We are in atomic context, but cgroup_event_remove()
			 * may sleep, so we have to call it in workqueue.
			 */
			schedule_work(&event->remove);
		}
3774
		spin_unlock(&memcg->event_list_lock);
3775 3776 3777 3778 3779
	}

	return 0;
}

3780
static void memcg_event_ptable_queue_proc(struct file *file,
3781 3782
		wait_queue_head_t *wqh, poll_table *pt)
{
3783 3784
	struct mem_cgroup_event *event =
		container_of(pt, struct mem_cgroup_event, pt);
3785 3786 3787 3788 3789 3790

	event->wqh = wqh;
	add_wait_queue(wqh, &event->wait);
}

/*
3791 3792
 * DO NOT USE IN NEW FILES.
 *
3793 3794 3795 3796 3797
 * Parse input and register new cgroup event handler.
 *
 * Input must be in format '<event_fd> <control_fd> <args>'.
 * Interpretation of args is defined by control file implementation.
 */
3798 3799
static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
					 char *buf, size_t nbytes, loff_t off)
3800
{
3801
	struct cgroup_subsys_state *css = of_css(of);
3802
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3803
	struct mem_cgroup_event *event;
3804 3805 3806 3807
	struct cgroup_subsys_state *cfile_css;
	unsigned int efd, cfd;
	struct fd efile;
	struct fd cfile;
3808
	const char *name;
3809 3810 3811
	char *endp;
	int ret;

3812 3813 3814
	buf = strstrip(buf);

	efd = simple_strtoul(buf, &endp, 10);
3815 3816
	if (*endp != ' ')
		return -EINVAL;
3817
	buf = endp + 1;
3818

3819
	cfd = simple_strtoul(buf, &endp, 10);
3820 3821
	if ((*endp != ' ') && (*endp != '\0'))
		return -EINVAL;
3822
	buf = endp + 1;
3823 3824 3825 3826 3827

	event = kzalloc(sizeof(*event), GFP_KERNEL);
	if (!event)
		return -ENOMEM;

3828
	event->memcg = memcg;
3829
	INIT_LIST_HEAD(&event->list);
3830 3831 3832
	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
	INIT_WORK(&event->remove, memcg_event_remove);
3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857

	efile = fdget(efd);
	if (!efile.file) {
		ret = -EBADF;
		goto out_kfree;
	}

	event->eventfd = eventfd_ctx_fileget(efile.file);
	if (IS_ERR(event->eventfd)) {
		ret = PTR_ERR(event->eventfd);
		goto out_put_efile;
	}

	cfile = fdget(cfd);
	if (!cfile.file) {
		ret = -EBADF;
		goto out_put_eventfd;
	}

	/* the process need read permission on control file */
	/* AV: shouldn't we check that it's been opened for read instead? */
	ret = inode_permission(file_inode(cfile.file), MAY_READ);
	if (ret < 0)
		goto out_put_cfile;

3858 3859 3860 3861 3862
	/*
	 * Determine the event callbacks and set them in @event.  This used
	 * to be done via struct cftype but cgroup core no longer knows
	 * about these events.  The following is crude but the whole thing
	 * is for compatibility anyway.
3863 3864
	 *
	 * DO NOT ADD NEW FILES.
3865
	 */
A
Al Viro 已提交
3866
	name = cfile.file->f_path.dentry->d_name.name;
3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877

	if (!strcmp(name, "memory.usage_in_bytes")) {
		event->register_event = mem_cgroup_usage_register_event;
		event->unregister_event = mem_cgroup_usage_unregister_event;
	} else if (!strcmp(name, "memory.oom_control")) {
		event->register_event = mem_cgroup_oom_register_event;
		event->unregister_event = mem_cgroup_oom_unregister_event;
	} else if (!strcmp(name, "memory.pressure_level")) {
		event->register_event = vmpressure_register_event;
		event->unregister_event = vmpressure_unregister_event;
	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
T
Tejun Heo 已提交
3878 3879
		event->register_event = memsw_cgroup_usage_register_event;
		event->unregister_event = memsw_cgroup_usage_unregister_event;
3880 3881 3882 3883 3884
	} else {
		ret = -EINVAL;
		goto out_put_cfile;
	}

3885
	/*
3886 3887 3888
	 * Verify @cfile should belong to @css.  Also, remaining events are
	 * automatically removed on cgroup destruction but the removal is
	 * asynchronous, so take an extra ref on @css.
3889
	 */
A
Al Viro 已提交
3890
	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3891
					       &memory_cgrp_subsys);
3892
	ret = -EINVAL;
3893
	if (IS_ERR(cfile_css))
3894
		goto out_put_cfile;
3895 3896
	if (cfile_css != css) {
		css_put(cfile_css);
3897
		goto out_put_cfile;
3898
	}
3899

3900
	ret = event->register_event(memcg, event->eventfd, buf);
3901 3902 3903 3904 3905
	if (ret)
		goto out_put_css;

	efile.file->f_op->poll(efile.file, &event->pt);

3906 3907 3908
	spin_lock(&memcg->event_list_lock);
	list_add(&event->list, &memcg->event_list);
	spin_unlock(&memcg->event_list_lock);
3909 3910 3911 3912

	fdput(cfile);
	fdput(efile);

3913
	return nbytes;
3914 3915

out_put_css:
3916
	css_put(css);
3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928
out_put_cfile:
	fdput(cfile);
out_put_eventfd:
	eventfd_ctx_put(event->eventfd);
out_put_efile:
	fdput(efile);
out_kfree:
	kfree(event);

	return ret;
}

3929
static struct cftype mem_cgroup_legacy_files[] = {
B
Balbir Singh 已提交
3930
	{
3931
		.name = "usage_in_bytes",
3932
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3933
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
3934
	},
3935 3936
	{
		.name = "max_usage_in_bytes",
3937
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3938
		.write = mem_cgroup_reset,
3939
		.read_u64 = mem_cgroup_read_u64,
3940
	},
B
Balbir Singh 已提交
3941
	{
3942
		.name = "limit_in_bytes",
3943
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3944
		.write = mem_cgroup_write,
3945
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
3946
	},
3947 3948 3949
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3950
		.write = mem_cgroup_write,
3951
		.read_u64 = mem_cgroup_read_u64,
3952
	},
B
Balbir Singh 已提交
3953 3954
	{
		.name = "failcnt",
3955
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3956
		.write = mem_cgroup_reset,
3957
		.read_u64 = mem_cgroup_read_u64,
B
Balbir Singh 已提交
3958
	},
3959 3960
	{
		.name = "stat",
3961
		.seq_show = memcg_stat_show,
3962
	},
3963 3964
	{
		.name = "force_empty",
3965
		.write = mem_cgroup_force_empty_write,
3966
	},
3967 3968 3969 3970 3971
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
3972
	{
3973
		.name = "cgroup.event_control",		/* XXX: for compat */
3974
		.write = memcg_write_event_control,
3975
		.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3976
	},
K
KOSAKI Motohiro 已提交
3977 3978 3979 3980 3981
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
3982 3983 3984 3985 3986
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
3987 3988
	{
		.name = "oom_control",
3989
		.seq_show = mem_cgroup_oom_control_read,
3990
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
3991 3992
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
3993 3994 3995
	{
		.name = "pressure_level",
	},
3996 3997 3998
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
3999
		.seq_show = memcg_numa_stat_show,
4000 4001
	},
#endif
4002 4003 4004
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
4005
		.write = mem_cgroup_write,
4006
		.read_u64 = mem_cgroup_read_u64,
4007 4008 4009 4010
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4011
		.read_u64 = mem_cgroup_read_u64,
4012 4013 4014 4015
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4016
		.write = mem_cgroup_reset,
4017
		.read_u64 = mem_cgroup_read_u64,
4018 4019 4020 4021
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4022
		.write = mem_cgroup_reset,
4023
		.read_u64 = mem_cgroup_read_u64,
4024
	},
4025 4026 4027
#ifdef CONFIG_SLABINFO
	{
		.name = "kmem.slabinfo",
4028 4029 4030 4031
		.seq_start = slab_start,
		.seq_next = slab_next,
		.seq_stop = slab_stop,
		.seq_show = memcg_slab_show,
4032 4033
	},
#endif
V
Vladimir Davydov 已提交
4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056
	{
		.name = "kmem.tcp.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.failcnt",
		.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "kmem.tcp.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
4057
	{ },	/* terminate */
4058
};
4059

4060
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4061 4062
{
	struct mem_cgroup_per_node *pn;
4063
	struct mem_cgroup_per_zone *mz;
4064
	int zone, tmp = node;
4065 4066 4067 4068 4069 4070 4071 4072
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
4073 4074
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
4075
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4076 4077
	if (!pn)
		return 1;
4078 4079 4080

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
4081
		lruvec_init(&mz->lruvec);
4082 4083
		mz->usage_in_excess = 0;
		mz->on_tree = false;
4084
		mz->memcg = memcg;
4085
	}
4086
	memcg->nodeinfo[node] = pn;
4087 4088 4089
	return 0;
}

4090
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
4091
{
4092
	kfree(memcg->nodeinfo[node]);
4093 4094
}

4095
static void mem_cgroup_free(struct mem_cgroup *memcg)
4096
{
4097
	int node;
4098

4099
	memcg_wb_domain_exit(memcg);
4100 4101 4102
	for_each_node(node)
		free_mem_cgroup_per_zone_info(memcg, node);
	free_percpu(memcg->stat);
4103
	kfree(memcg);
4104
}
4105

4106
static struct mem_cgroup *mem_cgroup_alloc(void)
B
Balbir Singh 已提交
4107
{
4108
	struct mem_cgroup *memcg;
4109
	size_t size;
4110
	int node;
B
Balbir Singh 已提交
4111

4112 4113 4114 4115
	size = sizeof(struct mem_cgroup);
	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);

	memcg = kzalloc(size, GFP_KERNEL);
4116
	if (!memcg)
4117 4118 4119 4120 4121
		return NULL;

	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat)
		goto fail;
4122

B
Bob Liu 已提交
4123
	for_each_node(node)
4124
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
4125
			goto fail;
4126

4127 4128
	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
		goto fail;
4129

4130
	INIT_WORK(&memcg->high_work, high_work_func);
4131 4132 4133 4134
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);
4135
	vmpressure_init(&memcg->vmpressure);
4136 4137
	INIT_LIST_HEAD(&memcg->event_list);
	spin_lock_init(&memcg->event_list_lock);
4138
	memcg->socket_pressure = jiffies;
4139
#ifndef CONFIG_SLOB
V
Vladimir Davydov 已提交
4140 4141
	memcg->kmemcg_id = -1;
#endif
4142 4143 4144
#ifdef CONFIG_CGROUP_WRITEBACK
	INIT_LIST_HEAD(&memcg->cgwb_list);
#endif
4145 4146 4147 4148
	return memcg;
fail:
	mem_cgroup_free(memcg);
	return NULL;
4149 4150
}

4151 4152
static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4153
{
4154 4155 4156
	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
	struct mem_cgroup *memcg;
	long error = -ENOMEM;
4157

4158 4159 4160
	memcg = mem_cgroup_alloc();
	if (!memcg)
		return ERR_PTR(error);
4161

4162 4163 4164 4165 4166 4167 4168 4169
	memcg->high = PAGE_COUNTER_MAX;
	memcg->soft_limit = PAGE_COUNTER_MAX;
	if (parent) {
		memcg->swappiness = mem_cgroup_swappiness(parent);
		memcg->oom_kill_disable = parent->oom_kill_disable;
	}
	if (parent && parent->use_hierarchy) {
		memcg->use_hierarchy = true;
4170
		page_counter_init(&memcg->memory, &parent->memory);
4171
		page_counter_init(&memcg->swap, &parent->swap);
4172 4173
		page_counter_init(&memcg->memsw, &parent->memsw);
		page_counter_init(&memcg->kmem, &parent->kmem);
4174
		page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4175
	} else {
4176
		page_counter_init(&memcg->memory, NULL);
4177
		page_counter_init(&memcg->swap, NULL);
4178 4179
		page_counter_init(&memcg->memsw, NULL);
		page_counter_init(&memcg->kmem, NULL);
4180
		page_counter_init(&memcg->tcpmem, NULL);
4181 4182 4183 4184 4185
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
4186
		if (parent != root_mem_cgroup)
4187
			memory_cgrp_subsys.broken_hierarchy = true;
4188
	}
4189

4190 4191 4192 4193 4194 4195
	/* The following stuff does not apply to the root */
	if (!parent) {
		root_mem_cgroup = memcg;
		return &memcg->css;
	}

4196
	error = memcg_online_kmem(memcg);
4197 4198
	if (error)
		goto fail;
4199

4200
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4201
		static_branch_inc(&memcg_sockets_enabled_key);
4202

4203 4204 4205
	return &memcg->css;
fail:
	mem_cgroup_free(memcg);
4206
	return ERR_PTR(-ENOMEM);
4207 4208 4209 4210 4211 4212 4213
}

static int
mem_cgroup_css_online(struct cgroup_subsys_state *css)
{
	if (css->id > MEM_CGROUP_ID_MAX)
		return -ENOSPC;
4214 4215

	return 0;
B
Balbir Singh 已提交
4216 4217
}

4218
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4219
{
4220
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4221
	struct mem_cgroup_event *event, *tmp;
4222 4223 4224 4225 4226 4227

	/*
	 * Unregister events and notify userspace.
	 * Notify userspace about cgroup removing only after rmdir of cgroup
	 * directory to avoid race between userspace and kernelspace.
	 */
4228 4229
	spin_lock(&memcg->event_list_lock);
	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4230 4231 4232
		list_del_init(&event->list);
		schedule_work(&event->remove);
	}
4233
	spin_unlock(&memcg->event_list_lock);
4234

4235
	memcg_offline_kmem(memcg);
4236
	wb_memcg_offline(memcg);
4237 4238
}

4239 4240 4241 4242 4243 4244 4245
static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	invalidate_reclaim_iterators(memcg);
}

4246
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
B
Balbir Singh 已提交
4247
{
4248
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4249

4250
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4251
		static_branch_dec(&memcg_sockets_enabled_key);
4252

4253
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
V
Vladimir Davydov 已提交
4254
		static_branch_dec(&memcg_sockets_enabled_key);
4255

4256 4257 4258
	vmpressure_cleanup(&memcg->vmpressure);
	cancel_work_sync(&memcg->high_work);
	mem_cgroup_remove_from_trees(memcg);
4259
	memcg_free_kmem(memcg);
4260
	mem_cgroup_free(memcg);
B
Balbir Singh 已提交
4261 4262
}

4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
/**
 * mem_cgroup_css_reset - reset the states of a mem_cgroup
 * @css: the target css
 *
 * Reset the states of the mem_cgroup associated with @css.  This is
 * invoked when the userland requests disabling on the default hierarchy
 * but the memcg is pinned through dependency.  The memcg should stop
 * applying policies and should revert to the vanilla state as it may be
 * made visible again.
 *
 * The current implementation only resets the essential configurations.
 * This needs to be expanded to cover all the visible parts.
 */
static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

4280 4281 4282 4283 4284
	page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
	page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
	page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
	page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
	page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4285 4286
	memcg->low = 0;
	memcg->high = PAGE_COUNTER_MAX;
4287
	memcg->soft_limit = PAGE_COUNTER_MAX;
4288
	memcg_wb_domain_size_changed(memcg);
4289 4290
}

4291
#ifdef CONFIG_MMU
4292
/* Handlers for move charge at task migration. */
4293
static int mem_cgroup_do_precharge(unsigned long count)
4294
{
4295
	int ret;
4296

4297 4298
	/* Try a single bulk charge without reclaim first, kswapd may wake */
	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4299
	if (!ret) {
4300 4301 4302
		mc.precharge += count;
		return ret;
	}
4303 4304

	/* Try charges one by one with reclaim */
4305
	while (count--) {
4306
		ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1);
4307 4308
		if (ret)
			return ret;
4309
		mc.precharge++;
4310
		cond_resched();
4311
	}
4312
	return 0;
4313 4314 4315 4316
}

union mc_target {
	struct page	*page;
4317
	swp_entry_t	ent;
4318 4319 4320
};

enum mc_target_type {
4321
	MC_TARGET_NONE = 0,
4322
	MC_TARGET_PAGE,
4323
	MC_TARGET_SWAP,
4324 4325
};

D
Daisuke Nishimura 已提交
4326 4327
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
4328
{
D
Daisuke Nishimura 已提交
4329
	struct page *page = vm_normal_page(vma, addr, ptent);
4330

D
Daisuke Nishimura 已提交
4331 4332 4333
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
4334
		if (!(mc.flags & MOVE_ANON))
D
Daisuke Nishimura 已提交
4335
			return NULL;
4336 4337 4338 4339
	} else {
		if (!(mc.flags & MOVE_FILE))
			return NULL;
	}
D
Daisuke Nishimura 已提交
4340 4341 4342 4343 4344 4345
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

4346
#ifdef CONFIG_SWAP
D
Daisuke Nishimura 已提交
4347 4348 4349 4350 4351 4352
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

4353
	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
D
Daisuke Nishimura 已提交
4354
		return NULL;
4355 4356 4357 4358
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
4359
	page = find_get_page(swap_address_space(ent), ent.val);
4360
	if (do_memsw_account())
D
Daisuke Nishimura 已提交
4361 4362 4363 4364
		entry->val = ent.val;

	return page;
}
4365 4366 4367 4368 4369 4370 4371
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
4372

4373 4374 4375 4376 4377 4378 4379 4380 4381
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
4382
	if (!(mc.flags & MOVE_FILE))
4383 4384 4385
		return NULL;

	mapping = vma->vm_file->f_mapping;
4386
	pgoff = linear_page_index(vma, addr);
4387 4388

	/* page is moved even if it's not RSS of this task(page-faulted). */
4389 4390
#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
4391 4392 4393 4394
	if (shmem_mapping(mapping)) {
		page = find_get_entry(mapping, pgoff);
		if (radix_tree_exceptional_entry(page)) {
			swp_entry_t swp = radix_to_swp_entry(page);
4395
			if (do_memsw_account())
4396 4397 4398 4399 4400 4401 4402
				*entry = swp;
			page = find_get_page(swap_address_space(swp), swp.val);
		}
	} else
		page = find_get_page(mapping, pgoff);
#else
	page = find_get_page(mapping, pgoff);
4403
#endif
4404 4405 4406
	return page;
}

4407 4408 4409 4410 4411 4412 4413
/**
 * mem_cgroup_move_account - move account of the page
 * @page: the page
 * @nr_pages: number of regular pages (>1 for huge pages)
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
4414
 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4415 4416 4417 4418 4419
 *
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
 */
static int mem_cgroup_move_account(struct page *page,
4420
				   bool compound,
4421 4422 4423 4424
				   struct mem_cgroup *from,
				   struct mem_cgroup *to)
{
	unsigned long flags;
4425
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4426
	int ret;
4427
	bool anon;
4428 4429 4430

	VM_BUG_ON(from == to);
	VM_BUG_ON_PAGE(PageLRU(page), page);
4431
	VM_BUG_ON(compound && !PageTransHuge(page));
4432 4433

	/*
4434
	 * Prevent mem_cgroup_migrate() from looking at
4435
	 * page->mem_cgroup of its source page while we change it.
4436
	 */
4437
	ret = -EBUSY;
4438 4439 4440 4441 4442 4443 4444
	if (!trylock_page(page))
		goto out;

	ret = -EINVAL;
	if (page->mem_cgroup != from)
		goto out_unlock;

4445 4446
	anon = PageAnon(page);

4447 4448
	spin_lock_irqsave(&from->move_lock, flags);

4449
	if (!anon && page_mapped(page)) {
4450 4451 4452 4453 4454 4455
		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
			       nr_pages);
		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
			       nr_pages);
	}

4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471
	/*
	 * move_lock grabbed above and caller set from->moving_account, so
	 * mem_cgroup_update_page_stat() will serialize updates to PageDirty.
	 * So mapping should be stable for dirty pages.
	 */
	if (!anon && PageDirty(page)) {
		struct address_space *mapping = page_mapping(page);

		if (mapping_cap_account_dirty(mapping)) {
			__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY],
				       nr_pages);
			__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY],
				       nr_pages);
		}
	}

4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491
	if (PageWriteback(page)) {
		__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
			       nr_pages);
		__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
			       nr_pages);
	}

	/*
	 * It is safe to change page->mem_cgroup here because the page
	 * is referenced, charged, and isolated - we can't race with
	 * uncharging, charging, migration, or LRU putback.
	 */

	/* caller should have done css_get */
	page->mem_cgroup = to;
	spin_unlock_irqrestore(&from->move_lock, flags);

	ret = 0;

	local_irq_disable();
4492
	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4493
	memcg_check_events(to, page);
4494
	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4495 4496 4497 4498 4499 4500 4501 4502
	memcg_check_events(from, page);
	local_irq_enable();
out_unlock:
	unlock_page(page);
out:
	return ret;
}

4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521
/**
 * get_mctgt_type - get target type of moving charge
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
 *
 * Called with pte lock held.
 */

4522
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
4523 4524 4525
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
4526
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
4527 4528 4529 4530 4531 4532
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4533
	else if (pte_none(ptent))
4534
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
4535 4536

	if (!page && !ent.val)
4537
		return ret;
4538 4539
	if (page) {
		/*
4540
		 * Do only loose check w/o serialization.
4541
		 * mem_cgroup_move_account() checks the page is valid or
4542
		 * not under LRU exclusion.
4543
		 */
4544
		if (page->mem_cgroup == mc.from) {
4545 4546 4547 4548 4549 4550 4551
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
4552 4553
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
L
Li Zefan 已提交
4554
	    mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4555 4556 4557
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
4558 4559 4560 4561
	}
	return ret;
}

4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
 * We don't consider swapping or file mapped pages because THP does not
 * support them for now.
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	enum mc_target_type ret = MC_TARGET_NONE;

	page = pmd_page(pmd);
4575
	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4576
	if (!(mc.flags & MOVE_ANON))
4577
		return ret;
4578
	if (page->mem_cgroup == mc.from) {
4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

4595 4596 4597 4598
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
4599
	struct vm_area_struct *vma = walk->vma;
4600 4601 4602
	pte_t *pte;
	spinlock_t *ptl;

4603 4604
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
4605 4606
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
4607
		spin_unlock(ptl);
4608
		return 0;
4609
	}
4610

4611 4612
	if (pmd_trans_unstable(pmd))
		return 0;
4613 4614
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
4615
		if (get_mctgt_type(vma, addr, *pte, NULL))
4616 4617 4618 4619
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

4620 4621 4622
	return 0;
}

4623 4624 4625 4626
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;

4627 4628 4629 4630
	struct mm_walk mem_cgroup_count_precharge_walk = {
		.pmd_entry = mem_cgroup_count_precharge_pte_range,
		.mm = mm,
	};
4631
	down_read(&mm->mmap_sem);
4632
	walk_page_range(0, ~0UL, &mem_cgroup_count_precharge_walk);
4633
	up_read(&mm->mmap_sem);
4634 4635 4636 4637 4638 4639 4640 4641 4642

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
4643 4644 4645 4646 4647
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
4648 4649
}

4650 4651
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
4652
{
4653 4654 4655
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

4656
	/* we must uncharge all the leftover precharges from mc.to */
4657
	if (mc.precharge) {
4658
		cancel_charge(mc.to, mc.precharge);
4659 4660 4661 4662 4663 4664 4665
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
4666
		cancel_charge(mc.from, mc.moved_charge);
4667
		mc.moved_charge = 0;
4668
	}
4669 4670 4671
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
4672
		if (!mem_cgroup_is_root(mc.from))
4673
			page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4674

4675
		/*
4676 4677
		 * we charged both to->memory and to->memsw, so we
		 * should uncharge to->memory.
4678
		 */
4679
		if (!mem_cgroup_is_root(mc.to))
4680 4681
			page_counter_uncharge(&mc.to->memory, mc.moved_swap);

4682
		css_put_many(&mc.from->css, mc.moved_swap);
4683

L
Li Zefan 已提交
4684
		/* we've already done css_get(mc.to) */
4685 4686
		mc.moved_swap = 0;
	}
4687 4688 4689 4690 4691 4692 4693
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
4694 4695
	struct mm_struct *mm = mc.mm;

4696 4697 4698 4699 4700 4701
	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
4702
	spin_lock(&mc.lock);
4703 4704
	mc.from = NULL;
	mc.to = NULL;
4705
	mc.mm = NULL;
4706
	spin_unlock(&mc.lock);
4707 4708

	mmput(mm);
4709 4710
}

4711
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4712
{
4713
	struct cgroup_subsys_state *css;
4714
	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4715
	struct mem_cgroup *from;
4716
	struct task_struct *leader, *p;
4717
	struct mm_struct *mm;
4718
	unsigned long move_flags;
4719
	int ret = 0;
4720

4721 4722
	/* charge immigration isn't supported on the default hierarchy */
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4723 4724
		return 0;

4725 4726 4727 4728 4729 4730 4731
	/*
	 * Multi-process migrations only happen on the default hierarchy
	 * where charge immigration is not used.  Perform charge
	 * immigration if @tset contains a leader and whine if there are
	 * multiple.
	 */
	p = NULL;
4732
	cgroup_taskset_for_each_leader(leader, css, tset) {
4733 4734
		WARN_ON_ONCE(p);
		p = leader;
4735
		memcg = mem_cgroup_from_css(css);
4736 4737 4738 4739
	}
	if (!p)
		return 0;

4740 4741 4742 4743 4744 4745 4746 4747 4748
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
	if (!move_flags)
		return 0;

4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764
	from = mem_cgroup_from_task(p);

	VM_BUG_ON(from == memcg);

	mm = get_task_mm(p);
	if (!mm)
		return 0;
	/* We move charges only when we move a owner of the mm */
	if (mm->owner == p) {
		VM_BUG_ON(mc.from);
		VM_BUG_ON(mc.to);
		VM_BUG_ON(mc.precharge);
		VM_BUG_ON(mc.moved_charge);
		VM_BUG_ON(mc.moved_swap);

		spin_lock(&mc.lock);
4765
		mc.mm = mm;
4766 4767 4768 4769 4770 4771 4772 4773 4774
		mc.from = from;
		mc.to = memcg;
		mc.flags = move_flags;
		spin_unlock(&mc.lock);
		/* We set mc.moving_task later */

		ret = mem_cgroup_precharge_mc(mm);
		if (ret)
			mem_cgroup_clear_mc();
4775 4776
	} else {
		mmput(mm);
4777 4778 4779 4780
	}
	return ret;
}

4781
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4782
{
4783 4784
	if (mc.to)
		mem_cgroup_clear_mc();
4785 4786
}

4787 4788 4789
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
4790
{
4791
	int ret = 0;
4792
	struct vm_area_struct *vma = walk->vma;
4793 4794
	pte_t *pte;
	spinlock_t *ptl;
4795 4796 4797
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
4798

4799 4800
	ptl = pmd_trans_huge_lock(pmd, vma);
	if (ptl) {
4801
		if (mc.precharge < HPAGE_PMD_NR) {
4802
			spin_unlock(ptl);
4803 4804 4805 4806 4807 4808
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
4809
				if (!mem_cgroup_move_account(page, true,
4810
							     mc.from, mc.to)) {
4811 4812 4813 4814 4815 4816 4817
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
		}
4818
		spin_unlock(ptl);
4819
		return 0;
4820 4821
	}

4822 4823
	if (pmd_trans_unstable(pmd))
		return 0;
4824 4825 4826 4827
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
4828
		swp_entry_t ent;
4829 4830 4831 4832

		if (!mc.precharge)
			break;

4833
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
4834 4835
		case MC_TARGET_PAGE:
			page = target.page;
4836 4837 4838 4839 4840 4841 4842 4843
			/*
			 * We can have a part of the split pmd here. Moving it
			 * can be done but it would be too convoluted so simply
			 * ignore such a partial THP and keep it in original
			 * memcg. There should be somebody mapping the head.
			 */
			if (PageTransCompound(page))
				goto put;
4844 4845
			if (isolate_lru_page(page))
				goto put;
4846 4847
			if (!mem_cgroup_move_account(page, false,
						mc.from, mc.to)) {
4848
				mc.precharge--;
4849 4850
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
4851 4852
			}
			putback_lru_page(page);
4853
put:			/* get_mctgt_type() gets the page */
4854 4855
			put_page(page);
			break;
4856 4857
		case MC_TARGET_SWAP:
			ent = target.ent;
4858
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4859
				mc.precharge--;
4860 4861 4862
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
4863
			break;
4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
4878
		ret = mem_cgroup_do_precharge(1);
4879 4880 4881 4882 4883 4884 4885
		if (!ret)
			goto retry;
	}

	return ret;
}

4886
static void mem_cgroup_move_charge(void)
4887
{
4888 4889
	struct mm_walk mem_cgroup_move_charge_walk = {
		.pmd_entry = mem_cgroup_move_charge_pte_range,
4890
		.mm = mc.mm,
4891
	};
4892 4893

	lru_add_drain_all();
4894
	/*
4895 4896 4897
	 * Signal lock_page_memcg() to take the memcg's move_lock
	 * while we're moving its pages to another memcg. Then wait
	 * for already started RCU-only updates to finish.
4898 4899 4900
	 */
	atomic_inc(&mc.from->moving_account);
	synchronize_rcu();
4901
retry:
4902
	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
4914 4915 4916 4917 4918
	/*
	 * When we have consumed all precharges and failed in doing
	 * additional charge, the page walk just aborts.
	 */
	walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk);
4919
	up_read(&mc.mm->mmap_sem);
4920
	atomic_dec(&mc.from->moving_account);
4921 4922
}

4923
static void mem_cgroup_move_task(void)
B
Balbir Singh 已提交
4924
{
4925 4926
	if (mc.to) {
		mem_cgroup_move_charge();
4927
		mem_cgroup_clear_mc();
4928
	}
B
Balbir Singh 已提交
4929
}
4930
#else	/* !CONFIG_MMU */
4931
static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4932 4933 4934
{
	return 0;
}
4935
static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4936 4937
{
}
4938
static void mem_cgroup_move_task(void)
4939 4940 4941
{
}
#endif
B
Balbir Singh 已提交
4942

4943 4944
/*
 * Cgroup retains root cgroups across [un]mount cycles making it necessary
4945 4946
 * to verify whether we're attached to the default hierarchy on each mount
 * attempt.
4947
 */
4948
static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
4949 4950
{
	/*
4951
	 * use_hierarchy is forced on the default hierarchy.  cgroup core
4952 4953 4954
	 * guarantees that @root doesn't have any children, so turning it
	 * on for the root memcg is enough.
	 */
4955
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4956 4957 4958
		root_mem_cgroup->use_hierarchy = true;
	else
		root_mem_cgroup->use_hierarchy = false;
4959 4960
}

4961 4962 4963
static u64 memory_current_read(struct cgroup_subsys_state *css,
			       struct cftype *cft)
{
4964 4965 4966
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
4967 4968 4969 4970 4971
}

static int memory_low_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
4972
	unsigned long low = READ_ONCE(memcg->low);
4973 4974

	if (low == PAGE_COUNTER_MAX)
4975
		seq_puts(m, "max\n");
4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989
	else
		seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);

	return 0;
}

static ssize_t memory_low_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long low;
	int err;

	buf = strstrip(buf);
4990
	err = page_counter_memparse(buf, "max", &low);
4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001
	if (err)
		return err;

	memcg->low = low;

	return nbytes;
}

static int memory_high_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5002
	unsigned long high = READ_ONCE(memcg->high);
5003 5004

	if (high == PAGE_COUNTER_MAX)
5005
		seq_puts(m, "max\n");
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015
	else
		seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);

	return 0;
}

static ssize_t memory_high_write(struct kernfs_open_file *of,
				 char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5016
	unsigned long nr_pages;
5017 5018 5019 5020
	unsigned long high;
	int err;

	buf = strstrip(buf);
5021
	err = page_counter_memparse(buf, "max", &high);
5022 5023 5024 5025 5026
	if (err)
		return err;

	memcg->high = high;

5027 5028 5029 5030 5031
	nr_pages = page_counter_read(&memcg->memory);
	if (nr_pages > high)
		try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
					     GFP_KERNEL, true);

5032
	memcg_wb_domain_size_changed(memcg);
5033 5034 5035 5036 5037 5038
	return nbytes;
}

static int memory_max_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5039
	unsigned long max = READ_ONCE(memcg->memory.limit);
5040 5041

	if (max == PAGE_COUNTER_MAX)
5042
		seq_puts(m, "max\n");
5043 5044 5045 5046 5047 5048 5049 5050 5051 5052
	else
		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);

	return 0;
}

static ssize_t memory_max_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5053 5054
	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
	bool drained = false;
5055 5056 5057 5058
	unsigned long max;
	int err;

	buf = strstrip(buf);
5059
	err = page_counter_memparse(buf, "max", &max);
5060 5061 5062
	if (err)
		return err;

5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092
	xchg(&memcg->memory.limit, max);

	for (;;) {
		unsigned long nr_pages = page_counter_read(&memcg->memory);

		if (nr_pages <= max)
			break;

		if (signal_pending(current)) {
			err = -EINTR;
			break;
		}

		if (!drained) {
			drain_all_stock(memcg);
			drained = true;
			continue;
		}

		if (nr_reclaims) {
			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
							  GFP_KERNEL, true))
				nr_reclaims--;
			continue;
		}

		mem_cgroup_events(memcg, MEMCG_OOM, 1);
		if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
			break;
	}
5093

5094
	memcg_wb_domain_size_changed(memcg);
5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109
	return nbytes;
}

static int memory_events_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));

	seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
	seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
	seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
	seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));

	return 0;
}

5110 5111 5112
static int memory_stat_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5113 5114
	unsigned long stat[MEMCG_NR_STAT];
	unsigned long events[MEMCG_NR_EVENTS];
5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127
	int i;

	/*
	 * Provide statistics on the state of the memory subsystem as
	 * well as cumulative event counters that show past behavior.
	 *
	 * This list is ordered following a combination of these gradients:
	 * 1) generic big picture -> specifics and details
	 * 2) reflecting userspace activity -> reflecting kernel heuristics
	 *
	 * Current memory state:
	 */

5128 5129 5130
	tree_stat(memcg, stat);
	tree_events(memcg, events);

5131
	seq_printf(m, "anon %llu\n",
5132
		   (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
5133
	seq_printf(m, "file %llu\n",
5134
		   (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
5135 5136
	seq_printf(m, "kernel_stack %llu\n",
		   (u64)stat[MEMCG_KERNEL_STACK] * PAGE_SIZE);
5137 5138 5139
	seq_printf(m, "slab %llu\n",
		   (u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
			 stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5140
	seq_printf(m, "sock %llu\n",
5141
		   (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5142 5143

	seq_printf(m, "file_mapped %llu\n",
5144
		   (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
5145
	seq_printf(m, "file_dirty %llu\n",
5146
		   (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
5147
	seq_printf(m, "file_writeback %llu\n",
5148
		   (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159

	for (i = 0; i < NR_LRU_LISTS; i++) {
		struct mem_cgroup *mi;
		unsigned long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i));
		seq_printf(m, "%s %llu\n",
			   mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
	}

5160 5161 5162 5163 5164
	seq_printf(m, "slab_reclaimable %llu\n",
		   (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
	seq_printf(m, "slab_unreclaimable %llu\n",
		   (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);

5165 5166 5167
	/* Accumulated memory events */

	seq_printf(m, "pgfault %lu\n",
5168
		   events[MEM_CGROUP_EVENTS_PGFAULT]);
5169
	seq_printf(m, "pgmajfault %lu\n",
5170
		   events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
5171 5172 5173 5174

	return 0;
}

5175 5176 5177
static struct cftype memory_files[] = {
	{
		.name = "current",
5178
		.flags = CFTYPE_NOT_ON_ROOT,
5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201
		.read_u64 = memory_current_read,
	},
	{
		.name = "low",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_low_show,
		.write = memory_low_write,
	},
	{
		.name = "high",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_high_show,
		.write = memory_high_write,
	},
	{
		.name = "max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_max_show,
		.write = memory_max_write,
	},
	{
		.name = "events",
		.flags = CFTYPE_NOT_ON_ROOT,
5202
		.file_offset = offsetof(struct mem_cgroup, events_file),
5203 5204
		.seq_show = memory_events_show,
	},
5205 5206 5207 5208 5209
	{
		.name = "stat",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = memory_stat_show,
	},
5210 5211 5212
	{ }	/* terminate */
};

5213
struct cgroup_subsys memory_cgrp_subsys = {
5214
	.css_alloc = mem_cgroup_css_alloc,
5215
	.css_online = mem_cgroup_css_online,
5216
	.css_offline = mem_cgroup_css_offline,
5217
	.css_released = mem_cgroup_css_released,
5218
	.css_free = mem_cgroup_css_free,
5219
	.css_reset = mem_cgroup_css_reset,
5220 5221
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
5222
	.post_attach = mem_cgroup_move_task,
5223
	.bind = mem_cgroup_bind,
5224 5225
	.dfl_cftypes = memory_files,
	.legacy_cftypes = mem_cgroup_legacy_files,
5226
	.early_init = 0,
B
Balbir Singh 已提交
5227
};
5228

5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250
/**
 * mem_cgroup_low - check if memory consumption is below the normal range
 * @root: the highest ancestor to consider
 * @memcg: the memory cgroup to check
 *
 * Returns %true if memory consumption of @memcg, and that of all
 * configurable ancestors up to @root, is below the normal range.
 */
bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return false;

	/*
	 * The toplevel group doesn't have a configurable range, so
	 * it's never low when looked at directly, and it is not
	 * considered an ancestor when assessing the hierarchy.
	 */

	if (memcg == root_mem_cgroup)
		return false;

M
Michal Hocko 已提交
5251
	if (page_counter_read(&memcg->memory) >= memcg->low)
5252 5253 5254 5255 5256 5257 5258 5259
		return false;

	while (memcg != root) {
		memcg = parent_mem_cgroup(memcg);

		if (memcg == root_mem_cgroup)
			break;

M
Michal Hocko 已提交
5260
		if (page_counter_read(&memcg->memory) >= memcg->low)
5261 5262 5263 5264 5265
			return false;
	}
	return true;
}

5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283
/**
 * mem_cgroup_try_charge - try charging a page
 * @page: page to charge
 * @mm: mm context of the victim
 * @gfp_mask: reclaim mode
 * @memcgp: charged memcg return
 *
 * Try to charge @page to the memcg that @mm belongs to, reclaiming
 * pages according to @gfp_mask if necessary.
 *
 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
 * Otherwise, an error code is returned.
 *
 * After page->mapping has been set up, the caller must finalize the
 * charge with mem_cgroup_commit_charge().  Or abort the transaction
 * with mem_cgroup_cancel_charge() in case page instantiation fails.
 */
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5284 5285
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound)
5286 5287
{
	struct mem_cgroup *memcg = NULL;
5288
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301
	int ret = 0;

	if (mem_cgroup_disabled())
		goto out;

	if (PageSwapCache(page)) {
		/*
		 * Every swap fault against a single page tries to charge the
		 * page, bail as early as possible.  shmem_unuse() encounters
		 * already charged pages, too.  The USED bit is protected by
		 * the page lock, which serializes swap cache removal, which
		 * in turn serializes uncharging.
		 */
5302
		VM_BUG_ON_PAGE(!PageLocked(page), page);
5303
		if (page->mem_cgroup)
5304
			goto out;
5305

5306
		if (do_swap_account) {
5307 5308 5309 5310 5311 5312 5313 5314 5315
			swp_entry_t ent = { .val = page_private(page), };
			unsigned short id = lookup_swap_cgroup_id(ent);

			rcu_read_lock();
			memcg = mem_cgroup_from_id(id);
			if (memcg && !css_tryget_online(&memcg->css))
				memcg = NULL;
			rcu_read_unlock();
		}
5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345
	}

	if (!memcg)
		memcg = get_mem_cgroup_from_mm(mm);

	ret = try_charge(memcg, gfp_mask, nr_pages);

	css_put(&memcg->css);
out:
	*memcgp = memcg;
	return ret;
}

/**
 * mem_cgroup_commit_charge - commit a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 * @lrucare: page might be on LRU already
 *
 * Finalize a charge transaction started by mem_cgroup_try_charge(),
 * after page->mapping has been set up.  This must happen atomically
 * as part of the page instantiation, i.e. under the page table lock
 * for anonymous pages, under the page lock for page and swap cache.
 *
 * In addition, the page must not be on the LRU during the commit, to
 * prevent racing with task migration.  If it might be, use @lrucare.
 *
 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
 */
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5346
			      bool lrucare, bool compound)
5347
{
5348
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362

	VM_BUG_ON_PAGE(!page->mapping, page);
	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

5363 5364 5365
	commit_charge(page, memcg, lrucare);

	local_irq_disable();
5366
	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5367 5368
	memcg_check_events(memcg, page);
	local_irq_enable();
5369

5370
	if (do_memsw_account() && PageSwapCache(page)) {
5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387
		swp_entry_t entry = { .val = page_private(page) };
		/*
		 * The swap entry might not get freed for a long time,
		 * let's not wait for it.  The page already received a
		 * memory+swap charge, drop the swap entry duplicate.
		 */
		mem_cgroup_uncharge_swap(entry);
	}
}

/**
 * mem_cgroup_cancel_charge - cancel a page charge
 * @page: page to charge
 * @memcg: memcg to charge the page to
 *
 * Cancel a charge transaction started by mem_cgroup_try_charge().
 */
5388 5389
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound)
5390
{
5391
	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405

	if (mem_cgroup_disabled())
		return;
	/*
	 * Swap faults will attempt to charge the same page multiple
	 * times.  But reuse_swap_page() might have removed the page
	 * from swapcache already, so we can't check PageSwapCache().
	 */
	if (!memcg)
		return;

	cancel_charge(memcg, nr_pages);
}

5406 5407 5408 5409
static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
			   unsigned long nr_anon, unsigned long nr_file,
			   unsigned long nr_huge, struct page *dummy_page)
{
5410
	unsigned long nr_pages = nr_anon + nr_file;
5411 5412
	unsigned long flags;

5413
	if (!mem_cgroup_is_root(memcg)) {
5414
		page_counter_uncharge(&memcg->memory, nr_pages);
5415
		if (do_memsw_account())
5416
			page_counter_uncharge(&memcg->memsw, nr_pages);
5417 5418
		memcg_oom_recover(memcg);
	}
5419 5420 5421 5422 5423 5424

	local_irq_save(flags);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
	__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5425
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5426 5427
	memcg_check_events(memcg, dummy_page);
	local_irq_restore(flags);
5428 5429

	if (!mem_cgroup_is_root(memcg))
5430
		css_put_many(&memcg->css, nr_pages);
5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442
}

static void uncharge_list(struct list_head *page_list)
{
	struct mem_cgroup *memcg = NULL;
	unsigned long nr_anon = 0;
	unsigned long nr_file = 0;
	unsigned long nr_huge = 0;
	unsigned long pgpgout = 0;
	struct list_head *next;
	struct page *page;

5443 5444 5445 5446
	/*
	 * Note that the list can be a single page->lru; hence the
	 * do-while loop instead of a simple list_for_each_entry().
	 */
5447 5448 5449 5450 5451 5452 5453 5454 5455 5456
	next = page_list->next;
	do {
		unsigned int nr_pages = 1;

		page = list_entry(next, struct page, lru);
		next = page->lru.next;

		VM_BUG_ON_PAGE(PageLRU(page), page);
		VM_BUG_ON_PAGE(page_count(page), page);

5457
		if (!page->mem_cgroup)
5458 5459 5460 5461
			continue;

		/*
		 * Nobody should be changing or seriously looking at
5462
		 * page->mem_cgroup at this point, we have fully
5463
		 * exclusive access to the page.
5464 5465
		 */

5466
		if (memcg != page->mem_cgroup) {
5467
			if (memcg) {
5468 5469 5470
				uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
					       nr_huge, page);
				pgpgout = nr_anon = nr_file = nr_huge = 0;
5471
			}
5472
			memcg = page->mem_cgroup;
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485
		}

		if (PageTransHuge(page)) {
			nr_pages <<= compound_order(page);
			VM_BUG_ON_PAGE(!PageTransHuge(page), page);
			nr_huge += nr_pages;
		}

		if (PageAnon(page))
			nr_anon += nr_pages;
		else
			nr_file += nr_pages;

5486
		page->mem_cgroup = NULL;
5487 5488 5489 5490 5491

		pgpgout++;
	} while (next != page_list);

	if (memcg)
5492 5493
		uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
			       nr_huge, page);
5494 5495
}

5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507
/**
 * mem_cgroup_uncharge - uncharge a page
 * @page: page to uncharge
 *
 * Uncharge a page previously charged with mem_cgroup_try_charge() and
 * mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge(struct page *page)
{
	if (mem_cgroup_disabled())
		return;

5508
	/* Don't touch page->lru of any random page, pre-check: */
5509
	if (!page->mem_cgroup)
5510 5511
		return;

5512 5513 5514
	INIT_LIST_HEAD(&page->lru);
	uncharge_list(&page->lru);
}
5515

5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526
/**
 * mem_cgroup_uncharge_list - uncharge a list of page
 * @page_list: list of pages to uncharge
 *
 * Uncharge a list of pages previously charged with
 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
 */
void mem_cgroup_uncharge_list(struct list_head *page_list)
{
	if (mem_cgroup_disabled())
		return;
5527

5528 5529
	if (!list_empty(page_list))
		uncharge_list(page_list);
5530 5531 5532
}

/**
5533 5534 5535
 * mem_cgroup_migrate - charge a page's replacement
 * @oldpage: currently circulating page
 * @newpage: replacement page
5536
 *
5537 5538
 * Charge @newpage as a replacement page for @oldpage. @oldpage will
 * be uncharged upon free.
5539 5540 5541
 *
 * Both pages must be locked, @newpage->mapping must be set up.
 */
5542
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5543
{
5544
	struct mem_cgroup *memcg;
5545 5546
	unsigned int nr_pages;
	bool compound;
5547
	unsigned long flags;
5548 5549 5550 5551

	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5552 5553
	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
		       newpage);
5554 5555 5556 5557 5558

	if (mem_cgroup_disabled())
		return;

	/* Page cache replacement: new page already charged? */
5559
	if (newpage->mem_cgroup)
5560 5561
		return;

5562
	/* Swapcache readahead pages can get replaced before being charged */
5563
	memcg = oldpage->mem_cgroup;
5564
	if (!memcg)
5565 5566
		return;

5567 5568 5569 5570 5571 5572 5573 5574
	/* Force-charge the new page. The old one will be freed soon */
	compound = PageTransHuge(newpage);
	nr_pages = compound ? hpage_nr_pages(newpage) : 1;

	page_counter_charge(&memcg->memory, nr_pages);
	if (do_memsw_account())
		page_counter_charge(&memcg->memsw, nr_pages);
	css_get_many(&memcg->css, nr_pages);
5575

5576
	commit_charge(newpage, memcg, false);
5577

5578
	local_irq_save(flags);
5579 5580
	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
	memcg_check_events(memcg, newpage);
5581
	local_irq_restore(flags);
5582 5583
}

5584
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606
EXPORT_SYMBOL(memcg_sockets_enabled_key);

void sock_update_memcg(struct sock *sk)
{
	struct mem_cgroup *memcg;

	/* Socket cloning can throw us here with sk_cgrp already
	 * filled. It won't however, necessarily happen from
	 * process context. So the test for root memcg given
	 * the current task's memcg won't help us in this case.
	 *
	 * Respecting the original socket's memcg is a better
	 * decision in this case.
	 */
	if (sk->sk_memcg) {
		BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
		css_get(&sk->sk_memcg->css);
		return;
	}

	rcu_read_lock();
	memcg = mem_cgroup_from_task(current);
5607 5608
	if (memcg == root_mem_cgroup)
		goto out;
5609
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5610 5611
		goto out;
	if (css_tryget_online(&memcg->css))
5612
		sk->sk_memcg = memcg;
5613
out:
5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633
	rcu_read_unlock();
}
EXPORT_SYMBOL(sock_update_memcg);

void sock_release_memcg(struct sock *sk)
{
	WARN_ON(!sk->sk_memcg);
	css_put(&sk->sk_memcg->css);
}

/**
 * mem_cgroup_charge_skmem - charge socket memory
 * @memcg: memcg to charge
 * @nr_pages: number of pages to charge
 *
 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
 * @memcg's configured limit, %false if the charge had to be forced.
 */
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
5634
	gfp_t gfp_mask = GFP_KERNEL;
5635

5636
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5637
		struct page_counter *fail;
5638

5639 5640
		if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
			memcg->tcpmem_pressure = 0;
5641 5642
			return true;
		}
5643 5644
		page_counter_charge(&memcg->tcpmem, nr_pages);
		memcg->tcpmem_pressure = 1;
5645
		return false;
5646
	}
5647

5648 5649 5650 5651
	/* Don't block in the packet receive path */
	if (in_softirq())
		gfp_mask = GFP_NOWAIT;

5652 5653
	this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);

5654 5655 5656 5657
	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
		return true;

	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5658 5659 5660 5661 5662 5663 5664 5665 5666 5667
	return false;
}

/**
 * mem_cgroup_uncharge_skmem - uncharge socket memory
 * @memcg - memcg to uncharge
 * @nr_pages - number of pages to uncharge
 */
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
{
5668
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5669
		page_counter_uncharge(&memcg->tcpmem, nr_pages);
5670 5671
		return;
	}
5672

5673 5674
	this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);

5675 5676
	page_counter_uncharge(&memcg->memory, nr_pages);
	css_put_many(&memcg->css, nr_pages);
5677 5678
}

5679 5680 5681 5682 5683 5684 5685 5686 5687
static int __init cgroup_memory(char *s)
{
	char *token;

	while ((token = strsep(&s, ",")) != NULL) {
		if (!*token)
			continue;
		if (!strcmp(token, "nosocket"))
			cgroup_memory_nosocket = true;
5688 5689
		if (!strcmp(token, "nokmem"))
			cgroup_memory_nokmem = true;
5690 5691 5692 5693
	}
	return 0;
}
__setup("cgroup.memory=", cgroup_memory);
5694

5695
/*
5696 5697 5698 5699 5700 5701
 * subsys_initcall() for memory controller.
 *
 * Some parts like hotcpu_notifier() have to be initialized from this context
 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
 * everything that doesn't depend on a specific mem_cgroup structure should
 * be initialized from here.
5702 5703 5704
 */
static int __init mem_cgroup_init(void)
{
5705 5706
	int cpu, node;

5707
	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729

	for_each_possible_cpu(cpu)
		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
			  drain_local_stock);

	for_each_node(node) {
		struct mem_cgroup_tree_per_node *rtpn;
		int zone;

		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
				    node_online(node) ? node : NUMA_NO_NODE);

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			struct mem_cgroup_tree_per_zone *rtpz;

			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
		soft_limit_tree.rb_tree_per_node[node] = rtpn;
	}

5730 5731 5732
	return 0;
}
subsys_initcall(mem_cgroup_init);
5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749

#ifdef CONFIG_MEMCG_SWAP
/**
 * mem_cgroup_swapout - transfer a memsw charge to swap
 * @page: page whose memsw charge to transfer
 * @entry: swap entry to move the charge to
 *
 * Transfer the memsw charge of @page to @entry.
 */
void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
{
	struct mem_cgroup *memcg;
	unsigned short oldid;

	VM_BUG_ON_PAGE(PageLRU(page), page);
	VM_BUG_ON_PAGE(page_count(page), page);

5750
	if (!do_memsw_account())
5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767
		return;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return;

	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
	VM_BUG_ON_PAGE(oldid, page);
	mem_cgroup_swap_statistics(memcg, true);

	page->mem_cgroup = NULL;

	if (!mem_cgroup_is_root(memcg))
		page_counter_uncharge(&memcg->memory, 1);

5768 5769 5770 5771 5772 5773 5774
	/*
	 * Interrupts should be disabled here because the caller holds the
	 * mapping->tree_lock lock which is taken with interrupts-off. It is
	 * important here to have the interrupts disabled because it is the
	 * only synchronisation we have for udpating the per-CPU variables.
	 */
	VM_BUG_ON(!irqs_disabled());
5775
	mem_cgroup_charge_statistics(memcg, page, false, -1);
5776 5777 5778
	memcg_check_events(memcg, page);
}

5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814
/*
 * mem_cgroup_try_charge_swap - try charging a swap entry
 * @page: page being added to swap
 * @entry: swap entry to charge
 *
 * Try to charge @entry to the memcg that @page belongs to.
 *
 * Returns 0 on success, -ENOMEM on failure.
 */
int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
{
	struct mem_cgroup *memcg;
	struct page_counter *counter;
	unsigned short oldid;

	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
		return 0;

	memcg = page->mem_cgroup;

	/* Readahead page, never charged */
	if (!memcg)
		return 0;

	if (!mem_cgroup_is_root(memcg) &&
	    !page_counter_try_charge(&memcg->swap, 1, &counter))
		return -ENOMEM;

	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
	VM_BUG_ON_PAGE(oldid, page);
	mem_cgroup_swap_statistics(memcg, true);

	css_get(&memcg->css);
	return 0;
}

5815 5816 5817 5818
/**
 * mem_cgroup_uncharge_swap - uncharge a swap entry
 * @entry: swap entry to uncharge
 *
5819
 * Drop the swap charge associated with @entry.
5820 5821 5822 5823 5824 5825
 */
void mem_cgroup_uncharge_swap(swp_entry_t entry)
{
	struct mem_cgroup *memcg;
	unsigned short id;

5826
	if (!do_swap_account)
5827 5828 5829 5830
		return;

	id = swap_cgroup_record(entry, 0);
	rcu_read_lock();
5831
	memcg = mem_cgroup_from_id(id);
5832
	if (memcg) {
5833 5834 5835 5836 5837 5838
		if (!mem_cgroup_is_root(memcg)) {
			if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
				page_counter_uncharge(&memcg->swap, 1);
			else
				page_counter_uncharge(&memcg->memsw, 1);
		}
5839 5840 5841 5842 5843 5844
		mem_cgroup_swap_statistics(memcg, false);
		css_put(&memcg->css);
	}
	rcu_read_unlock();
}

5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857
long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
	long nr_swap_pages = get_nr_swap_pages();

	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return nr_swap_pages;
	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
		nr_swap_pages = min_t(long, nr_swap_pages,
				      READ_ONCE(memcg->swap.limit) -
				      page_counter_read(&memcg->swap));
	return nr_swap_pages;
}

5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879
bool mem_cgroup_swap_full(struct page *page)
{
	struct mem_cgroup *memcg;

	VM_BUG_ON_PAGE(!PageLocked(page), page);

	if (vm_swap_full())
		return true;
	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
		return false;

	memcg = page->mem_cgroup;
	if (!memcg)
		return false;

	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
		if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
			return true;

	return false;
}

5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896
/* for remember boot option*/
#ifdef CONFIG_MEMCG_SWAP_ENABLED
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata;
#endif

static int __init enable_swap_account(char *s)
{
	if (!strcmp(s, "1"))
		really_do_swap_account = 1;
	else if (!strcmp(s, "0"))
		really_do_swap_account = 0;
	return 1;
}
__setup("swapaccount=", enable_swap_account);

5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953
static u64 swap_current_read(struct cgroup_subsys_state *css,
			     struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);

	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
}

static int swap_max_show(struct seq_file *m, void *v)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
	unsigned long max = READ_ONCE(memcg->swap.limit);

	if (max == PAGE_COUNTER_MAX)
		seq_puts(m, "max\n");
	else
		seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);

	return 0;
}

static ssize_t swap_max_write(struct kernfs_open_file *of,
			      char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned long max;
	int err;

	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &max);
	if (err)
		return err;

	mutex_lock(&memcg_limit_mutex);
	err = page_counter_limit(&memcg->swap, max);
	mutex_unlock(&memcg_limit_mutex);
	if (err)
		return err;

	return nbytes;
}

static struct cftype swap_files[] = {
	{
		.name = "swap.current",
		.flags = CFTYPE_NOT_ON_ROOT,
		.read_u64 = swap_current_read,
	},
	{
		.name = "swap.max",
		.flags = CFTYPE_NOT_ON_ROOT,
		.seq_show = swap_max_show,
		.write = swap_max_write,
	},
	{ }	/* terminate */
};

5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write = mem_cgroup_write,
		.read_u64 = mem_cgroup_read_u64,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.write = mem_cgroup_reset,
		.read_u64 = mem_cgroup_read_u64,
	},
	{ },	/* terminate */
};

static int __init mem_cgroup_swap_init(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
5985 5986
		WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
					       swap_files));
5987 5988 5989 5990 5991 5992 5993 5994
		WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
						  memsw_cgroup_files));
	}
	return 0;
}
subsys_initcall(mem_cgroup_swap_init);

#endif /* CONFIG_MEMCG_SWAP */