memcontrol.c 180.4 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
13 14 15 16
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
B
Balbir Singh 已提交
17 18 19 20 21 22 23 24 25 26 27 28 29 30
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
31
#include <linux/mm.h>
32
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
33
#include <linux/pagemap.h>
34
#include <linux/smp.h>
35
#include <linux/page-flags.h>
36
#include <linux/backing-dev.h>
37 38
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
39
#include <linux/limits.h>
40
#include <linux/export.h>
41
#include <linux/mutex.h>
42
#include <linux/rbtree.h>
43
#include <linux/slab.h>
44
#include <linux/swap.h>
45
#include <linux/swapops.h>
46
#include <linux/spinlock.h>
47 48
#include <linux/eventfd.h>
#include <linux/sort.h>
49
#include <linux/fs.h>
50
#include <linux/seq_file.h>
51
#include <linux/vmalloc.h>
52
#include <linux/mm_inline.h>
53
#include <linux/page_cgroup.h>
54
#include <linux/cpu.h>
55
#include <linux/oom.h>
K
KAMEZAWA Hiroyuki 已提交
56
#include "internal.h"
G
Glauber Costa 已提交
57
#include <net/sock.h>
M
Michal Hocko 已提交
58
#include <net/ip.h>
G
Glauber Costa 已提交
59
#include <net/tcp_memcontrol.h>
B
Balbir Singh 已提交
60

61 62
#include <asm/uaccess.h>

63 64
#include <trace/events/vmscan.h>

65
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
66 67
EXPORT_SYMBOL(mem_cgroup_subsys);

68
#define MEM_CGROUP_RECLAIM_RETRIES	5
69
static struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
70

A
Andrew Morton 已提交
71
#ifdef CONFIG_MEMCG_SWAP
L
Li Zefan 已提交
72
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
73
int do_swap_account __read_mostly;
74 75

/* for remember boot option*/
A
Andrew Morton 已提交
76
#ifdef CONFIG_MEMCG_SWAP_ENABLED
77 78 79 80 81
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata = 0;
#endif

82
#else
83
#define do_swap_account		0
84 85 86
#endif


87 88 89 90 91 92 93 94
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE, 	   /* # of pages charged as cache */
95
	MEM_CGROUP_STAT_RSS,	   /* # of pages charged as anon rss */
96
	MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
97
	MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
98 99 100
	MEM_CGROUP_STAT_NSTATS,
};

101 102 103 104 105 106 107
static const char * const mem_cgroup_stat_names[] = {
	"cache",
	"rss",
	"mapped_file",
	"swap",
};

108 109 110
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
111 112
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
113 114
	MEM_CGROUP_EVENTS_NSTATS,
};
115 116 117 118 119 120 121 122

static const char * const mem_cgroup_events_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

123 124 125 126 127 128 129 130
static const char * const mem_cgroup_lru_names[] = {
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

131 132 133 134 135 136 137 138 139
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
140
	MEM_CGROUP_TARGET_NUMAINFO,
141 142
	MEM_CGROUP_NTARGETS,
};
143 144 145
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
146

147
struct mem_cgroup_stat_cpu {
148
	long count[MEM_CGROUP_STAT_NSTATS];
149
	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
150
	unsigned long nr_page_events;
151
	unsigned long targets[MEM_CGROUP_NTARGETS];
152 153
};

154 155 156 157 158 159 160
struct mem_cgroup_reclaim_iter {
	/* css_id of the last scanned hierarchy member */
	int position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

161 162 163 164
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
165
	struct lruvec		lruvec;
166
	unsigned long		lru_size[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
167

168 169
	struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];

170 171 172 173
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
174
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
175
						/* use container_of	   */
176 177 178 179 180 181 182
};

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_lru_info {
183
	struct mem_cgroup_per_node *nodeinfo[0];
184 185
};

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

206 207 208 209 210
struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	u64 threshold;
};

K
KAMEZAWA Hiroyuki 已提交
211
/* For threshold */
212
struct mem_cgroup_threshold_ary {
213
	/* An array index points to threshold just below or equal to usage. */
214
	int current_threshold;
215 216 217 218 219
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};
220 221 222 223 224 225 226 227 228 229 230 231

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

K
KAMEZAWA Hiroyuki 已提交
232 233 234 235 236
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
237

238 239
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
240

B
Balbir Singh 已提交
241 242 243 244 245 246 247
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
248 249 250
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
251 252 253 254 255 256 257
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275

	union {
		/*
		 * the counter to account for mem+swap usage.
		 */
		struct res_counter memsw;

		/*
		 * rcu_freeing is used only when freeing struct mem_cgroup,
		 * so put it into a union to avoid wasting more memory.
		 * It must be disjoint from the css field.  It could be
		 * in a union with the res field, but res plays a much
		 * larger part in mem_cgroup life than memsw, and might
		 * be of interest, even at time of free, when debugging.
		 * So share rcu_head with the less interesting memsw.
		 */
		struct rcu_head rcu_freeing;
		/*
276 277
		 * We also need some space for a worker in deferred freeing.
		 * By the time we call it, rcu_freeing is no longer in use.
278 279 280 281
		 */
		struct work_struct work_freeing;
	};

282 283 284 285
	/*
	 * the counter to account for kernel memory usage.
	 */
	struct res_counter kmem;
286 287 288 289
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
290
	unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
291 292 293 294

	bool		oom_lock;
	atomic_t	under_oom;

295
	atomic_t	refcnt;
296

297
	int	swappiness;
298 299
	/* OOM-Killer disable */
	int		oom_kill_disable;
K
KOSAKI Motohiro 已提交
300

301 302 303
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

304 305 306 307
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
308
	struct mem_cgroup_thresholds thresholds;
309

310
	/* thresholds for mem+swap usage. RCU-protected */
311
	struct mem_cgroup_thresholds memsw_thresholds;
312

K
KAMEZAWA Hiroyuki 已提交
313 314
	/* For oom notifier event fd */
	struct list_head oom_notify;
315

316 317 318 319 320
	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long 	move_charge_at_immigrate;
321 322 323 324
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t	moving_account;
325 326
	/* taken only while moving_account > 0 */
	spinlock_t	move_lock;
327
	/*
328
	 * percpu counter.
329
	 */
330
	struct mem_cgroup_stat_cpu __percpu *stat;
331 332 333 334 335 336
	/*
	 * used when a cpu is offlined or other synchronizations
	 * See mem_cgroup_read_stat().
	 */
	struct mem_cgroup_stat_cpu nocpu_base;
	spinlock_t pcp_counter_lock;
G
Glauber Costa 已提交
337

M
Michal Hocko 已提交
338
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
G
Glauber Costa 已提交
339 340
	struct tcp_memcontrol tcp_mem;
#endif
341 342 343 344 345 346 347 348
#if defined(CONFIG_MEMCG_KMEM)
	/* analogous to slab_common's slab_caches list. per-memcg */
	struct list_head memcg_slab_caches;
	/* Not a spinlock, we can take a lot of time walking the list */
	struct mutex slab_caches_mutex;
        /* Index in the kmem_cache->memcg_params->memcg_caches array */
	int kmemcg_id;
#endif
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif
	/*
	 * Per cgroup active and inactive list, similar to the
	 * per zone LRU lists.
	 *
	 * WARNING: This has to be the last element of the struct. Don't
	 * add new fields after this point.
	 */
	struct mem_cgroup_lru_info info;
B
Balbir Singh 已提交
364 365
};

366 367 368 369 370 371
static size_t memcg_size(void)
{
	return sizeof(struct mem_cgroup) +
		nr_node_ids * sizeof(struct mem_cgroup_per_node);
}

372 373 374
/* internal only representation about the status of kmem accounting. */
enum {
	KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
375
	KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
376
	KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
377 378
};

379 380 381
/* We account when limit is on, but only after call sites are patched */
#define KMEM_ACCOUNTED_MASK \
		((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
382 383 384 385 386 387

#ifdef CONFIG_MEMCG_KMEM
static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
{
	set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
}
388 389 390 391 392 393

static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
{
	return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
}

394 395 396 397 398
static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
{
	set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
}

399 400 401 402 403
static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
{
	clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
}

404 405 406 407 408 409 410 411 412 413 414
static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
{
	if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
		set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
}

static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
{
	return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
				  &memcg->kmem_account_flags);
}
415 416
#endif

417 418
/* Stuffs for move charges at task migration. */
/*
419 420
 * Types of charges to be moved. "move_charge_at_immitgrate" and
 * "immigrate_flags" are treated as a left-shifted bitmap of these types.
421 422
 */
enum move_type {
423
	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */
424
	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */
425 426 427
	NR_MOVE_TYPE,
};

428 429
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
430
	spinlock_t	  lock; /* for from, to */
431 432
	struct mem_cgroup *from;
	struct mem_cgroup *to;
433
	unsigned long immigrate_flags;
434
	unsigned long precharge;
435
	unsigned long moved_charge;
436
	unsigned long moved_swap;
437 438 439
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
440
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
441 442
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
443

D
Daisuke Nishimura 已提交
444 445
static bool move_anon(void)
{
446
	return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
D
Daisuke Nishimura 已提交
447 448
}

449 450
static bool move_file(void)
{
451
	return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
452 453
}

454 455 456 457
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
458 459
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
460

461 462
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
463
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
464
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
465
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
466 467 468
	NR_CHARGE_TYPE,
};

469
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
470 471 472 473
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
474
	_KMEM,
G
Glauber Costa 已提交
475 476
};

477 478
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
479
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
480 481
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
482

483 484 485 486 487 488 489 490
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)

491 492 493 494 495 496 497
/*
 * The memcg_create_mutex will be held whenever a new cgroup is created.
 * As a consequence, any change that needs to protect against new child cgroups
 * appearing has to hold it as well.
 */
static DEFINE_MUTEX(memcg_create_mutex);

498 499
static void mem_cgroup_get(struct mem_cgroup *memcg);
static void mem_cgroup_put(struct mem_cgroup *memcg);
G
Glauber Costa 已提交
500

501 502 503 504 505 506
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
{
	return container_of(s, struct mem_cgroup, css);
}

507 508 509 510 511
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

G
Glauber Costa 已提交
512
/* Writing them here to avoid exposing memcg's inner layout */
M
Michal Hocko 已提交
513
#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
G
Glauber Costa 已提交
514 515 516

void sock_update_memcg(struct sock *sk)
{
517
	if (mem_cgroup_sockets_enabled) {
G
Glauber Costa 已提交
518
		struct mem_cgroup *memcg;
519
		struct cg_proto *cg_proto;
G
Glauber Costa 已提交
520 521 522

		BUG_ON(!sk->sk_prot->proto_cgroup);

523 524 525 526 527 528 529 530 531 532 533 534 535 536
		/* Socket cloning can throw us here with sk_cgrp already
		 * filled. It won't however, necessarily happen from
		 * process context. So the test for root memcg given
		 * the current task's memcg won't help us in this case.
		 *
		 * Respecting the original socket's memcg is a better
		 * decision in this case.
		 */
		if (sk->sk_cgrp) {
			BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
			mem_cgroup_get(sk->sk_cgrp->memcg);
			return;
		}

G
Glauber Costa 已提交
537 538
		rcu_read_lock();
		memcg = mem_cgroup_from_task(current);
539 540
		cg_proto = sk->sk_prot->proto_cgroup(memcg);
		if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
G
Glauber Costa 已提交
541
			mem_cgroup_get(memcg);
542
			sk->sk_cgrp = cg_proto;
G
Glauber Costa 已提交
543 544 545 546 547 548 549 550
		}
		rcu_read_unlock();
	}
}
EXPORT_SYMBOL(sock_update_memcg);

void sock_release_memcg(struct sock *sk)
{
551
	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
G
Glauber Costa 已提交
552 553 554 555 556 557
		struct mem_cgroup *memcg;
		WARN_ON(!sk->sk_cgrp->memcg);
		memcg = sk->sk_cgrp->memcg;
		mem_cgroup_put(memcg);
	}
}
G
Glauber Costa 已提交
558 559 560 561 562 563 564 565 566

struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg || mem_cgroup_is_root(memcg))
		return NULL;

	return &memcg->tcp_mem.cg_proto;
}
EXPORT_SYMBOL(tcp_proto_cgroup);
G
Glauber Costa 已提交
567

568 569 570 571 572 573 574 575 576 577 578 579
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
	if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
		return;
	static_key_slow_dec(&memcg_socket_limit_enabled);
}
#else
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
}
#endif

580
#ifdef CONFIG_MEMCG_KMEM
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
/*
 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
 * There are two main reasons for not using the css_id for this:
 *  1) this works better in sparse environments, where we have a lot of memcgs,
 *     but only a few kmem-limited. Or also, if we have, for instance, 200
 *     memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *     200 entry array for that.
 *
 *  2) In order not to violate the cgroup API, we would like to do all memory
 *     allocation in ->create(). At that point, we haven't yet allocated the
 *     css_id. Having a separate index prevents us from messing with the cgroup
 *     core for this
 *
 * The current size of the caches array is stored in
 * memcg_limited_groups_array_size.  It will double each time we have to
 * increase it.
 */
static DEFINE_IDA(kmem_limited_groups);
599 600
int memcg_limited_groups_array_size;

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
 * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
 * css_id space is not getting any smaller, and we don't have to necessarily
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
#define MEMCG_CACHES_MAX_SIZE 65535

616 617 618 619 620 621
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
622
struct static_key memcg_kmem_enabled_key;
623
EXPORT_SYMBOL(memcg_kmem_enabled_key);
624 625 626

static void disarm_kmem_keys(struct mem_cgroup *memcg)
{
627
	if (memcg_kmem_is_active(memcg)) {
628
		static_key_slow_dec(&memcg_kmem_enabled_key);
629 630
		ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
	}
631 632 633 634 635
	/*
	 * This check can't live in kmem destruction function,
	 * since the charges will outlive the cgroup
	 */
	WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
636 637 638 639 640 641 642 643 644 645 646 647 648
}
#else
static void disarm_kmem_keys(struct mem_cgroup *memcg)
{
}
#endif /* CONFIG_MEMCG_KMEM */

static void disarm_static_keys(struct mem_cgroup *memcg)
{
	disarm_sock_keys(memcg);
	disarm_kmem_keys(memcg);
}

649
static void drain_all_stock_async(struct mem_cgroup *memcg);
650

651
static struct mem_cgroup_per_zone *
652
mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
653
{
654
	VM_BUG_ON((unsigned)nid >= nr_node_ids);
655
	return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
656 657
}

658
struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
659
{
660
	return &memcg->css;
661 662
}

663
static struct mem_cgroup_per_zone *
664
page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
665
{
666 667
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
668

669
	return mem_cgroup_zoneinfo(memcg, nid, zid);
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
688
__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
689
				struct mem_cgroup_per_zone *mz,
690 691
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
692 693 694 695 696 697 698 699
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

700 701 702
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
719 720 721
}

static void
722
__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
723 724 725 726 727 728 729 730 731
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

732
static void
733
mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
734 735 736 737
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
738
	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
739 740 741 742
	spin_unlock(&mctz->lock);
}


743
static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
744
{
745
	unsigned long long excess;
746 747
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
748 749
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
750 751 752
	mctz = soft_limit_tree_from_page(page);

	/*
753 754
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
755
	 */
756 757 758
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		mz = mem_cgroup_zoneinfo(memcg, nid, zid);
		excess = res_counter_soft_limit_excess(&memcg->res);
759 760 761 762
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
763
		if (excess || mz->on_tree) {
764 765 766
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
767
				__mem_cgroup_remove_exceeded(memcg, mz, mctz);
768
			/*
769 770
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
771
			 */
772
			__mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
773 774
			spin_unlock(&mctz->lock);
		}
775 776 777
	}
}

778
static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
779 780 781 782 783
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

B
Bob Liu 已提交
784
	for_each_node(node) {
785
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
786
			mz = mem_cgroup_zoneinfo(memcg, node, zone);
787
			mctz = soft_limit_tree_node_zone(node, zone);
788
			mem_cgroup_remove_exceeded(memcg, mz, mctz);
789 790 791 792
		}
	}
}

793 794 795 796
static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
797
	struct mem_cgroup_per_zone *mz;
798 799

retry:
800
	mz = NULL;
801 802 803 804 805 806 807 808 809 810
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
811 812 813
	__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
		!css_tryget(&mz->memcg->css))
814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
849
static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
850
				 enum mem_cgroup_stat_index idx)
851
{
852
	long val = 0;
853 854
	int cpu;

855 856
	get_online_cpus();
	for_each_online_cpu(cpu)
857
		val += per_cpu(memcg->stat->count[idx], cpu);
858
#ifdef CONFIG_HOTPLUG_CPU
859 860 861
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.count[idx];
	spin_unlock(&memcg->pcp_counter_lock);
862 863
#endif
	put_online_cpus();
864 865 866
	return val;
}

867
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
868 869 870
					 bool charge)
{
	int val = (charge) ? 1 : -1;
871
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
872 873
}

874
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
875 876 877 878 879 880
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

	for_each_online_cpu(cpu)
881
		val += per_cpu(memcg->stat->events[idx], cpu);
882
#ifdef CONFIG_HOTPLUG_CPU
883 884 885
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.events[idx];
	spin_unlock(&memcg->pcp_counter_lock);
886 887 888 889
#endif
	return val;
}

890
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
891
					 bool anon, int nr_pages)
892
{
893 894
	preempt_disable();

895 896 897 898 899 900
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
	if (anon)
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
901
				nr_pages);
902
	else
903
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
904
				nr_pages);
905

906 907
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
908
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
909
	else {
910
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
911 912
		nr_pages = -nr_pages; /* for event */
	}
913

914
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
915

916
	preempt_enable();
917 918
}

919
unsigned long
920
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
921 922 923 924 925 926 927 928
{
	struct mem_cgroup_per_zone *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	return mz->lru_size[lru];
}

static unsigned long
929
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
930
			unsigned int lru_mask)
931 932
{
	struct mem_cgroup_per_zone *mz;
H
Hugh Dickins 已提交
933
	enum lru_list lru;
934 935
	unsigned long ret = 0;

936
	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
937

H
Hugh Dickins 已提交
938 939 940
	for_each_lru(lru) {
		if (BIT(lru) & lru_mask)
			ret += mz->lru_size[lru];
941 942 943 944 945
	}
	return ret;
}

static unsigned long
946
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
947 948
			int nid, unsigned int lru_mask)
{
949 950 951
	u64 total = 0;
	int zid;

952
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
953 954
		total += mem_cgroup_zone_nr_lru_pages(memcg,
						nid, zid, lru_mask);
955

956 957
	return total;
}
958

959
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
960
			unsigned int lru_mask)
961
{
962
	int nid;
963 964
	u64 total = 0;

965
	for_each_node_state(nid, N_MEMORY)
966
		total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
967
	return total;
968 969
}

970 971
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
972 973 974
{
	unsigned long val, next;

975
	val = __this_cpu_read(memcg->stat->nr_page_events);
976
	next = __this_cpu_read(memcg->stat->targets[target]);
977
	/* from time_after() in jiffies.h */
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
	if ((long)next - (long)val < 0) {
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
		__this_cpu_write(memcg->stat->targets[target], next);
		return true;
994
	}
995
	return false;
996 997 998 999 1000 1001
}

/*
 * Check events in order.
 *
 */
1002
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1003
{
1004
	preempt_disable();
1005
	/* threshold event is triggered in finer grain than soft limit */
1006 1007
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
1008 1009
		bool do_softlimit;
		bool do_numainfo __maybe_unused;
1010 1011 1012 1013 1014 1015 1016 1017 1018

		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
		preempt_enable();

1019
		mem_cgroup_threshold(memcg);
1020
		if (unlikely(do_softlimit))
1021
			mem_cgroup_update_tree(memcg, page);
1022
#if MAX_NUMNODES > 1
1023
		if (unlikely(do_numainfo))
1024
			atomic_inc(&memcg->numainfo_events);
1025
#endif
1026 1027
	} else
		preempt_enable();
1028 1029
}

G
Glauber Costa 已提交
1030
struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
1031
{
1032 1033
	return mem_cgroup_from_css(
		cgroup_subsys_state(cont, mem_cgroup_subsys_id));
B
Balbir Singh 已提交
1034 1035
}

1036
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1037
{
1038 1039 1040 1041 1042 1043 1044 1045
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

1046
	return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
1047 1048
}

1049
struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
1050
{
1051
	struct mem_cgroup *memcg = NULL;
1052 1053 1054

	if (!mm)
		return NULL;
1055 1056 1057 1058 1059 1060 1061
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
1062 1063
		memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!memcg))
1064
			break;
1065
	} while (!css_tryget(&memcg->css));
1066
	rcu_read_unlock();
1067
	return memcg;
1068 1069
}

1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
 * Reclaimers can specify a zone and a priority level in @reclaim to
 * divide up the memcgs in the hierarchy among all concurrent
 * reclaimers operating on the same zone and priority.
 */
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
				   struct mem_cgroup *prev,
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
1090
{
1091 1092
	struct mem_cgroup *memcg = NULL;
	int id = 0;
1093

1094 1095 1096
	if (mem_cgroup_disabled())
		return NULL;

1097 1098
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
1099

1100 1101
	if (prev && !reclaim)
		id = css_id(&prev->css);
K
KAMEZAWA Hiroyuki 已提交
1102

1103 1104
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
1105
			goto out_css_put;
1106 1107
		return root;
	}
K
KAMEZAWA Hiroyuki 已提交
1108

1109
	while (!memcg) {
1110
		struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1111
		struct cgroup_subsys_state *css;
1112

1113 1114 1115 1116 1117 1118 1119 1120
		if (reclaim) {
			int nid = zone_to_nid(reclaim->zone);
			int zid = zone_idx(reclaim->zone);
			struct mem_cgroup_per_zone *mz;

			mz = mem_cgroup_zoneinfo(root, nid, zid);
			iter = &mz->reclaim_iter[reclaim->priority];
			if (prev && reclaim->generation != iter->generation)
1121
				goto out_css_put;
1122 1123
			id = iter->position;
		}
K
KAMEZAWA Hiroyuki 已提交
1124

1125 1126 1127 1128
		rcu_read_lock();
		css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
		if (css) {
			if (css == &root->css || css_tryget(css))
1129
				memcg = mem_cgroup_from_css(css);
1130 1131
		} else
			id = 0;
K
KAMEZAWA Hiroyuki 已提交
1132 1133
		rcu_read_unlock();

1134 1135 1136 1137 1138 1139 1140
		if (reclaim) {
			iter->position = id;
			if (!css)
				iter->generation++;
			else if (!prev && memcg)
				reclaim->generation = iter->generation;
		}
1141 1142

		if (prev && !css)
1143
			goto out_css_put;
1144
	}
1145 1146 1147 1148
out_css_put:
	if (prev && prev != root)
		css_put(&prev->css);

1149
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
1150
}
K
KAMEZAWA Hiroyuki 已提交
1151

1152 1153 1154 1155 1156 1157 1158
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
1159 1160 1161 1162 1163 1164
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
1165

1166 1167 1168 1169 1170 1171
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
1172
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
1173
	     iter != NULL;				\
1174
	     iter = mem_cgroup_iter(root, iter, NULL))
1175

1176
#define for_each_mem_cgroup(iter)			\
1177
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
1178
	     iter != NULL;				\
1179
	     iter = mem_cgroup_iter(NULL, iter, NULL))
K
KAMEZAWA Hiroyuki 已提交
1180

1181
void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1182
{
1183
	struct mem_cgroup *memcg;
1184 1185

	rcu_read_lock();
1186 1187
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
1188 1189 1190 1191
		goto out;

	switch (idx) {
	case PGFAULT:
1192 1193 1194 1195
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
		break;
	case PGMAJFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1196 1197 1198 1199 1200 1201 1202
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
}
1203
EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1204

1205 1206 1207
/**
 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
 * @zone: zone of the wanted lruvec
1208
 * @memcg: memcg of the wanted lruvec
1209 1210 1211 1212 1213 1214 1215 1216 1217
 *
 * Returns the lru list vector holding pages for the given @zone and
 * @mem.  This can be the global zone lruvec, if the memory controller
 * is disabled.
 */
struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
				      struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_zone *mz;
1218
	struct lruvec *lruvec;
1219

1220 1221 1222 1223
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1224 1225

	mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
1236 1237
}

K
KAMEZAWA Hiroyuki 已提交
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
1251

1252
/**
1253
 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1254
 * @page: the page
1255
 * @zone: zone of the page
1256
 */
1257
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
K
KAMEZAWA Hiroyuki 已提交
1258 1259
{
	struct mem_cgroup_per_zone *mz;
1260 1261
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;
1262
	struct lruvec *lruvec;
1263

1264 1265 1266 1267
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1268

K
KAMEZAWA Hiroyuki 已提交
1269
	pc = lookup_page_cgroup(page);
1270
	memcg = pc->mem_cgroup;
1271 1272

	/*
1273
	 * Surreptitiously switch any uncharged offlist page to root:
1274 1275 1276 1277 1278 1279 1280
	 * an uncharged page off lru does nothing to secure
	 * its former mem_cgroup from sudden removal.
	 *
	 * Our caller holds lru_lock, and PageCgroupUsed is updated
	 * under page_cgroup lock: between them, they make all uses
	 * of pc->mem_cgroup safe.
	 */
1281
	if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1282 1283
		pc->mem_cgroup = memcg = root_mem_cgroup;

1284
	mz = page_cgroup_zoneinfo(memcg, page);
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1295
}
1296

1297
/**
1298 1299 1300 1301
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
 * @nr_pages: positive when adding or negative when removing
1302
 *
1303 1304
 * This function must be called when a page is added to or removed from an
 * lru list.
1305
 */
1306 1307
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
				int nr_pages)
1308 1309
{
	struct mem_cgroup_per_zone *mz;
1310
	unsigned long *lru_size;
1311 1312 1313 1314

	if (mem_cgroup_disabled())
		return;

1315 1316 1317 1318
	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	lru_size = mz->lru_size + lru;
	*lru_size += nr_pages;
	VM_BUG_ON((long)(*lru_size) < 0);
K
KAMEZAWA Hiroyuki 已提交
1319
}
1320

1321
/*
1322
 * Checks whether given mem is same or in the root_mem_cgroup's
1323 1324
 * hierarchy subtree
 */
1325 1326
bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				  struct mem_cgroup *memcg)
1327
{
1328 1329
	if (root_memcg == memcg)
		return true;
1330
	if (!root_memcg->use_hierarchy || !memcg)
1331
		return false;
1332 1333 1334 1335 1336 1337 1338 1339
	return css_is_ancestor(&memcg->css, &root_memcg->css);
}

static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				       struct mem_cgroup *memcg)
{
	bool ret;

1340
	rcu_read_lock();
1341
	ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1342 1343
	rcu_read_unlock();
	return ret;
1344 1345
}

1346
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
1347 1348
{
	int ret;
1349
	struct mem_cgroup *curr = NULL;
1350
	struct task_struct *p;
1351

1352
	p = find_lock_task_mm(task);
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
	if (p) {
		curr = try_get_mem_cgroup_from_mm(p->mm);
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
		task_lock(task);
		curr = mem_cgroup_from_task(task);
		if (curr)
			css_get(&curr->css);
		task_unlock(task);
	}
1368 1369
	if (!curr)
		return 0;
1370
	/*
1371
	 * We should check use_hierarchy of "memcg" not "curr". Because checking
1372
	 * use_hierarchy of "curr" here make this function true if hierarchy is
1373 1374
	 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
	 * hierarchy(even if use_hierarchy is disabled in "memcg").
1375
	 */
1376
	ret = mem_cgroup_same_or_subtree(memcg, curr);
1377
	css_put(&curr->css);
1378 1379 1380
	return ret;
}

1381
int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1382
{
1383
	unsigned long inactive_ratio;
1384
	unsigned long inactive;
1385
	unsigned long active;
1386
	unsigned long gb;
1387

1388 1389
	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1390

1391 1392 1393 1394 1395 1396
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

1397
	return inactive * inactive_ratio < active;
1398 1399
}

1400 1401 1402
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

1403
/**
1404
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1405
 * @memcg: the memory cgroup
1406
 *
1407
 * Returns the maximum amount of memory @mem can be charged with, in
1408
 * pages.
1409
 */
1410
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1411
{
1412 1413
	unsigned long long margin;

1414
	margin = res_counter_margin(&memcg->res);
1415
	if (do_swap_account)
1416
		margin = min(margin, res_counter_margin(&memcg->memsw));
1417
	return margin >> PAGE_SHIFT;
1418 1419
}

1420
int mem_cgroup_swappiness(struct mem_cgroup *memcg)
K
KOSAKI Motohiro 已提交
1421 1422 1423 1424 1425 1426 1427
{
	struct cgroup *cgrp = memcg->css.cgroup;

	/* root ? */
	if (cgrp->parent == NULL)
		return vm_swappiness;

1428
	return memcg->swappiness;
K
KOSAKI Motohiro 已提交
1429 1430
}

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
/*
 * memcg->moving_account is used for checking possibility that some thread is
 * calling move_account(). When a thread on CPU-A starts moving pages under
 * a memcg, other threads should check memcg->moving_account under
 * rcu_read_lock(), like this:
 *
 *         CPU-A                                    CPU-B
 *                                              rcu_read_lock()
 *         memcg->moving_account+1              if (memcg->mocing_account)
 *                                                   take heavy locks.
 *         synchronize_rcu()                    update something.
 *                                              rcu_read_unlock()
 *         start move here.
 */
1445 1446 1447 1448

/* for quick checking without looking up memcg */
atomic_t memcg_moving __read_mostly;

1449
static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1450
{
1451
	atomic_inc(&memcg_moving);
1452
	atomic_inc(&memcg->moving_account);
1453 1454 1455
	synchronize_rcu();
}

1456
static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1457
{
1458 1459 1460 1461
	/*
	 * Now, mem_cgroup_clear_mc() may call this function with NULL.
	 * We check NULL in callee rather than caller.
	 */
1462 1463
	if (memcg) {
		atomic_dec(&memcg_moving);
1464
		atomic_dec(&memcg->moving_account);
1465
	}
1466
}
1467

1468 1469 1470
/*
 * 2 routines for checking "mem" is under move_account() or not.
 *
1471 1472
 * mem_cgroup_stolen() -  checking whether a cgroup is mc.from or not. This
 *			  is used for avoiding races in accounting.  If true,
1473 1474 1475 1476 1477 1478 1479
 *			  pc->mem_cgroup may be overwritten.
 *
 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
 *			  under hierarchy of moving cgroups. This is for
 *			  waiting at hith-memory prressure caused by "move".
 */

1480
static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1481 1482
{
	VM_BUG_ON(!rcu_read_lock_held());
1483
	return atomic_read(&memcg->moving_account) > 0;
1484
}
1485

1486
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1487
{
1488 1489
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1490
	bool ret = false;
1491 1492 1493 1494 1495 1496 1497 1498 1499
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1500

1501 1502
	ret = mem_cgroup_same_or_subtree(memcg, from)
		|| mem_cgroup_same_or_subtree(memcg, to);
1503 1504
unlock:
	spin_unlock(&mc.lock);
1505 1506 1507
	return ret;
}

1508
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1509 1510
{
	if (mc.moving_task && current != mc.moving_task) {
1511
		if (mem_cgroup_under_move(memcg)) {
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1524 1525 1526 1527
/*
 * Take this lock when
 * - a code tries to modify page's memcg while it's USED.
 * - a code tries to modify page state accounting in a memcg.
1528
 * see mem_cgroup_stolen(), too.
1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
 */
static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
				  unsigned long *flags)
{
	spin_lock_irqsave(&memcg->move_lock, *flags);
}

static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
				unsigned long *flags)
{
	spin_unlock_irqrestore(&memcg->move_lock, *flags);
}

1542
#define K(x) ((x) << (PAGE_SHIFT-10))
1543
/**
1544
 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;
1562 1563
	struct mem_cgroup *iter;
	unsigned int i;
1564

1565
	if (!p)
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
		return;

	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

1584
	pr_info("Task in %s killed", memcg_name);
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
1597
	pr_cont(" as a result of limit of %s\n", memcg_name);
1598 1599
done:

1600
	pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1601 1602 1603
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
1604
	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1605 1606 1607
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1608
	pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1609 1610 1611
		res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635

	for_each_mem_cgroup_tree(iter, memcg) {
		pr_info("Memory cgroup stats");

		rcu_read_lock();
		ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
		if (!ret)
			pr_cont(" for %s", memcg_name);
		rcu_read_unlock();
		pr_cont(":");

		for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
			if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
				continue;
			pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
				K(mem_cgroup_read_stat(iter, i)));
		}

		for (i = 0; i < NR_LRU_LISTS; i++)
			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));

		pr_cont("\n");
	}
1636 1637
}

1638 1639 1640 1641
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1642
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1643 1644
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1645 1646
	struct mem_cgroup *iter;

1647
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1648
		num++;
1649 1650 1651
	return num;
}

D
David Rientjes 已提交
1652 1653 1654
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1655
static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1656 1657 1658
{
	u64 limit;

1659 1660
	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);

D
David Rientjes 已提交
1661
	/*
1662
	 * Do not consider swap space if we cannot swap due to swappiness
D
David Rientjes 已提交
1663
	 */
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
	if (mem_cgroup_swappiness(memcg)) {
		u64 memsw;

		limit += total_swap_pages << PAGE_SHIFT;
		memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);

		/*
		 * If memsw is finite and limits the amount of swap space
		 * available to this memcg, return that limit.
		 */
		limit = min(limit, memsw);
	}

	return limit;
D
David Rientjes 已提交
1678 1679
}

1680 1681
static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
				     int order)
1682 1683 1684 1685 1686 1687 1688
{
	struct mem_cgroup *iter;
	unsigned long chosen_points = 0;
	unsigned long totalpages;
	unsigned int points = 0;
	struct task_struct *chosen = NULL;

1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
	/*
	 * If current has a pending SIGKILL, then automatically select it.  The
	 * goal is to allow it to allocate so that it may quickly exit and free
	 * its memory.
	 */
	if (fatal_signal_pending(current)) {
		set_thread_flag(TIF_MEMDIE);
		return;
	}

	check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
	totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
	for_each_mem_cgroup_tree(iter, memcg) {
		struct cgroup *cgroup = iter->css.cgroup;
		struct cgroup_iter it;
		struct task_struct *task;

		cgroup_iter_start(cgroup, &it);
		while ((task = cgroup_iter_next(cgroup, &it))) {
			switch (oom_scan_process_thread(task, totalpages, NULL,
							false)) {
			case OOM_SCAN_SELECT:
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = ULONG_MAX;
				get_task_struct(chosen);
				/* fall through */
			case OOM_SCAN_CONTINUE:
				continue;
			case OOM_SCAN_ABORT:
				cgroup_iter_end(cgroup, &it);
				mem_cgroup_iter_break(memcg, iter);
				if (chosen)
					put_task_struct(chosen);
				return;
			case OOM_SCAN_OK:
				break;
			};
			points = oom_badness(task, memcg, NULL, totalpages);
			if (points > chosen_points) {
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = points;
				get_task_struct(chosen);
			}
		}
		cgroup_iter_end(cgroup, &it);
	}

	if (!chosen)
		return;
	points = chosen_points * 1000 / totalpages;
	oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
			 NULL, "Memory cgroup out of memory");
}

1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
					gfp_t gfp_mask,
					unsigned long flags)
{
	unsigned long total = 0;
	bool noswap = false;
	int loop;

	if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
		noswap = true;
	if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
		noswap = true;

	for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
		if (loop)
			drain_all_stock_async(memcg);
		total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
		/*
		 * Allow limit shrinkers, which are triggered directly
		 * by userspace, to catch signals and stop reclaim
		 * after minimal progress, regardless of the margin.
		 */
		if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
			break;
		if (mem_cgroup_margin(memcg))
			break;
		/*
		 * If nothing was reclaimed after two attempts, there
		 * may be no reclaimable pages in this hierarchy.
		 */
		if (loop && !total)
			break;
	}
	return total;
}

1783 1784
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1785
 * @memcg: the target memcg
1786 1787 1788 1789 1790 1791 1792
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1793
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1794 1795
		int nid, bool noswap)
{
1796
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1797 1798 1799
		return true;
	if (noswap || !total_swap_pages)
		return false;
1800
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1801 1802 1803 1804
		return true;
	return false;

}
1805 1806 1807 1808 1809 1810 1811 1812
#if MAX_NUMNODES > 1

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1813
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1814 1815
{
	int nid;
1816 1817 1818 1819
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1820
	if (!atomic_read(&memcg->numainfo_events))
1821
		return;
1822
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1823 1824 1825
		return;

	/* make a nodemask where this memcg uses memory from */
1826
	memcg->scan_nodes = node_states[N_MEMORY];
1827

1828
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1829

1830 1831
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1832
	}
1833

1834 1835
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1850
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1851 1852 1853
{
	int node;

1854 1855
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1856

1857
	node = next_node(node, memcg->scan_nodes);
1858
	if (node == MAX_NUMNODES)
1859
		node = first_node(memcg->scan_nodes);
1860 1861 1862 1863 1864 1865 1866 1867 1868
	/*
	 * We call this when we hit limit, not when pages are added to LRU.
	 * No LRU may hold pages because all pages are UNEVICTABLE or
	 * memcg is too small and all pages are not on LRU. In that case,
	 * we use curret node.
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1869
	memcg->last_scanned_node = node;
1870 1871 1872
	return node;
}

1873 1874 1875 1876 1877 1878
/*
 * Check all nodes whether it contains reclaimable pages or not.
 * For quick scan, we make use of scan_nodes. This will allow us to skip
 * unused nodes. But scan_nodes is lazily updated and may not cotain
 * enough new information. We need to do double check.
 */
1879
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1880 1881 1882 1883 1884 1885 1886
{
	int nid;

	/*
	 * quick check...making use of scan_node.
	 * We can skip unused nodes.
	 */
1887 1888
	if (!nodes_empty(memcg->scan_nodes)) {
		for (nid = first_node(memcg->scan_nodes);
1889
		     nid < MAX_NUMNODES;
1890
		     nid = next_node(nid, memcg->scan_nodes)) {
1891

1892
			if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1893 1894 1895 1896 1897 1898
				return true;
		}
	}
	/*
	 * Check rest of nodes.
	 */
1899
	for_each_node_state(nid, N_MEMORY) {
1900
		if (node_isset(nid, memcg->scan_nodes))
1901
			continue;
1902
		if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1903 1904 1905 1906 1907
			return true;
	}
	return false;
}

1908
#else
1909
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1910 1911 1912
{
	return 0;
}
1913

1914
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1915
{
1916
	return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1917
}
1918 1919
#endif

1920 1921 1922 1923
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
				   struct zone *zone,
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
1924
{
1925
	struct mem_cgroup *victim = NULL;
1926
	int total = 0;
K
KAMEZAWA Hiroyuki 已提交
1927
	int loop = 0;
1928
	unsigned long excess;
1929
	unsigned long nr_scanned;
1930 1931 1932 1933
	struct mem_cgroup_reclaim_cookie reclaim = {
		.zone = zone,
		.priority = 0,
	};
1934

1935
	excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
K
KAMEZAWA Hiroyuki 已提交
1936

1937
	while (1) {
1938
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1939
		if (!victim) {
K
KAMEZAWA Hiroyuki 已提交
1940
			loop++;
1941 1942 1943 1944 1945 1946
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
1947
				if (!total)
1948 1949
					break;
				/*
L
Lucas De Marchi 已提交
1950
				 * We want to do more targeted reclaim.
1951 1952 1953 1954 1955
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
1956
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1957 1958
					break;
			}
1959
			continue;
1960
		}
1961
		if (!mem_cgroup_reclaimable(victim, false))
1962
			continue;
1963 1964 1965 1966
		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
						     zone, &nr_scanned);
		*total_scanned += nr_scanned;
		if (!res_counter_soft_limit_excess(&root_memcg->res))
1967
			break;
1968
	}
1969
	mem_cgroup_iter_break(root_memcg, victim);
K
KAMEZAWA Hiroyuki 已提交
1970
	return total;
1971 1972
}

K
KAMEZAWA Hiroyuki 已提交
1973 1974 1975
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
1976
 * Has to be called with memcg_oom_lock
K
KAMEZAWA Hiroyuki 已提交
1977
 */
1978
static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
1979
{
1980
	struct mem_cgroup *iter, *failed = NULL;
1981

1982
	for_each_mem_cgroup_tree(iter, memcg) {
1983
		if (iter->oom_lock) {
1984 1985 1986 1987 1988
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
1989 1990
			mem_cgroup_iter_break(memcg, iter);
			break;
1991 1992
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
1993
	}
K
KAMEZAWA Hiroyuki 已提交
1994

1995
	if (!failed)
1996
		return true;
1997 1998 1999 2000 2001

	/*
	 * OK, we failed to lock the whole subtree so we have to clean up
	 * what we set up to the failing subtree
	 */
2002
	for_each_mem_cgroup_tree(iter, memcg) {
2003
		if (iter == failed) {
2004 2005
			mem_cgroup_iter_break(memcg, iter);
			break;
2006 2007 2008
		}
		iter->oom_lock = false;
	}
2009
	return false;
2010
}
2011

2012
/*
2013
 * Has to be called with memcg_oom_lock
2014
 */
2015
static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2016
{
K
KAMEZAWA Hiroyuki 已提交
2017 2018
	struct mem_cgroup *iter;

2019
	for_each_mem_cgroup_tree(iter, memcg)
2020 2021 2022 2023
		iter->oom_lock = false;
	return 0;
}

2024
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2025 2026 2027
{
	struct mem_cgroup *iter;

2028
	for_each_mem_cgroup_tree(iter, memcg)
2029 2030 2031
		atomic_inc(&iter->under_oom);
}

2032
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2033 2034 2035
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
2036 2037 2038 2039 2040
	/*
	 * When a new child is created while the hierarchy is under oom,
	 * mem_cgroup_oom_lock() may not be called. We have to use
	 * atomic_add_unless() here.
	 */
2041
	for_each_mem_cgroup_tree(iter, memcg)
2042
		atomic_add_unless(&iter->under_oom, -1, 0);
2043 2044
}

2045
static DEFINE_SPINLOCK(memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2046 2047
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
2048
struct oom_wait_info {
2049
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
2050 2051 2052 2053 2054 2055
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
2056 2057
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
2058 2059 2060
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2061
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
2062 2063

	/*
2064
	 * Both of oom_wait_info->memcg and wake_memcg are stable under us.
K
KAMEZAWA Hiroyuki 已提交
2065 2066
	 * Then we can use css_is_ancestor without taking care of RCU.
	 */
2067 2068
	if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
		&& !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
K
KAMEZAWA Hiroyuki 已提交
2069 2070 2071 2072
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

2073
static void memcg_wakeup_oom(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
2074
{
2075 2076
	/* for filtering, pass "memcg" as argument. */
	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
K
KAMEZAWA Hiroyuki 已提交
2077 2078
}

2079
static void memcg_oom_recover(struct mem_cgroup *memcg)
2080
{
2081 2082
	if (memcg && atomic_read(&memcg->under_oom))
		memcg_wakeup_oom(memcg);
2083 2084
}

K
KAMEZAWA Hiroyuki 已提交
2085 2086 2087
/*
 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
 */
2088 2089
static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
				  int order)
2090
{
K
KAMEZAWA Hiroyuki 已提交
2091
	struct oom_wait_info owait;
2092
	bool locked, need_to_kill;
K
KAMEZAWA Hiroyuki 已提交
2093

2094
	owait.memcg = memcg;
K
KAMEZAWA Hiroyuki 已提交
2095 2096 2097 2098
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
2099
	need_to_kill = true;
2100
	mem_cgroup_mark_under_oom(memcg);
2101

2102
	/* At first, try to OOM lock hierarchy under memcg.*/
2103
	spin_lock(&memcg_oom_lock);
2104
	locked = mem_cgroup_oom_lock(memcg);
K
KAMEZAWA Hiroyuki 已提交
2105 2106 2107 2108 2109
	/*
	 * Even if signal_pending(), we can't quit charge() loop without
	 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
	 * under OOM is always welcomed, use TASK_KILLABLE here.
	 */
2110
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2111
	if (!locked || memcg->oom_kill_disable)
2112 2113
		need_to_kill = false;
	if (locked)
2114
		mem_cgroup_oom_notify(memcg);
2115
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2116

2117 2118
	if (need_to_kill) {
		finish_wait(&memcg_oom_waitq, &owait.wait);
2119
		mem_cgroup_out_of_memory(memcg, mask, order);
2120
	} else {
K
KAMEZAWA Hiroyuki 已提交
2121
		schedule();
K
KAMEZAWA Hiroyuki 已提交
2122
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
2123
	}
2124
	spin_lock(&memcg_oom_lock);
2125
	if (locked)
2126 2127
		mem_cgroup_oom_unlock(memcg);
	memcg_wakeup_oom(memcg);
2128
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2129

2130
	mem_cgroup_unmark_under_oom(memcg);
2131

K
KAMEZAWA Hiroyuki 已提交
2132 2133 2134
	if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
		return false;
	/* Give chance to dying process */
2135
	schedule_timeout_uninterruptible(1);
K
KAMEZAWA Hiroyuki 已提交
2136
	return true;
2137 2138
}

2139 2140 2141
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
 *
 * Notes: Race condition
 *
 * We usually use page_cgroup_lock() for accessing page_cgroup member but
 * it tends to be costly. But considering some conditions, we doesn't need
 * to do so _always_.
 *
 * Considering "charge", lock_page_cgroup() is not required because all
 * file-stat operations happen after a page is attached to radix-tree. There
 * are no race with "charge".
 *
 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
 * if there are race with "uncharge". Statistics itself is properly handled
 * by flags.
 *
 * Considering "move", this is an only case we see a race. To make the race
2159 2160
 * small, we check mm->moving_account and detect there are possibility of race
 * If there is, we take a lock.
2161
 */
2162

2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
void __mem_cgroup_begin_update_page_stat(struct page *page,
				bool *locked, unsigned long *flags)
{
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
again:
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
		return;
	/*
	 * If this memory cgroup is not under account moving, we don't
2176
	 * need to take move_lock_mem_cgroup(). Because we already hold
2177
	 * rcu_read_lock(), any calls to move_account will be delayed until
2178
	 * rcu_read_unlock() if mem_cgroup_stolen() == true.
2179
	 */
2180
	if (!mem_cgroup_stolen(memcg))
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197
		return;

	move_lock_mem_cgroup(memcg, flags);
	if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
		move_unlock_mem_cgroup(memcg, flags);
		goto again;
	}
	*locked = true;
}

void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
{
	struct page_cgroup *pc = lookup_page_cgroup(page);

	/*
	 * It's guaranteed that pc->mem_cgroup never changes while
	 * lock is held because a routine modifies pc->mem_cgroup
2198
	 * should take move_lock_mem_cgroup().
2199 2200 2201 2202
	 */
	move_unlock_mem_cgroup(pc->mem_cgroup, flags);
}

2203 2204
void mem_cgroup_update_page_stat(struct page *page,
				 enum mem_cgroup_page_stat_item idx, int val)
2205
{
2206
	struct mem_cgroup *memcg;
2207
	struct page_cgroup *pc = lookup_page_cgroup(page);
2208
	unsigned long uninitialized_var(flags);
2209

2210
	if (mem_cgroup_disabled())
2211
		return;
2212

2213 2214
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
2215
		return;
2216 2217

	switch (idx) {
2218 2219
	case MEMCG_NR_FILE_MAPPED:
		idx = MEM_CGROUP_STAT_FILE_MAPPED;
2220 2221 2222
		break;
	default:
		BUG();
2223
	}
2224

2225
	this_cpu_add(memcg->stat->count[idx], val);
2226
}
2227

2228 2229 2230 2231
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
2232
#define CHARGE_BATCH	32U
2233 2234
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
2235
	unsigned int nr_pages;
2236
	struct work_struct work;
2237
	unsigned long flags;
2238
#define FLUSHING_CACHED_CHARGE	0
2239 2240
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2241
static DEFINE_MUTEX(percpu_charge_mutex);
2242

2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
2253
 */
2254
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2255 2256 2257 2258
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

2259 2260 2261
	if (nr_pages > CHARGE_BATCH)
		return false;

2262
	stock = &get_cpu_var(memcg_stock);
2263 2264
	if (memcg == stock->cached && stock->nr_pages >= nr_pages)
		stock->nr_pages -= nr_pages;
2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2278 2279 2280 2281
	if (stock->nr_pages) {
		unsigned long bytes = stock->nr_pages * PAGE_SIZE;

		res_counter_uncharge(&old->res, bytes);
2282
		if (do_swap_account)
2283 2284
			res_counter_uncharge(&old->memsw, bytes);
		stock->nr_pages = 0;
2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
2297
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2298 2299
}

2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
static void __init memcg_stock_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		struct memcg_stock_pcp *stock =
					&per_cpu(memcg_stock, cpu);
		INIT_WORK(&stock->work, drain_local_stock);
	}
}

2311 2312
/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
2313
 * This will be consumed by consume_stock() function, later.
2314
 */
2315
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2316 2317 2318
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

2319
	if (stock->cached != memcg) { /* reset if necessary */
2320
		drain_stock(stock);
2321
		stock->cached = memcg;
2322
	}
2323
	stock->nr_pages += nr_pages;
2324 2325 2326 2327
	put_cpu_var(memcg_stock);
}

/*
2328
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2329 2330
 * of the hierarchy under it. sync flag says whether we should block
 * until the work is done.
2331
 */
2332
static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2333
{
2334
	int cpu, curcpu;
2335

2336 2337
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
2338
	curcpu = get_cpu();
2339 2340
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2341
		struct mem_cgroup *memcg;
2342

2343 2344
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
2345
			continue;
2346
		if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2347
			continue;
2348 2349 2350 2351 2352 2353
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2354
	}
2355
	put_cpu();
2356 2357 2358 2359 2360 2361

	if (!sync)
		goto out;

	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2362
		if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2363 2364 2365
			flush_work(&stock->work);
	}
out:
2366
 	put_online_cpus();
2367 2368 2369 2370 2371 2372 2373 2374
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
2375
static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2376
{
2377 2378 2379 2380 2381
	/*
	 * If someone calls draining, avoid adding more kworker runs.
	 */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2382
	drain_all_stock(root_memcg, false);
2383
	mutex_unlock(&percpu_charge_mutex);
2384 2385 2386
}

/* This is a synchronous drain interface. */
2387
static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2388 2389
{
	/* called when force_empty is called */
2390
	mutex_lock(&percpu_charge_mutex);
2391
	drain_all_stock(root_memcg, true);
2392
	mutex_unlock(&percpu_charge_mutex);
2393 2394
}

2395 2396 2397 2398
/*
 * This function drains percpu counter value from DEAD cpu and
 * move it to local cpu. Note that this function can be preempted.
 */
2399
static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2400 2401 2402
{
	int i;

2403
	spin_lock(&memcg->pcp_counter_lock);
2404
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2405
		long x = per_cpu(memcg->stat->count[i], cpu);
2406

2407 2408
		per_cpu(memcg->stat->count[i], cpu) = 0;
		memcg->nocpu_base.count[i] += x;
2409
	}
2410
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2411
		unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2412

2413 2414
		per_cpu(memcg->stat->events[i], cpu) = 0;
		memcg->nocpu_base.events[i] += x;
2415
	}
2416
	spin_unlock(&memcg->pcp_counter_lock);
2417 2418 2419
}

static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2420 2421 2422 2423 2424
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;
2425
	struct mem_cgroup *iter;
2426

2427
	if (action == CPU_ONLINE)
2428 2429
		return NOTIFY_OK;

2430
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2431
		return NOTIFY_OK;
2432

2433
	for_each_mem_cgroup(iter)
2434 2435
		mem_cgroup_drain_pcp_counter(iter, cpu);

2436 2437 2438 2439 2440
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

2441 2442 2443 2444 2445 2446 2447 2448 2449 2450

/* See __mem_cgroup_try_charge() for details */
enum {
	CHARGE_OK,		/* success */
	CHARGE_RETRY,		/* need to retry but retry is not bad */
	CHARGE_NOMEM,		/* we can't do more. return -ENOMEM */
	CHARGE_WOULDBLOCK,	/* GFP_WAIT wasn't set and no enough res. */
	CHARGE_OOM_DIE,		/* the current is killed because of OOM */
};

2451
static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2452 2453
				unsigned int nr_pages, unsigned int min_pages,
				bool oom_check)
2454
{
2455
	unsigned long csize = nr_pages * PAGE_SIZE;
2456 2457 2458 2459 2460
	struct mem_cgroup *mem_over_limit;
	struct res_counter *fail_res;
	unsigned long flags = 0;
	int ret;

2461
	ret = res_counter_charge(&memcg->res, csize, &fail_res);
2462 2463 2464 2465

	if (likely(!ret)) {
		if (!do_swap_account)
			return CHARGE_OK;
2466
		ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2467 2468 2469
		if (likely(!ret))
			return CHARGE_OK;

2470
		res_counter_uncharge(&memcg->res, csize);
2471 2472 2473 2474
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
		flags |= MEM_CGROUP_RECLAIM_NOSWAP;
	} else
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2475 2476 2477 2478
	/*
	 * Never reclaim on behalf of optional batching, retry with a
	 * single page instead.
	 */
2479
	if (nr_pages > min_pages)
2480 2481 2482 2483 2484
		return CHARGE_RETRY;

	if (!(gfp_mask & __GFP_WAIT))
		return CHARGE_WOULDBLOCK;

2485 2486 2487
	if (gfp_mask & __GFP_NORETRY)
		return CHARGE_NOMEM;

2488
	ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2489
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2490
		return CHARGE_RETRY;
2491
	/*
2492 2493 2494 2495 2496 2497 2498
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
2499
	 */
2500
	if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513
		return CHARGE_RETRY;

	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		return CHARGE_RETRY;

	/* If we don't need to call oom-killer at el, return immediately */
	if (!oom_check)
		return CHARGE_NOMEM;
	/* check OOM */
2514
	if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
2515 2516 2517 2518 2519
		return CHARGE_OOM_DIE;

	return CHARGE_RETRY;
}

2520
/*
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
 * __mem_cgroup_try_charge() does
 * 1. detect memcg to be charged against from passed *mm and *ptr,
 * 2. update res_counter
 * 3. call memory reclaim if necessary.
 *
 * In some special case, if the task is fatal, fatal_signal_pending() or
 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
 * as possible without any hazards. 2: all pages should have a valid
 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
 * pointer, that is treated as a charge to root_mem_cgroup.
 *
 * So __mem_cgroup_try_charge() will return
 *  0       ...  on success, filling *ptr with a valid memcg pointer.
 *  -ENOMEM ...  charge failure because of resource limits.
 *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
 *
 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
 * the oom-killer can be invoked.
2540
 */
2541
static int __mem_cgroup_try_charge(struct mm_struct *mm,
A
Andrea Arcangeli 已提交
2542
				   gfp_t gfp_mask,
2543
				   unsigned int nr_pages,
2544
				   struct mem_cgroup **ptr,
2545
				   bool oom)
2546
{
2547
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2548
	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2549
	struct mem_cgroup *memcg = NULL;
2550
	int ret;
2551

K
KAMEZAWA Hiroyuki 已提交
2552 2553 2554 2555 2556 2557 2558 2559
	/*
	 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
	 * in system level. So, allow to go ahead dying process in addition to
	 * MEMDIE process.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)
		     || fatal_signal_pending(current)))
		goto bypass;
2560

2561
	/*
2562 2563
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
2564
	 * thread group leader migrates. It's possible that mm is not
2565
	 * set, if so charge the root memcg (happens for pagecache usage).
2566
	 */
2567
	if (!*ptr && !mm)
2568
		*ptr = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
2569
again:
2570 2571 2572
	if (*ptr) { /* css should be a valid one */
		memcg = *ptr;
		if (mem_cgroup_is_root(memcg))
K
KAMEZAWA Hiroyuki 已提交
2573
			goto done;
2574
		if (consume_stock(memcg, nr_pages))
K
KAMEZAWA Hiroyuki 已提交
2575
			goto done;
2576
		css_get(&memcg->css);
2577
	} else {
K
KAMEZAWA Hiroyuki 已提交
2578
		struct task_struct *p;
2579

K
KAMEZAWA Hiroyuki 已提交
2580 2581 2582
		rcu_read_lock();
		p = rcu_dereference(mm->owner);
		/*
2583
		 * Because we don't have task_lock(), "p" can exit.
2584
		 * In that case, "memcg" can point to root or p can be NULL with
2585 2586 2587 2588 2589 2590
		 * race with swapoff. Then, we have small risk of mis-accouning.
		 * But such kind of mis-account by race always happens because
		 * we don't have cgroup_mutex(). It's overkill and we allo that
		 * small race, here.
		 * (*) swapoff at el will charge against mm-struct not against
		 * task-struct. So, mm->owner can be NULL.
K
KAMEZAWA Hiroyuki 已提交
2591
		 */
2592
		memcg = mem_cgroup_from_task(p);
2593 2594 2595
		if (!memcg)
			memcg = root_mem_cgroup;
		if (mem_cgroup_is_root(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2596 2597 2598
			rcu_read_unlock();
			goto done;
		}
2599
		if (consume_stock(memcg, nr_pages)) {
K
KAMEZAWA Hiroyuki 已提交
2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
			/*
			 * It seems dagerous to access memcg without css_get().
			 * But considering how consume_stok works, it's not
			 * necessary. If consume_stock success, some charges
			 * from this memcg are cached on this cpu. So, we
			 * don't need to call css_get()/css_tryget() before
			 * calling consume_stock().
			 */
			rcu_read_unlock();
			goto done;
		}
		/* after here, we may be blocked. we need to get refcnt */
2612
		if (!css_tryget(&memcg->css)) {
K
KAMEZAWA Hiroyuki 已提交
2613 2614 2615 2616 2617
			rcu_read_unlock();
			goto again;
		}
		rcu_read_unlock();
	}
2618

2619 2620
	do {
		bool oom_check;
2621

2622
		/* If killed, bypass charge */
K
KAMEZAWA Hiroyuki 已提交
2623
		if (fatal_signal_pending(current)) {
2624
			css_put(&memcg->css);
2625
			goto bypass;
K
KAMEZAWA Hiroyuki 已提交
2626
		}
2627

2628 2629 2630 2631
		oom_check = false;
		if (oom && !nr_oom_retries) {
			oom_check = true;
			nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2632
		}
2633

2634 2635
		ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, nr_pages,
		    oom_check);
2636 2637 2638 2639
		switch (ret) {
		case CHARGE_OK:
			break;
		case CHARGE_RETRY: /* not in OOM situation but retry */
2640
			batch = nr_pages;
2641 2642
			css_put(&memcg->css);
			memcg = NULL;
K
KAMEZAWA Hiroyuki 已提交
2643
			goto again;
2644
		case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2645
			css_put(&memcg->css);
2646 2647
			goto nomem;
		case CHARGE_NOMEM: /* OOM routine works */
K
KAMEZAWA Hiroyuki 已提交
2648
			if (!oom) {
2649
				css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2650
				goto nomem;
K
KAMEZAWA Hiroyuki 已提交
2651
			}
2652 2653 2654 2655
			/* If oom, we never return -ENOMEM */
			nr_oom_retries--;
			break;
		case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2656
			css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2657
			goto bypass;
2658
		}
2659 2660
	} while (ret != CHARGE_OK);

2661
	if (batch > nr_pages)
2662 2663
		refill_stock(memcg, batch - nr_pages);
	css_put(&memcg->css);
2664
done:
2665
	*ptr = memcg;
2666 2667
	return 0;
nomem:
2668
	*ptr = NULL;
2669
	return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2670
bypass:
2671 2672
	*ptr = root_mem_cgroup;
	return -EINTR;
2673
}
2674

2675 2676 2677 2678 2679
/*
 * Somemtimes we have to undo a charge we got by try_charge().
 * This function is for that and do uncharge, put css's refcnt.
 * gotten by try_charge().
 */
2680
static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2681
				       unsigned int nr_pages)
2682
{
2683
	if (!mem_cgroup_is_root(memcg)) {
2684 2685
		unsigned long bytes = nr_pages * PAGE_SIZE;

2686
		res_counter_uncharge(&memcg->res, bytes);
2687
		if (do_swap_account)
2688
			res_counter_uncharge(&memcg->memsw, bytes);
2689
	}
2690 2691
}

2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709
/*
 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
 * This is useful when moving usage to parent cgroup.
 */
static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
					unsigned int nr_pages)
{
	unsigned long bytes = nr_pages * PAGE_SIZE;

	if (mem_cgroup_is_root(memcg))
		return;

	res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
	if (do_swap_account)
		res_counter_uncharge_until(&memcg->memsw,
						memcg->memsw.parent, bytes);
}

2710 2711
/*
 * A helper function to get mem_cgroup from ID. must be called under
T
Tejun Heo 已提交
2712 2713 2714
 * rcu_read_lock().  The caller is responsible for calling css_tryget if
 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
 * called against removed memcg.)
2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
2726
	return mem_cgroup_from_css(css);
2727 2728
}

2729
struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2730
{
2731
	struct mem_cgroup *memcg = NULL;
2732
	struct page_cgroup *pc;
2733
	unsigned short id;
2734 2735
	swp_entry_t ent;

2736 2737 2738
	VM_BUG_ON(!PageLocked(page));

	pc = lookup_page_cgroup(page);
2739
	lock_page_cgroup(pc);
2740
	if (PageCgroupUsed(pc)) {
2741 2742 2743
		memcg = pc->mem_cgroup;
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2744
	} else if (PageSwapCache(page)) {
2745
		ent.val = page_private(page);
2746
		id = lookup_swap_cgroup_id(ent);
2747
		rcu_read_lock();
2748 2749 2750
		memcg = mem_cgroup_lookup(id);
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2751
		rcu_read_unlock();
2752
	}
2753
	unlock_page_cgroup(pc);
2754
	return memcg;
2755 2756
}

2757
static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2758
				       struct page *page,
2759
				       unsigned int nr_pages,
2760 2761
				       enum charge_type ctype,
				       bool lrucare)
2762
{
2763
	struct page_cgroup *pc = lookup_page_cgroup(page);
2764
	struct zone *uninitialized_var(zone);
2765
	struct lruvec *lruvec;
2766
	bool was_on_lru = false;
2767
	bool anon;
2768

2769
	lock_page_cgroup(pc);
2770
	VM_BUG_ON(PageCgroupUsed(pc));
2771 2772 2773 2774
	/*
	 * we don't need page_cgroup_lock about tail pages, becase they are not
	 * accessed by any other context at this point.
	 */
2775 2776 2777 2778 2779 2780 2781 2782 2783

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
	if (lrucare) {
		zone = page_zone(page);
		spin_lock_irq(&zone->lru_lock);
		if (PageLRU(page)) {
2784
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2785
			ClearPageLRU(page);
2786
			del_page_from_lru_list(page, lruvec, page_lru(page));
2787 2788 2789 2790
			was_on_lru = true;
		}
	}

2791
	pc->mem_cgroup = memcg;
2792 2793 2794 2795 2796 2797 2798
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
2799
	smp_wmb();
2800
	SetPageCgroupUsed(pc);
2801

2802 2803
	if (lrucare) {
		if (was_on_lru) {
2804
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2805 2806
			VM_BUG_ON(PageLRU(page));
			SetPageLRU(page);
2807
			add_page_to_lru_list(page, lruvec, page_lru(page));
2808 2809 2810 2811
		}
		spin_unlock_irq(&zone->lru_lock);
	}

2812
	if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2813 2814 2815 2816 2817
		anon = true;
	else
		anon = false;

	mem_cgroup_charge_statistics(memcg, anon, nr_pages);
2818
	unlock_page_cgroup(pc);
2819

2820 2821 2822 2823 2824
	/*
	 * "charge_statistics" updated event counter. Then, check it.
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
	 */
2825
	memcg_check_events(memcg, page);
2826
}
2827

2828 2829
static DEFINE_MUTEX(set_limit_mutex);

2830 2831 2832 2833 2834 2835 2836
#ifdef CONFIG_MEMCG_KMEM
static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
{
	return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
		(memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
}

G
Glauber Costa 已提交
2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849
/*
 * This is a bit cumbersome, but it is rarely used and avoids a backpointer
 * in the memcg_cache_params struct.
 */
static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
{
	struct kmem_cache *cachep;

	VM_BUG_ON(p->is_root_cache);
	cachep = p->root_cache;
	return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
}

2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870
#ifdef CONFIG_SLABINFO
static int mem_cgroup_slabinfo_read(struct cgroup *cont, struct cftype *cft,
					struct seq_file *m)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
	struct memcg_cache_params *params;

	if (!memcg_can_account_kmem(memcg))
		return -EIO;

	print_slabinfo_header(m);

	mutex_lock(&memcg->slab_caches_mutex);
	list_for_each_entry(params, &memcg->memcg_slab_caches, list)
		cache_show(memcg_params_to_cache(params), m);
	mutex_unlock(&memcg->slab_caches_mutex);

	return 0;
}
#endif

2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
{
	struct res_counter *fail_res;
	struct mem_cgroup *_memcg;
	int ret = 0;
	bool may_oom;

	ret = res_counter_charge(&memcg->kmem, size, &fail_res);
	if (ret)
		return ret;

	/*
	 * Conditions under which we can wait for the oom_killer. Those are
	 * the same conditions tested by the core page allocator
	 */
	may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);

	_memcg = memcg;
	ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
				      &_memcg, may_oom);

	if (ret == -EINTR)  {
		/*
		 * __mem_cgroup_try_charge() chosed to bypass to root due to
		 * OOM kill or fatal signal.  Since our only options are to
		 * either fail the allocation or charge it to this cgroup, do
		 * it as a temporary condition. But we can't fail. From a
		 * kmem/slab perspective, the cache has already been selected,
		 * by mem_cgroup_kmem_get_cache(), so it is too late to change
		 * our minds.
		 *
		 * This condition will only trigger if the task entered
		 * memcg_charge_kmem in a sane state, but was OOM-killed during
		 * __mem_cgroup_try_charge() above. Tasks that were already
		 * dying when the allocation triggers should have been already
		 * directed to the root cgroup in memcontrol.h
		 */
		res_counter_charge_nofail(&memcg->res, size, &fail_res);
		if (do_swap_account)
			res_counter_charge_nofail(&memcg->memsw, size,
						  &fail_res);
		ret = 0;
	} else if (ret)
		res_counter_uncharge(&memcg->kmem, size);

	return ret;
}

static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
{
	res_counter_uncharge(&memcg->res, size);
	if (do_swap_account)
		res_counter_uncharge(&memcg->memsw, size);
2924 2925 2926 2927 2928 2929 2930

	/* Not down to 0 */
	if (res_counter_uncharge(&memcg->kmem, size))
		return;

	if (memcg_kmem_test_and_clear_dead(memcg))
		mem_cgroup_put(memcg);
2931 2932
}

2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
{
	if (!memcg)
		return;

	mutex_lock(&memcg->slab_caches_mutex);
	list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
	mutex_unlock(&memcg->slab_caches_mutex);
}

/*
 * helper for acessing a memcg's index. It will be used as an index in the
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}

2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
/*
 * This ends up being protected by the set_limit mutex, during normal
 * operation, because that is its main call site.
 *
 * But when we create a new cache, we can call this as well if its parent
 * is kmem-limited. That will have to hold set_limit_mutex as well.
 */
int memcg_update_cache_sizes(struct mem_cgroup *memcg)
{
	int num, ret;

	num = ida_simple_get(&kmem_limited_groups,
				0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (num < 0)
		return num;
	/*
	 * After this point, kmem_accounted (that we test atomically in
	 * the beginning of this conditional), is no longer 0. This
	 * guarantees only one process will set the following boolean
	 * to true. We don't need test_and_set because we're protected
	 * by the set_limit_mutex anyway.
	 */
	memcg_kmem_set_activated(memcg);

	ret = memcg_update_all_caches(num+1);
	if (ret) {
		ida_simple_remove(&kmem_limited_groups, num);
		memcg_kmem_clear_activated(memcg);
		return ret;
	}

	memcg->kmemcg_id = num;
	INIT_LIST_HEAD(&memcg->memcg_slab_caches);
	mutex_init(&memcg->slab_caches_mutex);
	return 0;
}

static size_t memcg_caches_array_size(int num_groups)
{
	ssize_t size;
	if (num_groups <= 0)
		return 0;

	size = 2 * num_groups;
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

	return size;
}

/*
 * We should update the current array size iff all caches updates succeed. This
 * can only be done from the slab side. The slab mutex needs to be held when
 * calling this.
 */
void memcg_update_array_size(int num)
{
	if (num > memcg_limited_groups_array_size)
		memcg_limited_groups_array_size = memcg_caches_array_size(num);
}

3016 3017
static void kmem_cache_destroy_work_func(struct work_struct *w);

3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036
int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
{
	struct memcg_cache_params *cur_params = s->memcg_params;

	VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);

	if (num_groups > memcg_limited_groups_array_size) {
		int i;
		ssize_t size = memcg_caches_array_size(num_groups);

		size *= sizeof(void *);
		size += sizeof(struct memcg_cache_params);

		s->memcg_params = kzalloc(size, GFP_KERNEL);
		if (!s->memcg_params) {
			s->memcg_params = cur_params;
			return -ENOMEM;
		}

3037 3038
		INIT_WORK(&s->memcg_params->destroy,
				kmem_cache_destroy_work_func);
3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070
		s->memcg_params->is_root_cache = true;

		/*
		 * There is the chance it will be bigger than
		 * memcg_limited_groups_array_size, if we failed an allocation
		 * in a cache, in which case all caches updated before it, will
		 * have a bigger array.
		 *
		 * But if that is the case, the data after
		 * memcg_limited_groups_array_size is certainly unused
		 */
		for (i = 0; i < memcg_limited_groups_array_size; i++) {
			if (!cur_params->memcg_caches[i])
				continue;
			s->memcg_params->memcg_caches[i] =
						cur_params->memcg_caches[i];
		}

		/*
		 * Ideally, we would wait until all caches succeed, and only
		 * then free the old one. But this is not worth the extra
		 * pointer per-cache we'd have to have for this.
		 *
		 * It is not a big deal if some caches are left with a size
		 * bigger than the others. And all updates will reset this
		 * anyway.
		 */
		kfree(cur_params);
	}
	return 0;
}

G
Glauber Costa 已提交
3071 3072
int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
			 struct kmem_cache *root_cache)
3073 3074 3075 3076 3077 3078
{
	size_t size = sizeof(struct memcg_cache_params);

	if (!memcg_kmem_enabled())
		return 0;

3079 3080 3081
	if (!memcg)
		size += memcg_limited_groups_array_size * sizeof(void *);

3082 3083 3084 3085
	s->memcg_params = kzalloc(size, GFP_KERNEL);
	if (!s->memcg_params)
		return -ENOMEM;

3086 3087
	INIT_WORK(&s->memcg_params->destroy,
			kmem_cache_destroy_work_func);
G
Glauber Costa 已提交
3088
	if (memcg) {
3089
		s->memcg_params->memcg = memcg;
G
Glauber Costa 已提交
3090
		s->memcg_params->root_cache = root_cache;
3091 3092 3093
	} else
		s->memcg_params->is_root_cache = true;

3094 3095 3096 3097 3098
	return 0;
}

void memcg_release_cache(struct kmem_cache *s)
{
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124
	struct kmem_cache *root;
	struct mem_cgroup *memcg;
	int id;

	/*
	 * This happens, for instance, when a root cache goes away before we
	 * add any memcg.
	 */
	if (!s->memcg_params)
		return;

	if (s->memcg_params->is_root_cache)
		goto out;

	memcg = s->memcg_params->memcg;
	id  = memcg_cache_id(memcg);

	root = s->memcg_params->root_cache;
	root->memcg_params->memcg_caches[id] = NULL;
	mem_cgroup_put(memcg);

	mutex_lock(&memcg->slab_caches_mutex);
	list_del(&s->memcg_params->list);
	mutex_unlock(&memcg->slab_caches_mutex);

out:
3125 3126 3127
	kfree(s->memcg_params);
}

3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158
/*
 * During the creation a new cache, we need to disable our accounting mechanism
 * altogether. This is true even if we are not creating, but rather just
 * enqueing new caches to be created.
 *
 * This is because that process will trigger allocations; some visible, like
 * explicit kmallocs to auxiliary data structures, name strings and internal
 * cache structures; some well concealed, like INIT_WORK() that can allocate
 * objects during debug.
 *
 * If any allocation happens during memcg_kmem_get_cache, we will recurse back
 * to it. This may not be a bounded recursion: since the first cache creation
 * failed to complete (waiting on the allocation), we'll just try to create the
 * cache again, failing at the same point.
 *
 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
 * memcg_kmem_skip_account. So we enclose anything that might allocate memory
 * inside the following two functions.
 */
static inline void memcg_stop_kmem_account(void)
{
	VM_BUG_ON(!current->mm);
	current->memcg_kmem_skip_account++;
}

static inline void memcg_resume_kmem_account(void)
{
	VM_BUG_ON(!current->mm);
	current->memcg_kmem_skip_account--;
}

G
Glauber Costa 已提交
3159 3160 3161 3162 3163 3164 3165 3166 3167
static void kmem_cache_destroy_work_func(struct work_struct *w)
{
	struct kmem_cache *cachep;
	struct memcg_cache_params *p;

	p = container_of(w, struct memcg_cache_params, destroy);

	cachep = memcg_params_to_cache(p);

G
Glauber Costa 已提交
3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188
	/*
	 * If we get down to 0 after shrink, we could delete right away.
	 * However, memcg_release_pages() already puts us back in the workqueue
	 * in that case. If we proceed deleting, we'll get a dangling
	 * reference, and removing the object from the workqueue in that case
	 * is unnecessary complication. We are not a fast path.
	 *
	 * Note that this case is fundamentally different from racing with
	 * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
	 * kmem_cache_shrink, not only we would be reinserting a dead cache
	 * into the queue, but doing so from inside the worker racing to
	 * destroy it.
	 *
	 * So if we aren't down to zero, we'll just schedule a worker and try
	 * again
	 */
	if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
		kmem_cache_shrink(cachep);
		if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
			return;
	} else
G
Glauber Costa 已提交
3189 3190 3191 3192 3193 3194 3195 3196
		kmem_cache_destroy(cachep);
}

void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
{
	if (!cachep->memcg_params->dead)
		return;

G
Glauber Costa 已提交
3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216
	/*
	 * There are many ways in which we can get here.
	 *
	 * We can get to a memory-pressure situation while the delayed work is
	 * still pending to run. The vmscan shrinkers can then release all
	 * cache memory and get us to destruction. If this is the case, we'll
	 * be executed twice, which is a bug (the second time will execute over
	 * bogus data). In this case, cancelling the work should be fine.
	 *
	 * But we can also get here from the worker itself, if
	 * kmem_cache_shrink is enough to shake all the remaining objects and
	 * get the page count to 0. In this case, we'll deadlock if we try to
	 * cancel the work (the worker runs with an internal lock held, which
	 * is the same lock we would hold for cancel_work_sync().)
	 *
	 * Since we can't possibly know who got us here, just refrain from
	 * running if there is already work pending
	 */
	if (work_pending(&cachep->memcg_params->destroy))
		return;
G
Glauber Costa 已提交
3217 3218 3219 3220 3221 3222 3223
	/*
	 * We have to defer the actual destroying to a workqueue, because
	 * we might currently be in a context that cannot sleep.
	 */
	schedule_work(&cachep->memcg_params->destroy);
}

3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
static char *memcg_cache_name(struct mem_cgroup *memcg, struct kmem_cache *s)
{
	char *name;
	struct dentry *dentry;

	rcu_read_lock();
	dentry = rcu_dereference(memcg->css.cgroup->dentry);
	rcu_read_unlock();

	BUG_ON(dentry == NULL);

	name = kasprintf(GFP_KERNEL, "%s(%d:%s)", s->name,
			 memcg_cache_id(memcg), dentry->d_name.name);

	return name;
}

static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
					 struct kmem_cache *s)
{
	char *name;
	struct kmem_cache *new;

	name = memcg_cache_name(memcg, s);
	if (!name)
		return NULL;

	new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
G
Glauber Costa 已提交
3252
				      (s->flags & ~SLAB_PANIC), s->ctor, s);
3253

3254 3255 3256
	if (new)
		new->allocflags |= __GFP_KMEMCG;

3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291
	kfree(name);
	return new;
}

/*
 * This lock protects updaters, not readers. We want readers to be as fast as
 * they can, and they will either see NULL or a valid cache value. Our model
 * allow them to see NULL, in which case the root memcg will be selected.
 *
 * We need this lock because multiple allocations to the same cache from a non
 * will span more than one worker. Only one of them can create the cache.
 */
static DEFINE_MUTEX(memcg_cache_mutex);
static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
						  struct kmem_cache *cachep)
{
	struct kmem_cache *new_cachep;
	int idx;

	BUG_ON(!memcg_can_account_kmem(memcg));

	idx = memcg_cache_id(memcg);

	mutex_lock(&memcg_cache_mutex);
	new_cachep = cachep->memcg_params->memcg_caches[idx];
	if (new_cachep)
		goto out;

	new_cachep = kmem_cache_dup(memcg, cachep);
	if (new_cachep == NULL) {
		new_cachep = cachep;
		goto out;
	}

	mem_cgroup_get(memcg);
G
Glauber Costa 已提交
3292
	atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304

	cachep->memcg_params->memcg_caches[idx] = new_cachep;
	/*
	 * the readers won't lock, make sure everybody sees the updated value,
	 * so they won't put stuff in the queue again for no reason
	 */
	wmb();
out:
	mutex_unlock(&memcg_cache_mutex);
	return new_cachep;
}

3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343
void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
{
	struct kmem_cache *c;
	int i;

	if (!s->memcg_params)
		return;
	if (!s->memcg_params->is_root_cache)
		return;

	/*
	 * If the cache is being destroyed, we trust that there is no one else
	 * requesting objects from it. Even if there are, the sanity checks in
	 * kmem_cache_destroy should caught this ill-case.
	 *
	 * Still, we don't want anyone else freeing memcg_caches under our
	 * noses, which can happen if a new memcg comes to life. As usual,
	 * we'll take the set_limit_mutex to protect ourselves against this.
	 */
	mutex_lock(&set_limit_mutex);
	for (i = 0; i < memcg_limited_groups_array_size; i++) {
		c = s->memcg_params->memcg_caches[i];
		if (!c)
			continue;

		/*
		 * We will now manually delete the caches, so to avoid races
		 * we need to cancel all pending destruction workers and
		 * proceed with destruction ourselves.
		 *
		 * kmem_cache_destroy() will call kmem_cache_shrink internally,
		 * and that could spawn the workers again: it is likely that
		 * the cache still have active pages until this very moment.
		 * This would lead us back to mem_cgroup_destroy_cache.
		 *
		 * But that will not execute at all if the "dead" flag is not
		 * set, so flip it down to guarantee we are in control.
		 */
		c->memcg_params->dead = false;
G
Glauber Costa 已提交
3344
		cancel_work_sync(&c->memcg_params->destroy);
3345 3346 3347 3348 3349
		kmem_cache_destroy(c);
	}
	mutex_unlock(&set_limit_mutex);
}

3350 3351 3352 3353 3354 3355
struct create_work {
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

G
Glauber Costa 已提交
3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372
static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
{
	struct kmem_cache *cachep;
	struct memcg_cache_params *params;

	if (!memcg_kmem_is_active(memcg))
		return;

	mutex_lock(&memcg->slab_caches_mutex);
	list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
		cachep = memcg_params_to_cache(params);
		cachep->memcg_params->dead = true;
		schedule_work(&cachep->memcg_params->destroy);
	}
	mutex_unlock(&memcg->slab_caches_mutex);
}

3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
static void memcg_create_cache_work_func(struct work_struct *w)
{
	struct create_work *cw;

	cw = container_of(w, struct create_work, work);
	memcg_create_kmem_cache(cw->memcg, cw->cachep);
	/* Drop the reference gotten when we enqueued. */
	css_put(&cw->memcg->css);
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 * Called with rcu_read_lock.
 */
3388 3389
static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
					 struct kmem_cache *cachep)
3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409
{
	struct create_work *cw;

	cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
	if (cw == NULL)
		return;

	/* The corresponding put will be done in the workqueue. */
	if (!css_tryget(&memcg->css)) {
		kfree(cw);
		return;
	}

	cw->memcg = memcg;
	cw->cachep = cachep;

	INIT_WORK(&cw->work, memcg_create_cache_work_func);
	schedule_work(&cw->work);
}

3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427
static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
				       struct kmem_cache *cachep)
{
	/*
	 * We need to stop accounting when we kmalloc, because if the
	 * corresponding kmalloc cache is not yet created, the first allocation
	 * in __memcg_create_cache_enqueue will recurse.
	 *
	 * However, it is better to enclose the whole function. Depending on
	 * the debugging options enabled, INIT_WORK(), for instance, can
	 * trigger an allocation. This too, will make us recurse. Because at
	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
	 * the safest choice is to do it like this, wrapping the whole function.
	 */
	memcg_stop_kmem_account();
	__memcg_create_cache_enqueue(memcg, cachep);
	memcg_resume_kmem_account();
}
3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449
/*
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
 * If the cache does not exist yet, if we are the first user of it,
 * we either create it immediately, if possible, or create it asynchronously
 * in a workqueue.
 * In the latter case, we will let the current allocation go through with
 * the original cache.
 *
 * Can't be called in interrupt context or from kernel threads.
 * This function needs to be called with rcu_read_lock() held.
 */
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
					  gfp_t gfp)
{
	struct mem_cgroup *memcg;
	int idx;

	VM_BUG_ON(!cachep->memcg_params);
	VM_BUG_ON(!cachep->memcg_params->is_root_cache);

3450 3451 3452
	if (!current->mm || current->memcg_kmem_skip_account)
		return cachep;

3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492
	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
	rcu_read_unlock();

	if (!memcg_can_account_kmem(memcg))
		return cachep;

	idx = memcg_cache_id(memcg);

	/*
	 * barrier to mare sure we're always seeing the up to date value.  The
	 * code updating memcg_caches will issue a write barrier to match this.
	 */
	read_barrier_depends();
	if (unlikely(cachep->memcg_params->memcg_caches[idx] == NULL)) {
		/*
		 * If we are in a safe context (can wait, and not in interrupt
		 * context), we could be be predictable and return right away.
		 * This would guarantee that the allocation being performed
		 * already belongs in the new cache.
		 *
		 * However, there are some clashes that can arrive from locking.
		 * For instance, because we acquire the slab_mutex while doing
		 * kmem_cache_dup, this means no further allocation could happen
		 * with the slab_mutex held.
		 *
		 * Also, because cache creation issue get_online_cpus(), this
		 * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
		 * that ends up reversed during cpu hotplug. (cpuset allocates
		 * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
		 * better to defer everything.
		 */
		memcg_create_cache_enqueue(memcg, cachep);
		return cachep;
	}

	return cachep->memcg_params->memcg_caches[idx];
}
EXPORT_SYMBOL(__memcg_kmem_get_cache);

3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587
/*
 * We need to verify if the allocation against current->mm->owner's memcg is
 * possible for the given order. But the page is not allocated yet, so we'll
 * need a further commit step to do the final arrangements.
 *
 * It is possible for the task to switch cgroups in this mean time, so at
 * commit time, we can't rely on task conversion any longer.  We'll then use
 * the handle argument to return to the caller which cgroup we should commit
 * against. We could also return the memcg directly and avoid the pointer
 * passing, but a boolean return value gives better semantics considering
 * the compiled-out case as well.
 *
 * Returning true means the allocation is possible.
 */
bool
__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
{
	struct mem_cgroup *memcg;
	int ret;

	*_memcg = NULL;
	memcg = try_get_mem_cgroup_from_mm(current->mm);

	/*
	 * very rare case described in mem_cgroup_from_task. Unfortunately there
	 * isn't much we can do without complicating this too much, and it would
	 * be gfp-dependent anyway. Just let it go
	 */
	if (unlikely(!memcg))
		return true;

	if (!memcg_can_account_kmem(memcg)) {
		css_put(&memcg->css);
		return true;
	}

	ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
	if (!ret)
		*_memcg = memcg;

	css_put(&memcg->css);
	return (ret == 0);
}

void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
			      int order)
{
	struct page_cgroup *pc;

	VM_BUG_ON(mem_cgroup_is_root(memcg));

	/* The page allocation failed. Revert */
	if (!page) {
		memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
		return;
	}

	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	pc->mem_cgroup = memcg;
	SetPageCgroupUsed(pc);
	unlock_page_cgroup(pc);
}

void __memcg_kmem_uncharge_pages(struct page *page, int order)
{
	struct mem_cgroup *memcg = NULL;
	struct page_cgroup *pc;


	pc = lookup_page_cgroup(page);
	/*
	 * Fast unlocked return. Theoretically might have changed, have to
	 * check again after locking.
	 */
	if (!PageCgroupUsed(pc))
		return;

	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
		memcg = pc->mem_cgroup;
		ClearPageCgroupUsed(pc);
	}
	unlock_page_cgroup(pc);

	/*
	 * We trust that only if there is a memcg associated with the page, it
	 * is a valid allocation
	 */
	if (!memcg)
		return;

	VM_BUG_ON(mem_cgroup_is_root(memcg));
	memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
}
G
Glauber Costa 已提交
3588 3589 3590 3591
#else
static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
{
}
3592 3593
#endif /* CONFIG_MEMCG_KMEM */

3594 3595
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

3596
#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
3597 3598
/*
 * Because tail pages are not marked as "used", set it. We're under
3599 3600 3601
 * zone->lru_lock, 'splitting on pmd' and compound_lock.
 * charge/uncharge will be never happen and move_account() is done under
 * compound_lock(), so we don't have to take care of races.
3602
 */
3603
void mem_cgroup_split_huge_fixup(struct page *head)
3604 3605
{
	struct page_cgroup *head_pc = lookup_page_cgroup(head);
3606 3607
	struct page_cgroup *pc;
	int i;
3608

3609 3610
	if (mem_cgroup_disabled())
		return;
3611 3612 3613 3614 3615 3616
	for (i = 1; i < HPAGE_PMD_NR; i++) {
		pc = head_pc + i;
		pc->mem_cgroup = head_pc->mem_cgroup;
		smp_wmb();/* see __commit_charge() */
		pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
	}
3617
}
3618
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3619

3620
/**
3621
 * mem_cgroup_move_account - move account of the page
3622
 * @page: the page
3623
 * @nr_pages: number of regular pages (>1 for huge pages)
3624 3625 3626 3627 3628
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
3629
 * - page is not on LRU (isolate_page() is useful.)
3630
 * - compound_lock is held when nr_pages > 1
3631
 *
3632 3633
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
3634
 */
3635 3636 3637 3638
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct page_cgroup *pc,
				   struct mem_cgroup *from,
3639
				   struct mem_cgroup *to)
3640
{
3641 3642
	unsigned long flags;
	int ret;
3643
	bool anon = PageAnon(page);
3644

3645
	VM_BUG_ON(from == to);
3646
	VM_BUG_ON(PageLRU(page));
3647 3648 3649 3650 3651 3652 3653
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
3654
	if (nr_pages > 1 && !PageTransHuge(page))
3655 3656 3657 3658 3659 3660 3661 3662
		goto out;

	lock_page_cgroup(pc);

	ret = -EINVAL;
	if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
		goto unlock;

3663
	move_lock_mem_cgroup(from, &flags);
3664

3665
	if (!anon && page_mapped(page)) {
3666 3667 3668 3669 3670
		/* Update mapped_file data for mem_cgroup */
		preempt_disable();
		__this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		__this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		preempt_enable();
3671
	}
3672
	mem_cgroup_charge_statistics(from, anon, -nr_pages);
3673

3674
	/* caller should have done css_get */
K
KAMEZAWA Hiroyuki 已提交
3675
	pc->mem_cgroup = to;
3676
	mem_cgroup_charge_statistics(to, anon, nr_pages);
3677
	move_unlock_mem_cgroup(from, &flags);
3678 3679
	ret = 0;
unlock:
3680
	unlock_page_cgroup(pc);
3681 3682 3683
	/*
	 * check events
	 */
3684 3685
	memcg_check_events(to, page);
	memcg_check_events(from, page);
3686
out:
3687 3688 3689
	return ret;
}

3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709
/**
 * mem_cgroup_move_parent - moves page to the parent group
 * @page: the page to move
 * @pc: page_cgroup of the page
 * @child: page's cgroup
 *
 * move charges to its parent or the root cgroup if the group has no
 * parent (aka use_hierarchy==0).
 * Although this might fail (get_page_unless_zero, isolate_lru_page or
 * mem_cgroup_move_account fails) the failure is always temporary and
 * it signals a race with a page removal/uncharge or migration. In the
 * first case the page is on the way out and it will vanish from the LRU
 * on the next attempt and the call should be retried later.
 * Isolation from the LRU fails only if page has been isolated from
 * the LRU since we looked at it and that usually means either global
 * reclaim or migration going on. The page will either get back to the
 * LRU or vanish.
 * Finaly mem_cgroup_move_account fails only if the page got uncharged
 * (!PageCgroupUsed) or moved to a different group. The page will
 * disappear in the next attempt.
3710
 */
3711 3712
static int mem_cgroup_move_parent(struct page *page,
				  struct page_cgroup *pc,
3713
				  struct mem_cgroup *child)
3714 3715
{
	struct mem_cgroup *parent;
3716
	unsigned int nr_pages;
3717
	unsigned long uninitialized_var(flags);
3718 3719
	int ret;

3720
	VM_BUG_ON(mem_cgroup_is_root(child));
3721

3722 3723 3724 3725 3726
	ret = -EBUSY;
	if (!get_page_unless_zero(page))
		goto out;
	if (isolate_lru_page(page))
		goto put;
3727

3728
	nr_pages = hpage_nr_pages(page);
K
KAMEZAWA Hiroyuki 已提交
3729

3730 3731 3732 3733 3734 3735
	parent = parent_mem_cgroup(child);
	/*
	 * If no parent, move charges to root cgroup.
	 */
	if (!parent)
		parent = root_mem_cgroup;
3736

3737 3738
	if (nr_pages > 1) {
		VM_BUG_ON(!PageTransHuge(page));
3739
		flags = compound_lock_irqsave(page);
3740
	}
3741

3742
	ret = mem_cgroup_move_account(page, nr_pages,
3743
				pc, child, parent);
3744 3745
	if (!ret)
		__mem_cgroup_cancel_local_charge(child, nr_pages);
3746

3747
	if (nr_pages > 1)
3748
		compound_unlock_irqrestore(page, flags);
K
KAMEZAWA Hiroyuki 已提交
3749
	putback_lru_page(page);
3750
put:
3751
	put_page(page);
3752
out:
3753 3754 3755
	return ret;
}

3756 3757 3758 3759 3760 3761 3762
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
3763
				gfp_t gfp_mask, enum charge_type ctype)
3764
{
3765
	struct mem_cgroup *memcg = NULL;
3766
	unsigned int nr_pages = 1;
3767
	bool oom = true;
3768
	int ret;
A
Andrea Arcangeli 已提交
3769

A
Andrea Arcangeli 已提交
3770
	if (PageTransHuge(page)) {
3771
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
3772
		VM_BUG_ON(!PageTransHuge(page));
3773 3774 3775 3776 3777
		/*
		 * Never OOM-kill a process for a huge page.  The
		 * fault handler will fall back to regular pages.
		 */
		oom = false;
A
Andrea Arcangeli 已提交
3778
	}
3779

3780
	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3781
	if (ret == -ENOMEM)
3782
		return ret;
3783
	__mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3784 3785 3786
	return 0;
}

3787 3788
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
3789
{
3790
	if (mem_cgroup_disabled())
3791
		return 0;
3792 3793 3794
	VM_BUG_ON(page_mapped(page));
	VM_BUG_ON(page->mapping && !PageAnon(page));
	VM_BUG_ON(!mm);
3795
	return mem_cgroup_charge_common(page, mm, gfp_mask,
3796
					MEM_CGROUP_CHARGE_TYPE_ANON);
3797 3798
}

3799 3800 3801
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
3802
 * struct page_cgroup is acquired. This refcnt will be consumed by
3803 3804
 * "commit()" or removed by "cancel()"
 */
3805 3806 3807 3808
static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
					  struct page *page,
					  gfp_t mask,
					  struct mem_cgroup **memcgp)
3809
{
3810
	struct mem_cgroup *memcg;
3811
	struct page_cgroup *pc;
3812
	int ret;
3813

3814 3815 3816 3817 3818 3819 3820 3821 3822 3823
	pc = lookup_page_cgroup(page);
	/*
	 * Every swap fault against a single page tries to charge the
	 * page, bail as early as possible.  shmem_unuse() encounters
	 * already charged pages, too.  The USED bit is protected by
	 * the page lock, which serializes swap cache removal, which
	 * in turn serializes uncharging.
	 */
	if (PageCgroupUsed(pc))
		return 0;
3824 3825
	if (!do_swap_account)
		goto charge_cur_mm;
3826 3827
	memcg = try_get_mem_cgroup_from_page(page);
	if (!memcg)
3828
		goto charge_cur_mm;
3829 3830
	*memcgp = memcg;
	ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
3831
	css_put(&memcg->css);
3832 3833
	if (ret == -EINTR)
		ret = 0;
3834
	return ret;
3835
charge_cur_mm:
3836 3837 3838 3839
	ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
	if (ret == -EINTR)
		ret = 0;
	return ret;
3840 3841
}

3842 3843 3844 3845 3846 3847
int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
				 gfp_t gfp_mask, struct mem_cgroup **memcgp)
{
	*memcgp = NULL;
	if (mem_cgroup_disabled())
		return 0;
3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
	/*
	 * A racing thread's fault, or swapoff, may have already
	 * updated the pte, and even removed page from swap cache: in
	 * those cases unuse_pte()'s pte_same() test will fail; but
	 * there's also a KSM case which does need to charge the page.
	 */
	if (!PageSwapCache(page)) {
		int ret;

		ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
		if (ret == -EINTR)
			ret = 0;
		return ret;
	}
3862 3863 3864
	return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
}

3865 3866 3867 3868 3869 3870 3871 3872 3873
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return;
	if (!memcg)
		return;
	__mem_cgroup_cancel_charge(memcg, 1);
}

D
Daisuke Nishimura 已提交
3874
static void
3875
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
D
Daisuke Nishimura 已提交
3876
					enum charge_type ctype)
3877
{
3878
	if (mem_cgroup_disabled())
3879
		return;
3880
	if (!memcg)
3881
		return;
3882

3883
	__mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
3884 3885 3886
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
3887 3888 3889
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
3890
	 */
3891
	if (do_swap_account && PageSwapCache(page)) {
3892
		swp_entry_t ent = {.val = page_private(page)};
3893
		mem_cgroup_uncharge_swap(ent);
3894
	}
3895 3896
}

3897 3898
void mem_cgroup_commit_charge_swapin(struct page *page,
				     struct mem_cgroup *memcg)
D
Daisuke Nishimura 已提交
3899
{
3900
	__mem_cgroup_commit_charge_swapin(page, memcg,
3901
					  MEM_CGROUP_CHARGE_TYPE_ANON);
D
Daisuke Nishimura 已提交
3902 3903
}

3904 3905
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
3906
{
3907 3908 3909 3910
	struct mem_cgroup *memcg = NULL;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
	int ret;

3911
	if (mem_cgroup_disabled())
3912 3913 3914 3915 3916 3917 3918
		return 0;
	if (PageCompound(page))
		return 0;

	if (!PageSwapCache(page))
		ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
	else { /* page is swapcache/shmem */
3919 3920
		ret = __mem_cgroup_try_charge_swapin(mm, page,
						     gfp_mask, &memcg);
3921 3922 3923 3924
		if (!ret)
			__mem_cgroup_commit_charge_swapin(page, memcg, type);
	}
	return ret;
3925 3926
}

3927
static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
3928 3929
				   unsigned int nr_pages,
				   const enum charge_type ctype)
3930 3931 3932
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
3933

3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
3945
		batch->memcg = memcg;
3946 3947
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
L
Lucas De Marchi 已提交
3948
	 * In those cases, all pages freed continuously can be expected to be in
3949 3950 3951 3952 3953 3954 3955 3956
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */

	if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

3957
	if (nr_pages > 1)
A
Andrea Arcangeli 已提交
3958 3959
		goto direct_uncharge;

3960 3961 3962 3963 3964
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
3965
	if (batch->memcg != memcg)
3966 3967
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
3968
	batch->nr_pages++;
3969
	if (uncharge_memsw)
3970
		batch->memsw_nr_pages++;
3971 3972
	return;
direct_uncharge:
3973
	res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
3974
	if (uncharge_memsw)
3975 3976 3977
		res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
	if (unlikely(batch->memcg != memcg))
		memcg_oom_recover(memcg);
3978
}
3979

3980
/*
3981
 * uncharge if !page_mapped(page)
3982
 */
3983
static struct mem_cgroup *
3984 3985
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
			     bool end_migration)
3986
{
3987
	struct mem_cgroup *memcg = NULL;
3988 3989
	unsigned int nr_pages = 1;
	struct page_cgroup *pc;
3990
	bool anon;
3991

3992
	if (mem_cgroup_disabled())
3993
		return NULL;
3994

3995
	VM_BUG_ON(PageSwapCache(page));
K
KAMEZAWA Hiroyuki 已提交
3996

A
Andrea Arcangeli 已提交
3997
	if (PageTransHuge(page)) {
3998
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
3999 4000
		VM_BUG_ON(!PageTransHuge(page));
	}
4001
	/*
4002
	 * Check if our page_cgroup is valid
4003
	 */
4004
	pc = lookup_page_cgroup(page);
4005
	if (unlikely(!PageCgroupUsed(pc)))
4006
		return NULL;
4007

4008
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
4009

4010
	memcg = pc->mem_cgroup;
4011

K
KAMEZAWA Hiroyuki 已提交
4012 4013 4014
	if (!PageCgroupUsed(pc))
		goto unlock_out;

4015 4016
	anon = PageAnon(page);

K
KAMEZAWA Hiroyuki 已提交
4017
	switch (ctype) {
4018
	case MEM_CGROUP_CHARGE_TYPE_ANON:
4019 4020 4021 4022 4023
		/*
		 * Generally PageAnon tells if it's the anon statistics to be
		 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
		 * used before page reached the stage of being marked PageAnon.
		 */
4024 4025
		anon = true;
		/* fallthrough */
K
KAMEZAWA Hiroyuki 已提交
4026
	case MEM_CGROUP_CHARGE_TYPE_DROP:
4027
		/* See mem_cgroup_prepare_migration() */
4028 4029 4030 4031 4032 4033 4034 4035 4036 4037
		if (page_mapped(page))
			goto unlock_out;
		/*
		 * Pages under migration may not be uncharged.  But
		 * end_migration() /must/ be the one uncharging the
		 * unused post-migration page and so it has to call
		 * here with the migration bit still set.  See the
		 * res_counter handling below.
		 */
		if (!end_migration && PageCgroupMigration(pc))
K
KAMEZAWA Hiroyuki 已提交
4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
4049
	}
K
KAMEZAWA Hiroyuki 已提交
4050

4051
	mem_cgroup_charge_statistics(memcg, anon, -nr_pages);
K
KAMEZAWA Hiroyuki 已提交
4052

4053
	ClearPageCgroupUsed(pc);
4054 4055 4056 4057 4058 4059
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
4060

4061
	unlock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
4062
	/*
4063
	 * even after unlock, we have memcg->res.usage here and this memcg
K
KAMEZAWA Hiroyuki 已提交
4064 4065
	 * will never be freed.
	 */
4066
	memcg_check_events(memcg, page);
K
KAMEZAWA Hiroyuki 已提交
4067
	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4068 4069
		mem_cgroup_swap_statistics(memcg, true);
		mem_cgroup_get(memcg);
K
KAMEZAWA Hiroyuki 已提交
4070
	}
4071 4072 4073 4074 4075 4076
	/*
	 * Migration does not charge the res_counter for the
	 * replacement page, so leave it alone when phasing out the
	 * page that is unused after the migration.
	 */
	if (!end_migration && !mem_cgroup_is_root(memcg))
4077
		mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4078

4079
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
4080 4081 4082

unlock_out:
	unlock_page_cgroup(pc);
4083
	return NULL;
4084 4085
}

4086 4087
void mem_cgroup_uncharge_page(struct page *page)
{
4088 4089 4090
	/* early check. */
	if (page_mapped(page))
		return;
4091
	VM_BUG_ON(page->mapping && !PageAnon(page));
4092 4093
	if (PageSwapCache(page))
		return;
4094
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4095 4096 4097 4098 4099
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
4100
	VM_BUG_ON(page->mapping);
4101
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4102 4103
}

4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
4118 4119
		current->memcg_batch.nr_pages = 0;
		current->memcg_batch.memsw_nr_pages = 0;
4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
4140 4141 4142 4143 4144 4145
	if (batch->nr_pages)
		res_counter_uncharge(&batch->memcg->res,
				     batch->nr_pages * PAGE_SIZE);
	if (batch->memsw_nr_pages)
		res_counter_uncharge(&batch->memcg->memsw,
				     batch->memsw_nr_pages * PAGE_SIZE);
4146
	memcg_oom_recover(batch->memcg);
4147 4148 4149 4150
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

4151
#ifdef CONFIG_SWAP
4152
/*
4153
 * called after __delete_from_swap_cache() and drop "page" account.
4154 4155
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
4156 4157
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4158 4159
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
4160 4161 4162 4163 4164
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

4165
	memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4166

K
KAMEZAWA Hiroyuki 已提交
4167 4168 4169 4170 4171
	/*
	 * record memcg information,  if swapout && memcg != NULL,
	 * mem_cgroup_get() was called in uncharge().
	 */
	if (do_swap_account && swapout && memcg)
4172
		swap_cgroup_record(ent, css_id(&memcg->css));
4173
}
4174
#endif
4175

A
Andrew Morton 已提交
4176
#ifdef CONFIG_MEMCG_SWAP
4177 4178 4179 4180 4181
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
4182
{
4183
	struct mem_cgroup *memcg;
4184
	unsigned short id;
4185 4186 4187 4188

	if (!do_swap_account)
		return;

4189 4190 4191
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
4192
	if (memcg) {
4193 4194 4195 4196
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
4197
		if (!mem_cgroup_is_root(memcg))
4198
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4199
		mem_cgroup_swap_statistics(memcg, false);
4200 4201
		mem_cgroup_put(memcg);
	}
4202
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
4203
}
4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
 * The caller must have charged to @to, IOW, called res_counter_charge() about
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
4220
				struct mem_cgroup *from, struct mem_cgroup *to)
4221 4222 4223 4224 4225 4226 4227 4228
{
	unsigned short old_id, new_id;

	old_id = css_id(&from->css);
	new_id = css_id(&to->css);

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
4229
		mem_cgroup_swap_statistics(to, true);
4230
		/*
4231 4232 4233 4234 4235 4236
		 * This function is only called from task migration context now.
		 * It postpones res_counter and refcount handling till the end
		 * of task migration(mem_cgroup_clear_mc()) for performance
		 * improvement. But we cannot postpone mem_cgroup_get(to)
		 * because if the process that has been moved to @to does
		 * swap-in, the refcount of @to might be decreased to 0.
4237 4238 4239 4240 4241 4242 4243 4244
		 */
		mem_cgroup_get(to);
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4245
				struct mem_cgroup *from, struct mem_cgroup *to)
4246 4247 4248
{
	return -EINVAL;
}
4249
#endif
K
KAMEZAWA Hiroyuki 已提交
4250

4251
/*
4252 4253
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
4254
 */
4255 4256
void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
				  struct mem_cgroup **memcgp)
4257
{
4258
	struct mem_cgroup *memcg = NULL;
4259
	unsigned int nr_pages = 1;
4260
	struct page_cgroup *pc;
4261
	enum charge_type ctype;
4262

4263
	*memcgp = NULL;
4264

4265
	if (mem_cgroup_disabled())
4266
		return;
4267

4268 4269 4270
	if (PageTransHuge(page))
		nr_pages <<= compound_order(page);

4271 4272 4273
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
4274 4275
		memcg = pc->mem_cgroup;
		css_get(&memcg->css);
4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
		/*
		 * At migrating an anonymous page, its mapcount goes down
		 * to 0 and uncharge() will be called. But, even if it's fully
		 * unmapped, migration may fail and this page has to be
		 * charged again. We set MIGRATION flag here and delay uncharge
		 * until end_migration() is called
		 *
		 * Corner Case Thinking
		 * A)
		 * When the old page was mapped as Anon and it's unmap-and-freed
		 * while migration was ongoing.
		 * If unmap finds the old page, uncharge() of it will be delayed
		 * until end_migration(). If unmap finds a new page, it's
		 * uncharged when it make mapcount to be 1->0. If unmap code
		 * finds swap_migration_entry, the new page will not be mapped
		 * and end_migration() will find it(mapcount==0).
		 *
		 * B)
		 * When the old page was mapped but migraion fails, the kernel
		 * remaps it. A charge for it is kept by MIGRATION flag even
		 * if mapcount goes down to 0. We can do remap successfully
		 * without charging it again.
		 *
		 * C)
		 * The "old" page is under lock_page() until the end of
		 * migration, so, the old page itself will not be swapped-out.
		 * If the new page is swapped out before end_migraton, our
		 * hook to usual swap-out path will catch the event.
		 */
		if (PageAnon(page))
			SetPageCgroupMigration(pc);
4307
	}
4308
	unlock_page_cgroup(pc);
4309 4310 4311 4312
	/*
	 * If the page is not charged at this point,
	 * we return here.
	 */
4313
	if (!memcg)
4314
		return;
4315

4316
	*memcgp = memcg;
4317 4318 4319 4320 4321 4322 4323
	/*
	 * We charge new page before it's used/mapped. So, even if unlock_page()
	 * is called before end_migration, we can catch all events on this new
	 * page. In the case new page is migrated but not remapped, new page's
	 * mapcount will be finally 0 and we call uncharge in end_migration().
	 */
	if (PageAnon(page))
4324
		ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4325
	else
4326
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4327 4328 4329 4330 4331
	/*
	 * The page is committed to the memcg, but it's not actually
	 * charged to the res_counter since we plan on replacing the
	 * old one and only one page is going to be left afterwards.
	 */
4332
	__mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4333
}
4334

4335
/* remove redundant charge if migration failed*/
4336
void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4337
	struct page *oldpage, struct page *newpage, bool migration_ok)
4338
{
4339
	struct page *used, *unused;
4340
	struct page_cgroup *pc;
4341
	bool anon;
4342

4343
	if (!memcg)
4344
		return;
4345

4346
	if (!migration_ok) {
4347 4348
		used = oldpage;
		unused = newpage;
4349
	} else {
4350
		used = newpage;
4351 4352
		unused = oldpage;
	}
4353
	anon = PageAnon(used);
4354 4355 4356 4357
	__mem_cgroup_uncharge_common(unused,
				     anon ? MEM_CGROUP_CHARGE_TYPE_ANON
				     : MEM_CGROUP_CHARGE_TYPE_CACHE,
				     true);
4358
	css_put(&memcg->css);
4359
	/*
4360 4361 4362
	 * We disallowed uncharge of pages under migration because mapcount
	 * of the page goes down to zero, temporarly.
	 * Clear the flag and check the page should be charged.
4363
	 */
4364 4365 4366 4367 4368
	pc = lookup_page_cgroup(oldpage);
	lock_page_cgroup(pc);
	ClearPageCgroupMigration(pc);
	unlock_page_cgroup(pc);

4369
	/*
4370 4371 4372 4373 4374 4375
	 * If a page is a file cache, radix-tree replacement is very atomic
	 * and we can skip this check. When it was an Anon page, its mapcount
	 * goes down to 0. But because we added MIGRATION flage, it's not
	 * uncharged yet. There are several case but page->mapcount check
	 * and USED bit check in mem_cgroup_uncharge_page() will do enough
	 * check. (see prepare_charge() also)
4376
	 */
4377
	if (anon)
4378
		mem_cgroup_uncharge_page(used);
4379
}
4380

4381 4382 4383 4384 4385 4386 4387 4388
/*
 * At replace page cache, newpage is not under any memcg but it's on
 * LRU. So, this function doesn't touch res_counter but handles LRU
 * in correct way. Both pages are locked so we cannot race with uncharge.
 */
void mem_cgroup_replace_page_cache(struct page *oldpage,
				  struct page *newpage)
{
4389
	struct mem_cgroup *memcg = NULL;
4390 4391 4392 4393 4394 4395 4396 4397 4398
	struct page_cgroup *pc;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;

	if (mem_cgroup_disabled())
		return;

	pc = lookup_page_cgroup(oldpage);
	/* fix accounting on old pages */
	lock_page_cgroup(pc);
4399 4400 4401 4402 4403
	if (PageCgroupUsed(pc)) {
		memcg = pc->mem_cgroup;
		mem_cgroup_charge_statistics(memcg, false, -1);
		ClearPageCgroupUsed(pc);
	}
4404 4405
	unlock_page_cgroup(pc);

4406 4407 4408 4409 4410 4411
	/*
	 * When called from shmem_replace_page(), in some cases the
	 * oldpage has already been charged, and in some cases not.
	 */
	if (!memcg)
		return;
4412 4413 4414 4415 4416
	/*
	 * Even if newpage->mapping was NULL before starting replacement,
	 * the newpage may be on LRU(or pagevec for LRU) already. We lock
	 * LRU while we overwrite pc->mem_cgroup.
	 */
4417
	__mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4418 4419
}

4420 4421 4422 4423 4424 4425
#ifdef CONFIG_DEBUG_VM
static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
4426 4427 4428 4429 4430
	/*
	 * Can be NULL while feeding pages into the page allocator for
	 * the first time, i.e. during boot or memory hotplug;
	 * or when mem_cgroup_disabled().
	 */
4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449
	if (likely(pc) && PageCgroupUsed(pc))
		return pc;
	return NULL;
}

bool mem_cgroup_bad_page_check(struct page *page)
{
	if (mem_cgroup_disabled())
		return false;

	return lookup_page_cgroup_used(page) != NULL;
}

void mem_cgroup_print_bad_page(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup_used(page);
	if (pc) {
4450 4451
		pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
			 pc, pc->flags, pc->mem_cgroup);
4452 4453 4454 4455
	}
}
#endif

4456
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4457
				unsigned long long val)
4458
{
4459
	int retry_count;
4460
	u64 memswlimit, memlimit;
4461
	int ret = 0;
4462 4463
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;
4464
	int enlarge;
4465 4466 4467 4468 4469 4470 4471 4472 4473

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4474

4475
	enlarge = 0;
4476
	while (retry_count) {
4477 4478 4479 4480
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
4481 4482 4483
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
4484
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4485 4486 4487 4488 4489 4490
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
4491 4492
			break;
		}
4493 4494 4495 4496 4497

		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit < val)
			enlarge = 1;

4498
		ret = res_counter_set_limit(&memcg->res, val);
4499 4500 4501 4502 4503 4504
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
4505 4506 4507 4508 4509
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

4510 4511
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_SHRINK);
4512 4513 4514 4515 4516 4517
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
4518
	}
4519 4520
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
4521

4522 4523 4524
	return ret;
}

L
Li Zefan 已提交
4525 4526
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
4527
{
4528
	int retry_count;
4529
	u64 memlimit, memswlimit, oldusage, curusage;
4530 4531
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
4532
	int enlarge = 0;
4533

4534 4535 4536
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4537 4538 4539 4540 4541 4542 4543 4544
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
4545
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4546 4547 4548 4549 4550 4551 4552 4553
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
4554 4555 4556
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val)
			enlarge = 1;
4557
		ret = res_counter_set_limit(&memcg->memsw, val);
4558 4559 4560 4561 4562 4563
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
4564 4565 4566 4567 4568
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

4569 4570 4571
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_NOSWAP |
				   MEM_CGROUP_RECLAIM_SHRINK);
4572
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4573
		/* Usage is reduced ? */
4574
		if (curusage >= oldusage)
4575
			retry_count--;
4576 4577
		else
			oldusage = curusage;
4578
	}
4579 4580
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
4581 4582 4583
	return ret;
}

4584
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4585 4586
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
4587 4588 4589 4590 4591 4592
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
4593
	unsigned long long excess;
4594
	unsigned long nr_scanned;
4595 4596 4597 4598

	if (order > 0)
		return 0;

4599
	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

4613
		nr_scanned = 0;
4614
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4615
						    gfp_mask, &nr_scanned);
4616
		nr_reclaimed += reclaimed;
4617
		*total_scanned += nr_scanned;
4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
4640
				if (next_mz == mz)
4641
					css_put(&next_mz->memcg->css);
4642
				else /* next_mz == NULL or other memcg */
4643 4644 4645
					break;
			} while (1);
		}
4646 4647
		__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
		excess = res_counter_soft_limit_excess(&mz->memcg->res);
4648 4649 4650 4651 4652 4653 4654 4655
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
4656
		/* If excess == 0, no tree ops */
4657
		__mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4658
		spin_unlock(&mctz->lock);
4659
		css_put(&mz->memcg->css);
4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
4672
		css_put(&next_mz->memcg->css);
4673 4674 4675
	return nr_reclaimed;
}

4676 4677 4678 4679 4680 4681 4682
/**
 * mem_cgroup_force_empty_list - clears LRU of a group
 * @memcg: group to clear
 * @node: NUMA node
 * @zid: zone id
 * @lru: lru to to clear
 *
4683
 * Traverse a specified page_cgroup list and try to drop them all.  This doesn't
4684 4685
 * reclaim the pages page themselves - pages are moved to the parent (or root)
 * group.
4686
 */
4687
static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
K
KAMEZAWA Hiroyuki 已提交
4688
				int node, int zid, enum lru_list lru)
4689
{
4690
	struct lruvec *lruvec;
4691
	unsigned long flags;
4692
	struct list_head *list;
4693 4694
	struct page *busy;
	struct zone *zone;
4695

K
KAMEZAWA Hiroyuki 已提交
4696
	zone = &NODE_DATA(node)->node_zones[zid];
4697 4698
	lruvec = mem_cgroup_zone_lruvec(zone, memcg);
	list = &lruvec->lists[lru];
4699

4700
	busy = NULL;
4701
	do {
4702
		struct page_cgroup *pc;
4703 4704
		struct page *page;

K
KAMEZAWA Hiroyuki 已提交
4705
		spin_lock_irqsave(&zone->lru_lock, flags);
4706
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
4707
			spin_unlock_irqrestore(&zone->lru_lock, flags);
4708
			break;
4709
		}
4710 4711 4712
		page = list_entry(list->prev, struct page, lru);
		if (busy == page) {
			list_move(&page->lru, list);
4713
			busy = NULL;
K
KAMEZAWA Hiroyuki 已提交
4714
			spin_unlock_irqrestore(&zone->lru_lock, flags);
4715 4716
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
4717
		spin_unlock_irqrestore(&zone->lru_lock, flags);
4718

4719
		pc = lookup_page_cgroup(page);
4720

4721
		if (mem_cgroup_move_parent(page, pc, memcg)) {
4722
			/* found lock contention or "pc" is obsolete. */
4723
			busy = page;
4724 4725 4726
			cond_resched();
		} else
			busy = NULL;
4727
	} while (!list_empty(list));
4728 4729 4730
}

/*
4731 4732
 * make mem_cgroup's charge to be 0 if there is no task by moving
 * all the charges and pages to the parent.
4733
 * This enables deleting this mem_cgroup.
4734 4735
 *
 * Caller is responsible for holding css reference on the memcg.
4736
 */
4737
static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4738
{
4739
	int node, zid;
4740
	u64 usage;
4741

4742
	do {
4743 4744
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
4745 4746
		drain_all_stock_sync(memcg);
		mem_cgroup_start_move(memcg);
4747
		for_each_node_state(node, N_MEMORY) {
4748
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
H
Hugh Dickins 已提交
4749 4750
				enum lru_list lru;
				for_each_lru(lru) {
4751
					mem_cgroup_force_empty_list(memcg,
H
Hugh Dickins 已提交
4752
							node, zid, lru);
4753
				}
4754
			}
4755
		}
4756 4757
		mem_cgroup_end_move(memcg);
		memcg_oom_recover(memcg);
4758
		cond_resched();
4759

4760
		/*
4761 4762 4763 4764 4765
		 * Kernel memory may not necessarily be trackable to a specific
		 * process. So they are not migrated, and therefore we can't
		 * expect their value to drop to 0 here.
		 * Having res filled up with kmem only is enough.
		 *
4766 4767 4768 4769 4770 4771
		 * This is a safety check because mem_cgroup_force_empty_list
		 * could have raced with mem_cgroup_replace_page_cache callers
		 * so the lru seemed empty but the page could have been added
		 * right after the check. RES_USAGE should be safe as we always
		 * charge before adding to the LRU.
		 */
4772 4773 4774
		usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
			res_counter_read_u64(&memcg->kmem, RES_USAGE);
	} while (usage > 0);
4775 4776
}

4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792
/*
 * This mainly exists for tests during the setting of set of use_hierarchy.
 * Since this is the very setting we are changing, the current hierarchy value
 * is meaningless
 */
static inline bool __memcg_has_children(struct mem_cgroup *memcg)
{
	struct cgroup *pos;

	/* bounce at first found */
	cgroup_for_each_child(pos, memcg->css.cgroup)
		return true;
	return false;
}

/*
4793 4794
 * Must be called with memcg_create_mutex held, unless the cgroup is guaranteed
 * to be already dead (as in mem_cgroup_force_empty, for instance).  This is
4795 4796 4797 4798 4799 4800 4801 4802 4803
 * from mem_cgroup_count_children(), in the sense that we don't really care how
 * many children we have; we only need to know if we have any.  It also counts
 * any memcg without hierarchy as infertile.
 */
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
	return memcg->use_hierarchy && __memcg_has_children(memcg);
}

4804 4805 4806 4807 4808 4809 4810 4811 4812 4813
/*
 * Reclaims as many pages from the given memcg as possible and moves
 * the rest to the parent.
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
	struct cgroup *cgrp = memcg->css.cgroup;
4814

4815
	/* returns EBUSY if there is a task or if we come here twice. */
4816 4817 4818
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
		return -EBUSY;

4819 4820
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
4821
	/* try to free all pages in this cgroup */
4822
	while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
4823
		int progress;
4824

4825 4826 4827
		if (signal_pending(current))
			return -EINTR;

4828
		progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
4829
						false);
4830
		if (!progress) {
4831
			nr_retries--;
4832
			/* maybe some writeback is necessary */
4833
			congestion_wait(BLK_RW_ASYNC, HZ/10);
4834
		}
4835 4836

	}
K
KAMEZAWA Hiroyuki 已提交
4837
	lru_add_drain();
4838 4839 4840
	mem_cgroup_reparent_charges(memcg);

	return 0;
4841 4842
}

4843
static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
4844
{
4845 4846 4847
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
	int ret;

4848 4849
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
4850 4851 4852 4853 4854
	css_get(&memcg->css);
	ret = mem_cgroup_force_empty(memcg);
	css_put(&memcg->css);

	return ret;
4855 4856 4857
}


4858 4859 4860 4861 4862 4863 4864 4865 4866
static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
{
	return mem_cgroup_from_cont(cont)->use_hierarchy;
}

static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
					u64 val)
{
	int retval = 0;
4867
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4868
	struct cgroup *parent = cont->parent;
4869
	struct mem_cgroup *parent_memcg = NULL;
4870 4871

	if (parent)
4872
		parent_memcg = mem_cgroup_from_cont(parent);
4873

4874
	mutex_lock(&memcg_create_mutex);
4875 4876 4877 4878

	if (memcg->use_hierarchy == val)
		goto out;

4879
	/*
4880
	 * If parent's use_hierarchy is set, we can't make any modifications
4881 4882 4883 4884 4885 4886
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
4887
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
4888
				(val == 1 || val == 0)) {
4889
		if (!__memcg_has_children(memcg))
4890
			memcg->use_hierarchy = val;
4891 4892 4893 4894
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
4895 4896

out:
4897
	mutex_unlock(&memcg_create_mutex);
4898 4899 4900 4901

	return retval;
}

4902

4903
static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
4904
					       enum mem_cgroup_stat_index idx)
4905
{
K
KAMEZAWA Hiroyuki 已提交
4906
	struct mem_cgroup *iter;
4907
	long val = 0;
4908

4909
	/* Per-cpu values can be negative, use a signed accumulator */
4910
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
4911 4912 4913 4914 4915
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
4916 4917
}

4918
static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
4919
{
K
KAMEZAWA Hiroyuki 已提交
4920
	u64 val;
4921

4922
	if (!mem_cgroup_is_root(memcg)) {
4923
		if (!swap)
4924
			return res_counter_read_u64(&memcg->res, RES_USAGE);
4925
		else
4926
			return res_counter_read_u64(&memcg->memsw, RES_USAGE);
4927 4928
	}

4929 4930
	val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
4931

K
KAMEZAWA Hiroyuki 已提交
4932
	if (swap)
4933
		val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
4934 4935 4936 4937

	return val << PAGE_SHIFT;
}

4938 4939 4940
static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
			       struct file *file, char __user *buf,
			       size_t nbytes, loff_t *ppos)
B
Balbir Singh 已提交
4941
{
4942
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4943
	char str[64];
4944
	u64 val;
G
Glauber Costa 已提交
4945 4946
	int name, len;
	enum res_type type;
4947 4948 4949

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
4950 4951 4952 4953

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

4954 4955
	switch (type) {
	case _MEM:
4956
		if (name == RES_USAGE)
4957
			val = mem_cgroup_usage(memcg, false);
4958
		else
4959
			val = res_counter_read_u64(&memcg->res, name);
4960 4961
		break;
	case _MEMSWAP:
4962
		if (name == RES_USAGE)
4963
			val = mem_cgroup_usage(memcg, true);
4964
		else
4965
			val = res_counter_read_u64(&memcg->memsw, name);
4966
		break;
4967 4968 4969
	case _KMEM:
		val = res_counter_read_u64(&memcg->kmem, name);
		break;
4970 4971 4972
	default:
		BUG();
	}
4973 4974 4975

	len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
	return simple_read_from_buffer(buf, nbytes, ppos, str, len);
B
Balbir Singh 已提交
4976
}
4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994

static int memcg_update_kmem_limit(struct cgroup *cont, u64 val)
{
	int ret = -EINVAL;
#ifdef CONFIG_MEMCG_KMEM
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
	/*
	 * For simplicity, we won't allow this to be disabled.  It also can't
	 * be changed if the cgroup has children already, or if tasks had
	 * already joined.
	 *
	 * If tasks join before we set the limit, a person looking at
	 * kmem.usage_in_bytes will have no way to determine when it took
	 * place, which makes the value quite meaningless.
	 *
	 * After it first became limited, changes in the value of the limit are
	 * of course permitted.
	 */
4995
	mutex_lock(&memcg_create_mutex);
4996 4997
	mutex_lock(&set_limit_mutex);
	if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
4998
		if (cgroup_task_count(cont) || memcg_has_children(memcg)) {
4999 5000 5001 5002 5003 5004
			ret = -EBUSY;
			goto out;
		}
		ret = res_counter_set_limit(&memcg->kmem, val);
		VM_BUG_ON(ret);

5005 5006 5007 5008 5009
		ret = memcg_update_cache_sizes(memcg);
		if (ret) {
			res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
			goto out;
		}
5010 5011 5012 5013 5014 5015 5016
		static_key_slow_inc(&memcg_kmem_enabled_key);
		/*
		 * setting the active bit after the inc will guarantee no one
		 * starts accounting before all call sites are patched
		 */
		memcg_kmem_set_active(memcg);

5017 5018 5019 5020 5021 5022 5023
		/*
		 * kmem charges can outlive the cgroup. In the case of slab
		 * pages, for instance, a page contain objects from various
		 * processes, so it is unfeasible to migrate them away. We
		 * need to reference count the memcg because of that.
		 */
		mem_cgroup_get(memcg);
5024 5025 5026 5027
	} else
		ret = res_counter_set_limit(&memcg->kmem, val);
out:
	mutex_unlock(&set_limit_mutex);
5028
	mutex_unlock(&memcg_create_mutex);
5029 5030 5031 5032
#endif
	return ret;
}

5033
#ifdef CONFIG_MEMCG_KMEM
5034
static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5035
{
5036
	int ret = 0;
5037 5038
	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
	if (!parent)
5039 5040
		goto out;

5041
	memcg->kmem_account_flags = parent->kmem_account_flags;
5042 5043 5044 5045 5046 5047 5048 5049 5050 5051
	/*
	 * When that happen, we need to disable the static branch only on those
	 * memcgs that enabled it. To achieve this, we would be forced to
	 * complicate the code by keeping track of which memcgs were the ones
	 * that actually enabled limits, and which ones got it from its
	 * parents.
	 *
	 * It is a lot simpler just to do static_key_slow_inc() on every child
	 * that is accounted.
	 */
5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068
	if (!memcg_kmem_is_active(memcg))
		goto out;

	/*
	 * destroy(), called if we fail, will issue static_key_slow_inc() and
	 * mem_cgroup_put() if kmem is enabled. We have to either call them
	 * unconditionally, or clear the KMEM_ACTIVE flag. I personally find
	 * this more consistent, since it always leads to the same destroy path
	 */
	mem_cgroup_get(memcg);
	static_key_slow_inc(&memcg_kmem_enabled_key);

	mutex_lock(&set_limit_mutex);
	ret = memcg_update_cache_sizes(memcg);
	mutex_unlock(&set_limit_mutex);
out:
	return ret;
5069
}
5070
#endif /* CONFIG_MEMCG_KMEM */
5071

5072 5073 5074 5075
/*
 * The user of this function is...
 * RES_LIMIT.
 */
5076 5077
static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
			    const char *buffer)
B
Balbir Singh 已提交
5078
{
5079
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
G
Glauber Costa 已提交
5080 5081
	enum res_type type;
	int name;
5082 5083 5084
	unsigned long long val;
	int ret;

5085 5086
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
5087 5088 5089 5090

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

5091
	switch (name) {
5092
	case RES_LIMIT:
5093 5094 5095 5096
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
5097 5098
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
5099 5100 5101
		if (ret)
			break;
		if (type == _MEM)
5102
			ret = mem_cgroup_resize_limit(memcg, val);
5103
		else if (type == _MEMSWAP)
5104
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
5105 5106 5107 5108
		else if (type == _KMEM)
			ret = memcg_update_kmem_limit(cont, val);
		else
			return -EINVAL;
5109
		break;
5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
5124 5125 5126 5127 5128
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
5129 5130
}

5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	struct cgroup *cgroup;
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	cgroup = memcg->css.cgroup;
	if (!memcg->use_hierarchy)
		goto out;

	while (cgroup->parent) {
		cgroup = cgroup->parent;
		memcg = mem_cgroup_from_cont(cgroup);
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
}

5158
static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
5159
{
5160
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
G
Glauber Costa 已提交
5161 5162
	int name;
	enum res_type type;
5163

5164 5165
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
5166 5167 5168 5169

	if (!do_swap_account && type == _MEMSWAP)
		return -EOPNOTSUPP;

5170
	switch (name) {
5171
	case RES_MAX_USAGE:
5172
		if (type == _MEM)
5173
			res_counter_reset_max(&memcg->res);
5174
		else if (type == _MEMSWAP)
5175
			res_counter_reset_max(&memcg->memsw);
5176 5177 5178 5179
		else if (type == _KMEM)
			res_counter_reset_max(&memcg->kmem);
		else
			return -EINVAL;
5180 5181
		break;
	case RES_FAILCNT:
5182
		if (type == _MEM)
5183
			res_counter_reset_failcnt(&memcg->res);
5184
		else if (type == _MEMSWAP)
5185
			res_counter_reset_failcnt(&memcg->memsw);
5186 5187 5188 5189
		else if (type == _KMEM)
			res_counter_reset_failcnt(&memcg->kmem);
		else
			return -EINVAL;
5190 5191
		break;
	}
5192

5193
	return 0;
5194 5195
}

5196 5197 5198 5199 5200 5201
static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
					struct cftype *cft)
{
	return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
}

5202
#ifdef CONFIG_MMU
5203 5204 5205
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
5206
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5207 5208 5209

	if (val >= (1 << NR_MOVE_TYPE))
		return -EINVAL;
5210

5211
	/*
5212 5213 5214 5215
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
5216
	 */
5217
	memcg->move_charge_at_immigrate = val;
5218 5219
	return 0;
}
5220 5221 5222 5223 5224 5225 5226
#else
static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
5227

5228
#ifdef CONFIG_NUMA
5229
static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
5230
				      struct seq_file *m)
5231 5232 5233 5234
{
	int nid;
	unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
	unsigned long node_nr;
5235
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5236

5237
	total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5238
	seq_printf(m, "total=%lu", total_nr);
5239
	for_each_node_state(nid, N_MEMORY) {
5240
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5241 5242 5243 5244
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5245
	file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5246
	seq_printf(m, "file=%lu", file_nr);
5247
	for_each_node_state(nid, N_MEMORY) {
5248
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5249
				LRU_ALL_FILE);
5250 5251 5252 5253
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5254
	anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5255
	seq_printf(m, "anon=%lu", anon_nr);
5256
	for_each_node_state(nid, N_MEMORY) {
5257
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5258
				LRU_ALL_ANON);
5259 5260 5261 5262
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5263
	unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5264
	seq_printf(m, "unevictable=%lu", unevictable_nr);
5265
	for_each_node_state(nid, N_MEMORY) {
5266
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5267
				BIT(LRU_UNEVICTABLE));
5268 5269 5270 5271 5272 5273 5274
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');
	return 0;
}
#endif /* CONFIG_NUMA */

5275 5276 5277 5278 5279
static inline void mem_cgroup_lru_names_not_uptodate(void)
{
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
}

5280
static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
5281
				 struct seq_file *m)
5282
{
5283
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5284 5285
	struct mem_cgroup *mi;
	unsigned int i;
5286

5287
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5288
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5289
			continue;
5290 5291
		seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5292
	}
L
Lee Schermerhorn 已提交
5293

5294 5295 5296 5297 5298 5299 5300 5301
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
			   mem_cgroup_read_events(memcg, i));

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
5302
	/* Hierarchical information */
5303 5304
	{
		unsigned long long limit, memsw_limit;
5305
		memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5306
		seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5307
		if (do_swap_account)
5308 5309
			seq_printf(m, "hierarchical_memsw_limit %llu\n",
				   memsw_limit);
5310
	}
K
KOSAKI Motohiro 已提交
5311

5312 5313 5314
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
		long long val = 0;

5315
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5316
			continue;
5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336
		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
		seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
	}

	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_events(mi, i);
		seq_printf(m, "total_%s %llu\n",
			   mem_cgroup_events_names[i], val);
	}

	for (i = 0; i < NR_LRU_LISTS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5337
	}
K
KAMEZAWA Hiroyuki 已提交
5338

K
KOSAKI Motohiro 已提交
5339 5340 5341 5342
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
5343
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
5344 5345 5346 5347 5348
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5349
				mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5350
				rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
5351

5352 5353 5354 5355
				recent_rotated[0] += rstat->recent_rotated[0];
				recent_rotated[1] += rstat->recent_rotated[1];
				recent_scanned[0] += rstat->recent_scanned[0];
				recent_scanned[1] += rstat->recent_scanned[1];
K
KOSAKI Motohiro 已提交
5356
			}
5357 5358 5359 5360
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
5361 5362 5363
	}
#endif

5364 5365 5366
	return 0;
}

K
KOSAKI Motohiro 已提交
5367 5368 5369 5370
static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);

5371
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
5372 5373 5374 5375 5376 5377 5378
}

static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
				       u64 val)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup *parent;
5379

K
KOSAKI Motohiro 已提交
5380 5381 5382 5383 5384 5385 5386
	if (val > 100)
		return -EINVAL;

	if (cgrp->parent == NULL)
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);
5387

5388
	mutex_lock(&memcg_create_mutex);
5389

K
KOSAKI Motohiro 已提交
5390
	/* If under hierarchy, only empty-root can set this value */
5391
	if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5392
		mutex_unlock(&memcg_create_mutex);
K
KOSAKI Motohiro 已提交
5393
		return -EINVAL;
5394
	}
K
KOSAKI Motohiro 已提交
5395 5396 5397

	memcg->swappiness = val;

5398
	mutex_unlock(&memcg_create_mutex);
5399

K
KOSAKI Motohiro 已提交
5400 5401 5402
	return 0;
}

5403 5404 5405 5406 5407 5408 5409 5410
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
	u64 usage;
	int i;

	rcu_read_lock();
	if (!swap)
5411
		t = rcu_dereference(memcg->thresholds.primary);
5412
	else
5413
		t = rcu_dereference(memcg->memsw_thresholds.primary);
5414 5415 5416 5417 5418 5419 5420

	if (!t)
		goto unlock;

	usage = mem_cgroup_usage(memcg, swap);

	/*
5421
	 * current_threshold points to threshold just below or equal to usage.
5422 5423 5424
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
5425
	i = t->current_threshold;
5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
5449
	t->current_threshold = i - 1;
5450 5451 5452 5453 5454 5455
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
5456 5457 5458 5459 5460 5461 5462
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
5463 5464 5465 5466 5467 5468 5469 5470 5471 5472
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

	return _a->threshold - _b->threshold;
}

5473
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
5474 5475 5476
{
	struct mem_cgroup_eventfd_list *ev;

5477
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
5478 5479 5480 5481
		eventfd_signal(ev->eventfd, 1);
	return 0;
}

5482
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
5483
{
K
KAMEZAWA Hiroyuki 已提交
5484 5485
	struct mem_cgroup *iter;

5486
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
5487
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
5488 5489 5490 5491
}

static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5492 5493
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5494 5495
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
G
Glauber Costa 已提交
5496
	enum res_type type = MEMFILE_TYPE(cft->private);
5497
	u64 threshold, usage;
5498
	int i, size, ret;
5499 5500 5501 5502 5503 5504

	ret = res_counter_memparse_write_strategy(args, &threshold);
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
5505

5506
	if (type == _MEM)
5507
		thresholds = &memcg->thresholds;
5508
	else if (type == _MEMSWAP)
5509
		thresholds = &memcg->memsw_thresholds;
5510 5511 5512 5513 5514 5515
	else
		BUG();

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before adding a new one */
5516
	if (thresholds->primary)
5517 5518
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

5519
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5520 5521

	/* Allocate memory for new array of thresholds */
5522
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
5523
			GFP_KERNEL);
5524
	if (!new) {
5525 5526 5527
		ret = -ENOMEM;
		goto unlock;
	}
5528
	new->size = size;
5529 5530

	/* Copy thresholds (if any) to new array */
5531 5532
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5533
				sizeof(struct mem_cgroup_threshold));
5534 5535
	}

5536
	/* Add new threshold */
5537 5538
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
5539 5540

	/* Sort thresholds. Registering of new threshold isn't time-critical */
5541
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5542 5543 5544
			compare_thresholds, NULL);

	/* Find current threshold */
5545
	new->current_threshold = -1;
5546
	for (i = 0; i < size; i++) {
5547
		if (new->entries[i].threshold <= usage) {
5548
			/*
5549 5550
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
5551 5552
			 * it here.
			 */
5553
			++new->current_threshold;
5554 5555
		} else
			break;
5556 5557
	}

5558 5559 5560 5561 5562
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
5563

5564
	/* To be sure that nobody uses thresholds */
5565 5566 5567 5568 5569 5570 5571 5572
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

5573
static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
5574
	struct cftype *cft, struct eventfd_ctx *eventfd)
5575 5576
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5577 5578
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
G
Glauber Costa 已提交
5579
	enum res_type type = MEMFILE_TYPE(cft->private);
5580
	u64 usage;
5581
	int i, j, size;
5582 5583 5584

	mutex_lock(&memcg->thresholds_lock);
	if (type == _MEM)
5585
		thresholds = &memcg->thresholds;
5586
	else if (type == _MEMSWAP)
5587
		thresholds = &memcg->memsw_thresholds;
5588 5589 5590
	else
		BUG();

5591 5592 5593
	if (!thresholds->primary)
		goto unlock;

5594 5595 5596 5597 5598 5599
	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
5600 5601 5602
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
5603 5604 5605
			size++;
	}

5606
	new = thresholds->spare;
5607

5608 5609
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
5610 5611
		kfree(new);
		new = NULL;
5612
		goto swap_buffers;
5613 5614
	}

5615
	new->size = size;
5616 5617

	/* Copy thresholds and find current threshold */
5618 5619 5620
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
5621 5622
			continue;

5623
		new->entries[j] = thresholds->primary->entries[i];
5624
		if (new->entries[j].threshold <= usage) {
5625
			/*
5626
			 * new->current_threshold will not be used
5627 5628 5629
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
5630
			++new->current_threshold;
5631 5632 5633 5634
		}
		j++;
	}

5635
swap_buffers:
5636 5637
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
5638 5639 5640 5641 5642 5643
	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}

5644
	rcu_assign_pointer(thresholds->primary, new);
5645

5646
	/* To be sure that nobody uses thresholds */
5647
	synchronize_rcu();
5648
unlock:
5649 5650
	mutex_unlock(&memcg->thresholds_lock);
}
5651

K
KAMEZAWA Hiroyuki 已提交
5652 5653 5654 5655 5656
static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *event;
G
Glauber Costa 已提交
5657
	enum res_type type = MEMFILE_TYPE(cft->private);
K
KAMEZAWA Hiroyuki 已提交
5658 5659 5660 5661 5662 5663

	BUG_ON(type != _OOM_TYPE);
	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

5664
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5665 5666 5667 5668 5669

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
5670
	if (atomic_read(&memcg->under_oom))
K
KAMEZAWA Hiroyuki 已提交
5671
		eventfd_signal(eventfd, 1);
5672
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5673 5674 5675 5676

	return 0;
}

5677
static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
5678 5679
	struct cftype *cft, struct eventfd_ctx *eventfd)
{
5680
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
K
KAMEZAWA Hiroyuki 已提交
5681
	struct mem_cgroup_eventfd_list *ev, *tmp;
G
Glauber Costa 已提交
5682
	enum res_type type = MEMFILE_TYPE(cft->private);
K
KAMEZAWA Hiroyuki 已提交
5683 5684 5685

	BUG_ON(type != _OOM_TYPE);

5686
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5687

5688
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
5689 5690 5691 5692 5693 5694
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

5695
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5696 5697
}

5698 5699 5700
static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
	struct cftype *cft,  struct cgroup_map_cb *cb)
{
5701
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5702

5703
	cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5704

5705
	if (atomic_read(&memcg->under_oom))
5706 5707 5708 5709 5710 5711 5712 5713 5714
		cb->fill(cb, "under_oom", 1);
	else
		cb->fill(cb, "under_oom", 0);
	return 0;
}

static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
	struct cftype *cft, u64 val)
{
5715
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5716 5717 5718 5719 5720 5721 5722 5723
	struct mem_cgroup *parent;

	/* cannot set to root cgroup and only 0 and 1 are allowed */
	if (!cgrp->parent || !((val == 0) || (val == 1)))
		return -EINVAL;

	parent = mem_cgroup_from_cont(cgrp->parent);

5724
	mutex_lock(&memcg_create_mutex);
5725
	/* oom-kill-disable is a flag for subhierarchy. */
5726
	if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5727
		mutex_unlock(&memcg_create_mutex);
5728 5729
		return -EINVAL;
	}
5730
	memcg->oom_kill_disable = val;
5731
	if (!val)
5732
		memcg_oom_recover(memcg);
5733
	mutex_unlock(&memcg_create_mutex);
5734 5735 5736
	return 0;
}

A
Andrew Morton 已提交
5737
#ifdef CONFIG_MEMCG_KMEM
5738
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5739
{
5740 5741
	int ret;

5742
	memcg->kmemcg_id = -1;
5743 5744 5745
	ret = memcg_propagate_kmem(memcg);
	if (ret)
		return ret;
5746

5747
	return mem_cgroup_sockets_init(memcg, ss);
5748 5749
};

5750
static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
5751
{
5752
	mem_cgroup_sockets_destroy(memcg);
5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766

	memcg_kmem_mark_dead(memcg);

	if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
		return;

	/*
	 * Charges already down to 0, undo mem_cgroup_get() done in the charge
	 * path here, being careful not to race with memcg_uncharge_kmem: it is
	 * possible that the charges went down to 0 between mark_dead and the
	 * res_counter read, so in that case, we don't need the put
	 */
	if (memcg_kmem_test_and_clear_dead(memcg))
		mem_cgroup_put(memcg);
G
Glauber Costa 已提交
5767
}
5768
#else
5769
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5770 5771 5772
{
	return 0;
}
G
Glauber Costa 已提交
5773

5774
static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
5775 5776
{
}
5777 5778
#endif

B
Balbir Singh 已提交
5779 5780
static struct cftype mem_cgroup_files[] = {
	{
5781
		.name = "usage_in_bytes",
5782
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
5783
		.read = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
5784 5785
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
B
Balbir Singh 已提交
5786
	},
5787 5788
	{
		.name = "max_usage_in_bytes",
5789
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
5790
		.trigger = mem_cgroup_reset,
5791
		.read = mem_cgroup_read,
5792
	},
B
Balbir Singh 已提交
5793
	{
5794
		.name = "limit_in_bytes",
5795
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
5796
		.write_string = mem_cgroup_write,
5797
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
5798
	},
5799 5800 5801 5802
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
5803
		.read = mem_cgroup_read,
5804
	},
B
Balbir Singh 已提交
5805 5806
	{
		.name = "failcnt",
5807
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
5808
		.trigger = mem_cgroup_reset,
5809
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
5810
	},
5811 5812
	{
		.name = "stat",
5813
		.read_seq_string = memcg_stat_show,
5814
	},
5815 5816 5817 5818
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
5819 5820 5821 5822 5823
	{
		.name = "use_hierarchy",
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
5824 5825 5826 5827 5828
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
5829 5830 5831 5832 5833
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
5834 5835
	{
		.name = "oom_control",
5836 5837
		.read_map = mem_cgroup_oom_control_read,
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
5838 5839 5840 5841
		.register_event = mem_cgroup_oom_register_event,
		.unregister_event = mem_cgroup_oom_unregister_event,
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
5842 5843 5844
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
5845
		.read_seq_string = memcg_numa_stat_show,
5846 5847
	},
#endif
5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871
#ifdef CONFIG_MEMCG_KMEM
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
5872 5873 5874 5875 5876 5877
#ifdef CONFIG_SLABINFO
	{
		.name = "kmem.slabinfo",
		.read_seq_string = mem_cgroup_slabinfo_read,
	},
#endif
5878
#endif
5879
	{ },	/* terminate */
5880
};
5881

5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911
#ifdef CONFIG_MEMCG_SWAP
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read = mem_cgroup_read,
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read = mem_cgroup_read,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{ },	/* terminate */
};
#endif
5912
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
5913 5914
{
	struct mem_cgroup_per_node *pn;
5915
	struct mem_cgroup_per_zone *mz;
5916
	int zone, tmp = node;
5917 5918 5919 5920 5921 5922 5923 5924
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
5925 5926
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
5927
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
5928 5929
	if (!pn)
		return 1;
5930 5931 5932

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
5933
		lruvec_init(&mz->lruvec);
5934
		mz->usage_in_excess = 0;
5935
		mz->on_tree = false;
5936
		mz->memcg = memcg;
5937
	}
5938
	memcg->info.nodeinfo[node] = pn;
5939 5940 5941
	return 0;
}

5942
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
5943
{
5944
	kfree(memcg->info.nodeinfo[node]);
5945 5946
}

5947 5948
static struct mem_cgroup *mem_cgroup_alloc(void)
{
5949
	struct mem_cgroup *memcg;
5950
	size_t size = memcg_size();
5951

5952
	/* Can be very big if nr_node_ids is very big */
5953
	if (size < PAGE_SIZE)
5954
		memcg = kzalloc(size, GFP_KERNEL);
5955
	else
5956
		memcg = vzalloc(size);
5957

5958
	if (!memcg)
5959 5960
		return NULL;

5961 5962
	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat)
5963
		goto out_free;
5964 5965
	spin_lock_init(&memcg->pcp_counter_lock);
	return memcg;
5966 5967 5968

out_free:
	if (size < PAGE_SIZE)
5969
		kfree(memcg);
5970
	else
5971
		vfree(memcg);
5972
	return NULL;
5973 5974
}

5975
/*
5976 5977 5978 5979 5980 5981 5982 5983
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
5984
 */
5985 5986

static void __mem_cgroup_free(struct mem_cgroup *memcg)
5987
{
5988
	int node;
5989
	size_t size = memcg_size();
5990

5991 5992 5993 5994 5995 5996 5997 5998
	mem_cgroup_remove_from_trees(memcg);
	free_css_id(&mem_cgroup_subsys, &memcg->css);

	for_each_node(node)
		free_mem_cgroup_per_zone_info(memcg, node);

	free_percpu(memcg->stat);

5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009
	/*
	 * We need to make sure that (at least for now), the jump label
	 * destruction code runs outside of the cgroup lock. This is because
	 * get_online_cpus(), which is called from the static_branch update,
	 * can't be called inside the cgroup_lock. cpusets are the ones
	 * enforcing this dependency, so if they ever change, we might as well.
	 *
	 * schedule_work() will guarantee this happens. Be careful if you need
	 * to move this code around, and make sure it is outside
	 * the cgroup_lock.
	 */
6010
	disarm_static_keys(memcg);
6011 6012 6013 6014
	if (size < PAGE_SIZE)
		kfree(memcg);
	else
		vfree(memcg);
6015
}
6016

6017

6018
/*
6019 6020 6021
 * Helpers for freeing a kmalloc()ed/vzalloc()ed mem_cgroup by RCU,
 * but in process context.  The work_freeing structure is overlaid
 * on the rcu_freeing structure, which itself is overlaid on memsw.
6022
 */
6023
static void free_work(struct work_struct *work)
6024
{
6025
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
6026

6027 6028 6029
	memcg = container_of(work, struct mem_cgroup, work_freeing);
	__mem_cgroup_free(memcg);
}
K
KAMEZAWA Hiroyuki 已提交
6030

6031 6032 6033
static void free_rcu(struct rcu_head *rcu_head)
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
6034

6035 6036 6037
	memcg = container_of(rcu_head, struct mem_cgroup, rcu_freeing);
	INIT_WORK(&memcg->work_freeing, free_work);
	schedule_work(&memcg->work_freeing);
6038 6039
}

6040
static void mem_cgroup_get(struct mem_cgroup *memcg)
6041
{
6042
	atomic_inc(&memcg->refcnt);
6043 6044
}

6045
static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
6046
{
6047 6048
	if (atomic_sub_and_test(count, &memcg->refcnt)) {
		struct mem_cgroup *parent = parent_mem_cgroup(memcg);
6049
		call_rcu(&memcg->rcu_freeing, free_rcu);
6050 6051 6052
		if (parent)
			mem_cgroup_put(parent);
	}
6053 6054
}

6055
static void mem_cgroup_put(struct mem_cgroup *memcg)
6056
{
6057
	__mem_cgroup_put(memcg, 1);
6058 6059
}

6060 6061 6062
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
G
Glauber Costa 已提交
6063
struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
6064
{
6065
	if (!memcg->res.parent)
6066
		return NULL;
6067
	return mem_cgroup_from_res_counter(memcg->res.parent, res);
6068
}
G
Glauber Costa 已提交
6069
EXPORT_SYMBOL(parent_mem_cgroup);
6070

6071
static void __init mem_cgroup_soft_limit_tree_init(void)
6072 6073 6074 6075 6076
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

B
Bob Liu 已提交
6077
	for_each_node(node) {
6078 6079 6080 6081
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6082
		BUG_ON(!rtpn);
6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
}

L
Li Zefan 已提交
6094
static struct cgroup_subsys_state * __ref
6095
mem_cgroup_css_alloc(struct cgroup *cont)
B
Balbir Singh 已提交
6096
{
6097
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
6098
	long error = -ENOMEM;
6099
	int node;
B
Balbir Singh 已提交
6100

6101 6102
	memcg = mem_cgroup_alloc();
	if (!memcg)
K
KAMEZAWA Hiroyuki 已提交
6103
		return ERR_PTR(error);
6104

B
Bob Liu 已提交
6105
	for_each_node(node)
6106
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
6107
			goto free_out;
6108

6109
	/* root ? */
6110
	if (cont->parent == NULL) {
6111
		root_mem_cgroup = memcg;
6112 6113 6114
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
		res_counter_init(&memcg->kmem, NULL);
6115
	}
6116

6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	atomic_set(&memcg->refcnt, 1);
	memcg->move_charge_at_immigrate = 0;
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);

	return &memcg->css;

free_out:
	__mem_cgroup_free(memcg);
	return ERR_PTR(error);
}

static int
mem_cgroup_css_online(struct cgroup *cont)
{
	struct mem_cgroup *memcg, *parent;
	int error = 0;

	if (!cont->parent)
		return 0;

6140
	mutex_lock(&memcg_create_mutex);
6141 6142 6143 6144 6145 6146 6147 6148
	memcg = mem_cgroup_from_cont(cont);
	parent = mem_cgroup_from_cont(cont->parent);

	memcg->use_hierarchy = parent->use_hierarchy;
	memcg->oom_kill_disable = parent->oom_kill_disable;
	memcg->swappiness = mem_cgroup_swappiness(parent);

	if (parent->use_hierarchy) {
6149 6150
		res_counter_init(&memcg->res, &parent->res);
		res_counter_init(&memcg->memsw, &parent->memsw);
6151
		res_counter_init(&memcg->kmem, &parent->kmem);
6152

6153 6154 6155 6156 6157 6158 6159
		/*
		 * We increment refcnt of the parent to ensure that we can
		 * safely access it on res_counter_charge/uncharge.
		 * This refcnt will be decremented when freeing this
		 * mem_cgroup(see mem_cgroup_put).
		 */
		mem_cgroup_get(parent);
6160
	} else {
6161 6162
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
6163
		res_counter_init(&memcg->kmem, NULL);
6164 6165 6166 6167 6168
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
6169
		if (parent != root_mem_cgroup)
6170
			mem_cgroup_subsys.broken_hierarchy = true;
6171
	}
6172 6173

	error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
6174
	mutex_unlock(&memcg_create_mutex);
6175 6176 6177 6178 6179 6180 6181
	if (error) {
		/*
		 * We call put now because our (and parent's) refcnts
		 * are already in place. mem_cgroup_put() will internally
		 * call __mem_cgroup_free, so return directly
		 */
		mem_cgroup_put(memcg);
6182 6183
		if (parent->use_hierarchy)
			mem_cgroup_put(parent);
6184
	}
6185
	return error;
B
Balbir Singh 已提交
6186 6187
}

6188
static void mem_cgroup_css_offline(struct cgroup *cont)
6189
{
6190
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
6191

6192
	mem_cgroup_reparent_charges(memcg);
G
Glauber Costa 已提交
6193
	mem_cgroup_destroy_all_caches(memcg);
6194 6195
}

6196
static void mem_cgroup_css_free(struct cgroup *cont)
B
Balbir Singh 已提交
6197
{
6198
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
6199

6200
	kmem_cgroup_destroy(memcg);
G
Glauber Costa 已提交
6201

6202
	mem_cgroup_put(memcg);
B
Balbir Singh 已提交
6203 6204
}

6205
#ifdef CONFIG_MMU
6206
/* Handlers for move charge at task migration. */
6207 6208
#define PRECHARGE_COUNT_AT_ONCE	256
static int mem_cgroup_do_precharge(unsigned long count)
6209
{
6210 6211
	int ret = 0;
	int batch_count = PRECHARGE_COUNT_AT_ONCE;
6212
	struct mem_cgroup *memcg = mc.to;
6213

6214
	if (mem_cgroup_is_root(memcg)) {
6215 6216 6217 6218 6219 6220 6221 6222
		mc.precharge += count;
		/* we don't need css_get for root */
		return ret;
	}
	/* try to charge at once */
	if (count > 1) {
		struct res_counter *dummy;
		/*
6223
		 * "memcg" cannot be under rmdir() because we've already checked
6224 6225 6226 6227
		 * by cgroup_lock_live_cgroup() that it is not removed and we
		 * are still under the same cgroup_mutex. So we can postpone
		 * css_get().
		 */
6228
		if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
6229
			goto one_by_one;
6230
		if (do_swap_account && res_counter_charge(&memcg->memsw,
6231
						PAGE_SIZE * count, &dummy)) {
6232
			res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248
			goto one_by_one;
		}
		mc.precharge += count;
		return ret;
	}
one_by_one:
	/* fall back to one by one charge */
	while (count--) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		if (!batch_count--) {
			batch_count = PRECHARGE_COUNT_AT_ONCE;
			cond_resched();
		}
6249 6250
		ret = __mem_cgroup_try_charge(NULL,
					GFP_KERNEL, 1, &memcg, false);
6251
		if (ret)
6252
			/* mem_cgroup_clear_mc() will do uncharge later */
6253
			return ret;
6254 6255
		mc.precharge++;
	}
6256 6257 6258 6259
	return ret;
}

/**
6260
 * get_mctgt_type - get target type of moving charge
6261 6262 6263
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
6264
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
6265 6266 6267 6268 6269 6270
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
6271 6272 6273
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
6274 6275 6276 6277 6278
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
6279
	swp_entry_t	ent;
6280 6281 6282
};

enum mc_target_type {
6283
	MC_TARGET_NONE = 0,
6284
	MC_TARGET_PAGE,
6285
	MC_TARGET_SWAP,
6286 6287
};

D
Daisuke Nishimura 已提交
6288 6289
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
6290
{
D
Daisuke Nishimura 已提交
6291
	struct page *page = vm_normal_page(vma, addr, ptent);
6292

D
Daisuke Nishimura 已提交
6293 6294 6295 6296
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
		/* we don't move shared anon */
6297
		if (!move_anon())
D
Daisuke Nishimura 已提交
6298
			return NULL;
6299 6300
	} else if (!move_file())
		/* we ignore mapcount for file pages */
D
Daisuke Nishimura 已提交
6301 6302 6303 6304 6305 6306 6307
		return NULL;
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

6308
#ifdef CONFIG_SWAP
D
Daisuke Nishimura 已提交
6309 6310 6311 6312 6313 6314 6315 6316
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

	if (!move_anon() || non_swap_entry(ent))
		return NULL;
6317 6318 6319 6320
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
6321
	page = find_get_page(swap_address_space(ent), ent.val);
D
Daisuke Nishimura 已提交
6322 6323 6324 6325 6326
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}
6327 6328 6329 6330 6331 6332 6333
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
6334

6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
	if (!move_file())
		return NULL;

	mapping = vma->vm_file->f_mapping;
	if (pte_none(ptent))
		pgoff = linear_page_index(vma, addr);
	else /* pte_file(ptent) is true */
		pgoff = pte_to_pgoff(ptent);

	/* page is moved even if it's not RSS of this task(page-faulted). */
6354 6355 6356 6357 6358 6359
	page = find_get_page(mapping, pgoff);

#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
	if (radix_tree_exceptional_entry(page)) {
		swp_entry_t swap = radix_to_swp_entry(page);
6360
		if (do_swap_account)
6361
			*entry = swap;
6362
		page = find_get_page(swap_address_space(swap), swap.val);
6363
	}
6364
#endif
6365 6366 6367
	return page;
}

6368
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
6369 6370 6371 6372
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
6373
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
6374 6375 6376 6377 6378 6379
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
6380 6381
	else if (pte_none(ptent) || pte_file(ptent))
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
6382 6383

	if (!page && !ent.val)
6384
		return ret;
6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399
	if (page) {
		pc = lookup_page_cgroup(page);
		/*
		 * Do only loose check w/o page_cgroup lock.
		 * mem_cgroup_move_account() checks the pc is valid or not under
		 * the lock.
		 */
		if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
6400 6401
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
6402
			css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
6403 6404 6405
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
6406 6407 6408 6409
	}
	return ret;
}

6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
 * We don't consider swapping or file mapped pages because THP does not
 * support them for now.
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
	enum mc_target_type ret = MC_TARGET_NONE;

	page = pmd_page(pmd);
	VM_BUG_ON(!page || !PageHead(page));
	if (!move_anon())
		return ret;
	pc = lookup_page_cgroup(page);
	if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

6445 6446 6447 6448 6449 6450 6451 6452
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

6453 6454 6455 6456
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
		spin_unlock(&vma->vm_mm->page_table_lock);
6457
		return 0;
6458
	}
6459

6460 6461
	if (pmd_trans_unstable(pmd))
		return 0;
6462 6463
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
6464
		if (get_mctgt_type(vma, addr, *pte, NULL))
6465 6466 6467 6468
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

6469 6470 6471
	return 0;
}

6472 6473 6474 6475 6476
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;
	struct vm_area_struct *vma;

6477
	down_read(&mm->mmap_sem);
6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		struct mm_walk mem_cgroup_count_precharge_walk = {
			.pmd_entry = mem_cgroup_count_precharge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		walk_page_range(vma->vm_start, vma->vm_end,
					&mem_cgroup_count_precharge_walk);
	}
6489
	up_read(&mm->mmap_sem);
6490 6491 6492 6493 6494 6495 6496 6497 6498

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
6499 6500 6501 6502 6503
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
6504 6505
}

6506 6507
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
6508
{
6509 6510 6511
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;

6512
	/* we must uncharge all the leftover precharges from mc.to */
6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523
	if (mc.precharge) {
		__mem_cgroup_cancel_charge(mc.to, mc.precharge);
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
		mc.moved_charge = 0;
6524
	}
6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
		if (!mem_cgroup_is_root(mc.from))
			res_counter_uncharge(&mc.from->memsw,
						PAGE_SIZE * mc.moved_swap);
		__mem_cgroup_put(mc.from, mc.moved_swap);

		if (!mem_cgroup_is_root(mc.to)) {
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			res_counter_uncharge(&mc.to->res,
						PAGE_SIZE * mc.moved_swap);
		}
		/* we've already done mem_cgroup_get(mc.to) */
		mc.moved_swap = 0;
	}
6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	struct mem_cgroup *from = mc.from;

	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
6559
	spin_lock(&mc.lock);
6560 6561
	mc.from = NULL;
	mc.to = NULL;
6562
	spin_unlock(&mc.lock);
6563
	mem_cgroup_end_move(from);
6564 6565
}

6566 6567
static int mem_cgroup_can_attach(struct cgroup *cgroup,
				 struct cgroup_taskset *tset)
6568
{
6569
	struct task_struct *p = cgroup_taskset_first(tset);
6570
	int ret = 0;
6571
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
6572
	unsigned long move_charge_at_immigrate;
6573

6574 6575 6576 6577 6578 6579 6580
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
	move_charge_at_immigrate  = memcg->move_charge_at_immigrate;
	if (move_charge_at_immigrate) {
6581 6582 6583
		struct mm_struct *mm;
		struct mem_cgroup *from = mem_cgroup_from_task(p);

6584
		VM_BUG_ON(from == memcg);
6585 6586 6587 6588 6589

		mm = get_task_mm(p);
		if (!mm)
			return 0;
		/* We move charges only when we move a owner of the mm */
6590 6591 6592 6593
		if (mm->owner == p) {
			VM_BUG_ON(mc.from);
			VM_BUG_ON(mc.to);
			VM_BUG_ON(mc.precharge);
6594
			VM_BUG_ON(mc.moved_charge);
6595
			VM_BUG_ON(mc.moved_swap);
6596
			mem_cgroup_start_move(from);
6597
			spin_lock(&mc.lock);
6598
			mc.from = from;
6599
			mc.to = memcg;
6600
			mc.immigrate_flags = move_charge_at_immigrate;
6601
			spin_unlock(&mc.lock);
6602
			/* We set mc.moving_task later */
6603 6604 6605 6606

			ret = mem_cgroup_precharge_mc(mm);
			if (ret)
				mem_cgroup_clear_mc();
6607 6608
		}
		mmput(mm);
6609 6610 6611 6612
	}
	return ret;
}

6613 6614
static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
				     struct cgroup_taskset *tset)
6615
{
6616
	mem_cgroup_clear_mc();
6617 6618
}

6619 6620 6621
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
6622
{
6623 6624 6625 6626
	int ret = 0;
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;
6627 6628 6629 6630
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
	struct page_cgroup *pc;
6631

6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642
	/*
	 * We don't take compound_lock() here but no race with splitting thp
	 * happens because:
	 *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
	 *    under splitting, which means there's no concurrent thp split,
	 *  - if another thread runs into split_huge_page() just after we
	 *    entered this if-block, the thread must wait for page table lock
	 *    to be unlocked in __split_huge_page_splitting(), where the main
	 *    part of thp split is not executed yet.
	 */
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
6643
		if (mc.precharge < HPAGE_PMD_NR) {
6644 6645 6646 6647 6648 6649 6650 6651 6652
			spin_unlock(&vma->vm_mm->page_table_lock);
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
				pc = lookup_page_cgroup(page);
				if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
6653
							pc, mc.from, mc.to)) {
6654 6655 6656 6657 6658 6659 6660 6661
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
		}
		spin_unlock(&vma->vm_mm->page_table_lock);
6662
		return 0;
6663 6664
	}

6665 6666
	if (pmd_trans_unstable(pmd))
		return 0;
6667 6668 6669 6670
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
6671
		swp_entry_t ent;
6672 6673 6674 6675

		if (!mc.precharge)
			break;

6676
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
6677 6678 6679 6680 6681
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
			pc = lookup_page_cgroup(page);
6682
			if (!mem_cgroup_move_account(page, 1, pc,
6683
						     mc.from, mc.to)) {
6684
				mc.precharge--;
6685 6686
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
6687 6688
			}
			putback_lru_page(page);
6689
put:			/* get_mctgt_type() gets the page */
6690 6691
			put_page(page);
			break;
6692 6693
		case MC_TARGET_SWAP:
			ent = target.ent;
6694
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
6695
				mc.precharge--;
6696 6697 6698
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
6699
			break;
6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
6714
		ret = mem_cgroup_do_precharge(1);
6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	lru_add_drain_all();
6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		int ret;
		struct mm_walk mem_cgroup_move_charge_walk = {
			.pmd_entry = mem_cgroup_move_charge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		ret = walk_page_range(vma->vm_start, vma->vm_end,
						&mem_cgroup_move_charge_walk);
		if (ret)
			/*
			 * means we have consumed all precharges and failed in
			 * doing additional charge. Just abandon here.
			 */
			break;
	}
6758
	up_read(&mm->mmap_sem);
6759 6760
}

6761 6762
static void mem_cgroup_move_task(struct cgroup *cont,
				 struct cgroup_taskset *tset)
B
Balbir Singh 已提交
6763
{
6764
	struct task_struct *p = cgroup_taskset_first(tset);
6765
	struct mm_struct *mm = get_task_mm(p);
6766 6767

	if (mm) {
6768 6769
		if (mc.to)
			mem_cgroup_move_charge(mm);
6770 6771
		mmput(mm);
	}
6772 6773
	if (mc.to)
		mem_cgroup_clear_mc();
B
Balbir Singh 已提交
6774
}
6775
#else	/* !CONFIG_MMU */
6776 6777
static int mem_cgroup_can_attach(struct cgroup *cgroup,
				 struct cgroup_taskset *tset)
6778 6779 6780
{
	return 0;
}
6781 6782
static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
				     struct cgroup_taskset *tset)
6783 6784
{
}
6785 6786
static void mem_cgroup_move_task(struct cgroup *cont,
				 struct cgroup_taskset *tset)
6787 6788 6789
{
}
#endif
B
Balbir Singh 已提交
6790

B
Balbir Singh 已提交
6791 6792 6793
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
6794
	.css_alloc = mem_cgroup_css_alloc,
6795
	.css_online = mem_cgroup_css_online,
6796 6797
	.css_offline = mem_cgroup_css_offline,
	.css_free = mem_cgroup_css_free,
6798 6799
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
6800
	.attach = mem_cgroup_move_task,
6801
	.base_cftypes = mem_cgroup_files,
6802
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
6803
	.use_id = 1,
B
Balbir Singh 已提交
6804
};
6805

A
Andrew Morton 已提交
6806
#ifdef CONFIG_MEMCG_SWAP
6807 6808 6809
static int __init enable_swap_account(char *s)
{
	/* consider enabled if no parameter or 1 is given */
6810
	if (!strcmp(s, "1"))
6811
		really_do_swap_account = 1;
6812
	else if (!strcmp(s, "0"))
6813 6814 6815
		really_do_swap_account = 0;
	return 1;
}
6816
__setup("swapaccount=", enable_swap_account);
6817

6818 6819
static void __init memsw_file_init(void)
{
6820 6821 6822 6823 6824 6825 6826 6827 6828
	WARN_ON(cgroup_add_cftypes(&mem_cgroup_subsys, memsw_cgroup_files));
}

static void __init enable_swap_cgroup(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
		memsw_file_init();
	}
6829
}
6830

6831
#else
6832
static void __init enable_swap_cgroup(void)
6833 6834
{
}
6835
#endif
6836 6837

/*
6838 6839 6840 6841 6842 6843
 * subsys_initcall() for memory controller.
 *
 * Some parts like hotcpu_notifier() have to be initialized from this context
 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
 * everything that doesn't depend on a specific mem_cgroup structure should
 * be initialized from here.
6844 6845 6846 6847
 */
static int __init mem_cgroup_init(void)
{
	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
6848
	enable_swap_cgroup();
6849
	mem_cgroup_soft_limit_tree_init();
6850
	memcg_stock_init();
6851 6852 6853
	return 0;
}
subsys_initcall(mem_cgroup_init);