memcontrol.c 185.7 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.c - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9 10 11 12
 * Memory thresholds
 * Copyright (C) 2009 Nokia Corporation
 * Author: Kirill A. Shutemov
 *
13 14 15 16
 * Kernel Memory Controller
 * Copyright (C) 2012 Parallels Inc. and Google Inc.
 * Authors: Glauber Costa and Suleiman Souhlal
 *
B
Balbir Singh 已提交
17 18 19 20 21 22 23 24 25 26 27 28 29 30
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
31
#include <linux/mm.h>
32
#include <linux/hugetlb.h>
K
KAMEZAWA Hiroyuki 已提交
33
#include <linux/pagemap.h>
34
#include <linux/smp.h>
35
#include <linux/page-flags.h>
36
#include <linux/backing-dev.h>
37 38
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
39
#include <linux/limits.h>
40
#include <linux/export.h>
41
#include <linux/mutex.h>
42
#include <linux/rbtree.h>
43
#include <linux/slab.h>
44
#include <linux/swap.h>
45
#include <linux/swapops.h>
46
#include <linux/spinlock.h>
47 48
#include <linux/eventfd.h>
#include <linux/sort.h>
49
#include <linux/fs.h>
50
#include <linux/seq_file.h>
51
#include <linux/vmalloc.h>
52
#include <linux/vmpressure.h>
53
#include <linux/mm_inline.h>
54
#include <linux/page_cgroup.h>
55
#include <linux/cpu.h>
56
#include <linux/oom.h>
K
KAMEZAWA Hiroyuki 已提交
57
#include "internal.h"
G
Glauber Costa 已提交
58
#include <net/sock.h>
M
Michal Hocko 已提交
59
#include <net/ip.h>
G
Glauber Costa 已提交
60
#include <net/tcp_memcontrol.h>
B
Balbir Singh 已提交
61

62 63
#include <asm/uaccess.h>

64 65
#include <trace/events/vmscan.h>

66
struct cgroup_subsys mem_cgroup_subsys __read_mostly;
67 68
EXPORT_SYMBOL(mem_cgroup_subsys);

69
#define MEM_CGROUP_RECLAIM_RETRIES	5
70
static struct mem_cgroup *root_mem_cgroup __read_mostly;
B
Balbir Singh 已提交
71

A
Andrew Morton 已提交
72
#ifdef CONFIG_MEMCG_SWAP
L
Li Zefan 已提交
73
/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
74
int do_swap_account __read_mostly;
75 76

/* for remember boot option*/
A
Andrew Morton 已提交
77
#ifdef CONFIG_MEMCG_SWAP_ENABLED
78 79 80 81 82
static int really_do_swap_account __initdata = 1;
#else
static int really_do_swap_account __initdata = 0;
#endif

83
#else
84
#define do_swap_account		0
85 86 87
#endif


88 89 90 91 92 93 94
/*
 * Statistics for memory cgroup.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
95 96 97 98 99
	MEM_CGROUP_STAT_CACHE,		/* # of pages charged as cache */
	MEM_CGROUP_STAT_RSS,		/* # of pages charged as anon rss */
	MEM_CGROUP_STAT_RSS_HUGE,	/* # of pages charged as anon huge */
	MEM_CGROUP_STAT_FILE_MAPPED,	/* # of pages charged as file rss */
	MEM_CGROUP_STAT_SWAP,		/* # of pages, swapped out */
100 101 102
	MEM_CGROUP_STAT_NSTATS,
};

103 104 105
static const char * const mem_cgroup_stat_names[] = {
	"cache",
	"rss",
106
	"rss_huge",
107 108 109 110
	"mapped_file",
	"swap",
};

111 112 113
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
114 115
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
116 117
	MEM_CGROUP_EVENTS_NSTATS,
};
118 119 120 121 122 123 124 125

static const char * const mem_cgroup_events_names[] = {
	"pgpgin",
	"pgpgout",
	"pgfault",
	"pgmajfault",
};

126 127 128 129 130 131 132 133
static const char * const mem_cgroup_lru_names[] = {
	"inactive_anon",
	"active_anon",
	"inactive_file",
	"active_file",
	"unevictable",
};

134 135 136 137 138 139 140 141 142
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
143
	MEM_CGROUP_TARGET_NUMAINFO,
144 145
	MEM_CGROUP_NTARGETS,
};
146 147 148
#define THRESHOLDS_EVENTS_TARGET 128
#define SOFTLIMIT_EVENTS_TARGET 1024
#define NUMAINFO_EVENTS_TARGET	1024
149

150
struct mem_cgroup_stat_cpu {
151
	long count[MEM_CGROUP_STAT_NSTATS];
152
	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
153
	unsigned long nr_page_events;
154
	unsigned long targets[MEM_CGROUP_NTARGETS];
155 156
};

157
struct mem_cgroup_reclaim_iter {
M
Michal Hocko 已提交
158 159 160 161
	/*
	 * last scanned hierarchy member. Valid only if last_dead_count
	 * matches memcg->dead_count of the hierarchy root group.
	 */
162
	struct mem_cgroup *last_visited;
M
Michal Hocko 已提交
163 164
	unsigned long last_dead_count;

165 166 167 168
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

169 170 171 172
/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
173
	struct lruvec		lruvec;
174
	unsigned long		lru_size[NR_LRU_LISTS];
K
KOSAKI Motohiro 已提交
175

176 177
	struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];

178 179 180 181
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long long	usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
182
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
183
						/* use container_of	   */
184 185 186 187 188 189
};

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
/*
 * Cgroups above their limits are maintained in a RB-Tree, independent of
 * their hierarchy representation
 */

struct mem_cgroup_tree_per_zone {
	struct rb_root rb_root;
	spinlock_t lock;
};

struct mem_cgroup_tree_per_node {
	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
};

struct mem_cgroup_tree {
	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
};

static struct mem_cgroup_tree soft_limit_tree __read_mostly;

210 211 212 213 214
struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	u64 threshold;
};

K
KAMEZAWA Hiroyuki 已提交
215
/* For threshold */
216
struct mem_cgroup_threshold_ary {
217
	/* An array index points to threshold just below or equal to usage. */
218
	int current_threshold;
219 220 221 222 223
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};
224 225 226 227 228 229 230 231 232 233 234 235

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

K
KAMEZAWA Hiroyuki 已提交
236 237 238 239 240
/* for OOM */
struct mem_cgroup_eventfd_list {
	struct list_head list;
	struct eventfd_ctx *eventfd;
};
241

242 243
static void mem_cgroup_threshold(struct mem_cgroup *memcg);
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
244

B
Balbir Singh 已提交
245 246 247 248 249 250 251
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 *
 * TODO: Add a water mark for the memory controller. Reclaim will begin when
252 253 254
 * we hit the water mark. May be even add a low water mark, such that
 * no reclaim occurs from a cgroup at it's low water mark, this is
 * a feature that will be implemented much later in the future.
B
Balbir Singh 已提交
255 256 257 258 259 260 261
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;
	/*
	 * the counter to account for memory usage
	 */
	struct res_counter res;
262

263 264 265
	/* vmpressure notifications */
	struct vmpressure vmpressure;

266 267 268 269
	/*
	 * the counter to account for mem+swap usage.
	 */
	struct res_counter memsw;
270

271 272 273 274
	/*
	 * the counter to account for kernel memory usage.
	 */
	struct res_counter kmem;
275 276 277 278
	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;
279
	unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
280 281 282 283

	bool		oom_lock;
	atomic_t	under_oom;

284
	int	swappiness;
285 286
	/* OOM-Killer disable */
	int		oom_kill_disable;
K
KOSAKI Motohiro 已提交
287

288 289 290
	/* set when res.limit == memsw.limit */
	bool		memsw_is_minimum;

291 292 293 294
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
295
	struct mem_cgroup_thresholds thresholds;
296

297
	/* thresholds for mem+swap usage. RCU-protected */
298
	struct mem_cgroup_thresholds memsw_thresholds;
299

K
KAMEZAWA Hiroyuki 已提交
300 301
	/* For oom notifier event fd */
	struct list_head oom_notify;
302

303 304 305 306 307
	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long 	move_charge_at_immigrate;
308 309 310 311
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t	moving_account;
312 313
	/* taken only while moving_account > 0 */
	spinlock_t	move_lock;
314
	/*
315
	 * percpu counter.
316
	 */
317
	struct mem_cgroup_stat_cpu __percpu *stat;
318 319 320 321 322 323
	/*
	 * used when a cpu is offlined or other synchronizations
	 * See mem_cgroup_read_stat().
	 */
	struct mem_cgroup_stat_cpu nocpu_base;
	spinlock_t pcp_counter_lock;
G
Glauber Costa 已提交
324

M
Michal Hocko 已提交
325
	atomic_t	dead_count;
M
Michal Hocko 已提交
326
#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
G
Glauber Costa 已提交
327 328
	struct tcp_memcontrol tcp_mem;
#endif
329 330 331 332 333 334 335 336
#if defined(CONFIG_MEMCG_KMEM)
	/* analogous to slab_common's slab_caches list. per-memcg */
	struct list_head memcg_slab_caches;
	/* Not a spinlock, we can take a lot of time walking the list */
	struct mutex slab_caches_mutex;
        /* Index in the kmem_cache->memcg_params->memcg_caches array */
	int kmemcg_id;
#endif
337 338 339 340 341 342 343

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif
344

345 346
	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
B
Balbir Singh 已提交
347 348
};

349 350 351 352 353 354
static size_t memcg_size(void)
{
	return sizeof(struct mem_cgroup) +
		nr_node_ids * sizeof(struct mem_cgroup_per_node);
}

355 356 357
/* internal only representation about the status of kmem accounting. */
enum {
	KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
358
	KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
359
	KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
360 361
};

362 363 364
/* We account when limit is on, but only after call sites are patched */
#define KMEM_ACCOUNTED_MASK \
		((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
365 366 367 368 369 370

#ifdef CONFIG_MEMCG_KMEM
static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
{
	set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
}
371 372 373 374 375 376

static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
{
	return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
}

377 378 379 380 381
static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
{
	set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
}

382 383 384 385 386
static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
{
	clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
}

387 388
static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
{
389 390 391 392 393
	/*
	 * Our caller must use css_get() first, because memcg_uncharge_kmem()
	 * will call css_put() if it sees the memcg is dead.
	 */
	smp_wmb();
394 395 396 397 398 399 400 401 402
	if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
		set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
}

static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
{
	return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
				  &memcg->kmem_account_flags);
}
403 404
#endif

405 406
/* Stuffs for move charges at task migration. */
/*
407 408
 * Types of charges to be moved. "move_charge_at_immitgrate" and
 * "immigrate_flags" are treated as a left-shifted bitmap of these types.
409 410
 */
enum move_type {
411
	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */
412
	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */
413 414 415
	NR_MOVE_TYPE,
};

416 417
/* "mc" and its members are protected by cgroup_mutex */
static struct move_charge_struct {
418
	spinlock_t	  lock; /* for from, to */
419 420
	struct mem_cgroup *from;
	struct mem_cgroup *to;
421
	unsigned long immigrate_flags;
422
	unsigned long precharge;
423
	unsigned long moved_charge;
424
	unsigned long moved_swap;
425 426 427
	struct task_struct *moving_task;	/* a task moving charges */
	wait_queue_head_t waitq;		/* a waitq for other context */
} mc = {
428
	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
429 430
	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
};
431

D
Daisuke Nishimura 已提交
432 433
static bool move_anon(void)
{
434
	return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
D
Daisuke Nishimura 已提交
435 436
}

437 438
static bool move_file(void)
{
439
	return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
440 441
}

442 443 444 445
/*
 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
 * limit reclaim to prevent infinite loops, if they ever occur.
 */
446 447
#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
448

449 450
enum charge_type {
	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
451
	MEM_CGROUP_CHARGE_TYPE_ANON,
K
KAMEZAWA Hiroyuki 已提交
452
	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */
K
KAMEZAWA Hiroyuki 已提交
453
	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */
454 455 456
	NR_CHARGE_TYPE,
};

457
/* for encoding cft->private value on file */
G
Glauber Costa 已提交
458 459 460 461
enum res_type {
	_MEM,
	_MEMSWAP,
	_OOM_TYPE,
462
	_KMEM,
G
Glauber Costa 已提交
463 464
};

465 466
#define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))
#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)
467
#define MEMFILE_ATTR(val)	((val) & 0xffff)
K
KAMEZAWA Hiroyuki 已提交
468 469
/* Used for OOM nofiier */
#define OOM_CONTROL		(0)
470

471 472 473 474 475 476 477 478
/*
 * Reclaim flags for mem_cgroup_hierarchical_reclaim
 */
#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)

479 480 481 482 483 484 485
/*
 * The memcg_create_mutex will be held whenever a new cgroup is created.
 * As a consequence, any change that needs to protect against new child cgroups
 * appearing has to hold it as well.
 */
static DEFINE_MUTEX(memcg_create_mutex);

486 487
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
{
488
	return s ? container_of(s, struct mem_cgroup, css) : NULL;
489 490
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
/* Some nice accessors for the vmpressure. */
struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
{
	if (!memcg)
		memcg = root_mem_cgroup;
	return &memcg->vmpressure;
}

struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
{
	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}

struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css)
{
	return &mem_cgroup_from_css(css)->vmpressure;
}

509 510 511 512 513
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

G
Glauber Costa 已提交
514
/* Writing them here to avoid exposing memcg's inner layout */
M
Michal Hocko 已提交
515
#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
G
Glauber Costa 已提交
516 517 518

void sock_update_memcg(struct sock *sk)
{
519
	if (mem_cgroup_sockets_enabled) {
G
Glauber Costa 已提交
520
		struct mem_cgroup *memcg;
521
		struct cg_proto *cg_proto;
G
Glauber Costa 已提交
522 523 524

		BUG_ON(!sk->sk_prot->proto_cgroup);

525 526 527 528 529 530 531 532 533 534
		/* Socket cloning can throw us here with sk_cgrp already
		 * filled. It won't however, necessarily happen from
		 * process context. So the test for root memcg given
		 * the current task's memcg won't help us in this case.
		 *
		 * Respecting the original socket's memcg is a better
		 * decision in this case.
		 */
		if (sk->sk_cgrp) {
			BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
535
			css_get(&sk->sk_cgrp->memcg->css);
536 537 538
			return;
		}

G
Glauber Costa 已提交
539 540
		rcu_read_lock();
		memcg = mem_cgroup_from_task(current);
541
		cg_proto = sk->sk_prot->proto_cgroup(memcg);
542 543
		if (!mem_cgroup_is_root(memcg) &&
		    memcg_proto_active(cg_proto) && css_tryget(&memcg->css)) {
544
			sk->sk_cgrp = cg_proto;
G
Glauber Costa 已提交
545 546 547 548 549 550 551 552
		}
		rcu_read_unlock();
	}
}
EXPORT_SYMBOL(sock_update_memcg);

void sock_release_memcg(struct sock *sk)
{
553
	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
G
Glauber Costa 已提交
554 555 556
		struct mem_cgroup *memcg;
		WARN_ON(!sk->sk_cgrp->memcg);
		memcg = sk->sk_cgrp->memcg;
557
		css_put(&sk->sk_cgrp->memcg->css);
G
Glauber Costa 已提交
558 559
	}
}
G
Glauber Costa 已提交
560 561 562 563 564 565 566 567 568

struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg || mem_cgroup_is_root(memcg))
		return NULL;

	return &memcg->tcp_mem.cg_proto;
}
EXPORT_SYMBOL(tcp_proto_cgroup);
G
Glauber Costa 已提交
569

570 571 572 573 574 575 576 577 578 579 580 581
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
	if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
		return;
	static_key_slow_dec(&memcg_socket_limit_enabled);
}
#else
static void disarm_sock_keys(struct mem_cgroup *memcg)
{
}
#endif

582
#ifdef CONFIG_MEMCG_KMEM
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
/*
 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
 * There are two main reasons for not using the css_id for this:
 *  1) this works better in sparse environments, where we have a lot of memcgs,
 *     but only a few kmem-limited. Or also, if we have, for instance, 200
 *     memcgs, and none but the 200th is kmem-limited, we'd have to have a
 *     200 entry array for that.
 *
 *  2) In order not to violate the cgroup API, we would like to do all memory
 *     allocation in ->create(). At that point, we haven't yet allocated the
 *     css_id. Having a separate index prevents us from messing with the cgroup
 *     core for this
 *
 * The current size of the caches array is stored in
 * memcg_limited_groups_array_size.  It will double each time we have to
 * increase it.
 */
static DEFINE_IDA(kmem_limited_groups);
601 602
int memcg_limited_groups_array_size;

603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
/*
 * MIN_SIZE is different than 1, because we would like to avoid going through
 * the alloc/free process all the time. In a small machine, 4 kmem-limited
 * cgroups is a reasonable guess. In the future, it could be a parameter or
 * tunable, but that is strictly not necessary.
 *
 * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
 * this constant directly from cgroup, but it is understandable that this is
 * better kept as an internal representation in cgroup.c. In any case, the
 * css_id space is not getting any smaller, and we don't have to necessarily
 * increase ours as well if it increases.
 */
#define MEMCG_CACHES_MIN_SIZE 4
#define MEMCG_CACHES_MAX_SIZE 65535

618 619 620 621 622 623
/*
 * A lot of the calls to the cache allocation functions are expected to be
 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
 * conditional to this static branch, we'll have to allow modules that does
 * kmem_cache_alloc and the such to see this symbol as well
 */
624
struct static_key memcg_kmem_enabled_key;
625
EXPORT_SYMBOL(memcg_kmem_enabled_key);
626 627 628

static void disarm_kmem_keys(struct mem_cgroup *memcg)
{
629
	if (memcg_kmem_is_active(memcg)) {
630
		static_key_slow_dec(&memcg_kmem_enabled_key);
631 632
		ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
	}
633 634 635 636 637
	/*
	 * This check can't live in kmem destruction function,
	 * since the charges will outlive the cgroup
	 */
	WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
638 639 640 641 642 643 644 645 646 647 648 649 650
}
#else
static void disarm_kmem_keys(struct mem_cgroup *memcg)
{
}
#endif /* CONFIG_MEMCG_KMEM */

static void disarm_static_keys(struct mem_cgroup *memcg)
{
	disarm_sock_keys(memcg);
	disarm_kmem_keys(memcg);
}

651
static void drain_all_stock_async(struct mem_cgroup *memcg);
652

653
static struct mem_cgroup_per_zone *
654
mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
655
{
656
	VM_BUG_ON((unsigned)nid >= nr_node_ids);
657
	return &memcg->nodeinfo[nid]->zoneinfo[zid];
658 659
}

660
struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
661
{
662
	return &memcg->css;
663 664
}

665
static struct mem_cgroup_per_zone *
666
page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
667
{
668 669
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
670

671
	return mem_cgroup_zoneinfo(memcg, nid, zid);
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_node_zone(int nid, int zid)
{
	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static struct mem_cgroup_tree_per_zone *
soft_limit_tree_from_page(struct page *page)
{
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);

	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
}

static void
690
__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
691
				struct mem_cgroup_per_zone *mz,
692 693
				struct mem_cgroup_tree_per_zone *mctz,
				unsigned long long new_usage_in_excess)
694 695 696 697 698 699 700 701
{
	struct rb_node **p = &mctz->rb_root.rb_node;
	struct rb_node *parent = NULL;
	struct mem_cgroup_per_zone *mz_node;

	if (mz->on_tree)
		return;

702 703 704
	mz->usage_in_excess = new_usage_in_excess;
	if (!mz->usage_in_excess)
		return;
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
	while (*p) {
		parent = *p;
		mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
					tree_node);
		if (mz->usage_in_excess < mz_node->usage_in_excess)
			p = &(*p)->rb_left;
		/*
		 * We can't avoid mem cgroups that are over their soft
		 * limit by the same amount
		 */
		else if (mz->usage_in_excess >= mz_node->usage_in_excess)
			p = &(*p)->rb_right;
	}
	rb_link_node(&mz->tree_node, parent, p);
	rb_insert_color(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = true;
721 722 723
}

static void
724
__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
725 726 727 728 729 730 731 732 733
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	if (!mz->on_tree)
		return;
	rb_erase(&mz->tree_node, &mctz->rb_root);
	mz->on_tree = false;
}

734
static void
735
mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
736 737 738 739
				struct mem_cgroup_per_zone *mz,
				struct mem_cgroup_tree_per_zone *mctz)
{
	spin_lock(&mctz->lock);
740
	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
741 742 743 744
	spin_unlock(&mctz->lock);
}


745
static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
746
{
747
	unsigned long long excess;
748 749
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;
750 751
	int nid = page_to_nid(page);
	int zid = page_zonenum(page);
752 753 754
	mctz = soft_limit_tree_from_page(page);

	/*
755 756
	 * Necessary to update all ancestors when hierarchy is used.
	 * because their event counter is not touched.
757
	 */
758 759 760
	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
		mz = mem_cgroup_zoneinfo(memcg, nid, zid);
		excess = res_counter_soft_limit_excess(&memcg->res);
761 762 763 764
		/*
		 * We have to update the tree if mz is on RB-tree or
		 * mem is over its softlimit.
		 */
765
		if (excess || mz->on_tree) {
766 767 768
			spin_lock(&mctz->lock);
			/* if on-tree, remove it */
			if (mz->on_tree)
769
				__mem_cgroup_remove_exceeded(memcg, mz, mctz);
770
			/*
771 772
			 * Insert again. mz->usage_in_excess will be updated.
			 * If excess is 0, no tree ops.
773
			 */
774
			__mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
775 776
			spin_unlock(&mctz->lock);
		}
777 778 779
	}
}

780
static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
781 782 783 784 785
{
	int node, zone;
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup_tree_per_zone *mctz;

B
Bob Liu 已提交
786
	for_each_node(node) {
787
		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
788
			mz = mem_cgroup_zoneinfo(memcg, node, zone);
789
			mctz = soft_limit_tree_node_zone(node, zone);
790
			mem_cgroup_remove_exceeded(memcg, mz, mctz);
791 792 793 794
		}
	}
}

795 796 797 798
static struct mem_cgroup_per_zone *
__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct rb_node *rightmost = NULL;
799
	struct mem_cgroup_per_zone *mz;
800 801

retry:
802
	mz = NULL;
803 804 805 806 807 808 809 810 811 812
	rightmost = rb_last(&mctz->rb_root);
	if (!rightmost)
		goto done;		/* Nothing to reclaim from */

	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
	/*
	 * Remove the node now but someone else can add it back,
	 * we will to add it back at the end of reclaim to its correct
	 * position in the tree.
	 */
813 814 815
	__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
	if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
		!css_tryget(&mz->memcg->css))
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
		goto retry;
done:
	return mz;
}

static struct mem_cgroup_per_zone *
mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
{
	struct mem_cgroup_per_zone *mz;

	spin_lock(&mctz->lock);
	mz = __mem_cgroup_largest_soft_limit_node(mctz);
	spin_unlock(&mctz->lock);
	return mz;
}

832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
/*
 * Implementation Note: reading percpu statistics for memcg.
 *
 * Both of vmstat[] and percpu_counter has threshold and do periodic
 * synchronization to implement "quick" read. There are trade-off between
 * reading cost and precision of value. Then, we may have a chance to implement
 * a periodic synchronizion of counter in memcg's counter.
 *
 * But this _read() function is used for user interface now. The user accounts
 * memory usage by memory cgroup and he _always_ requires exact value because
 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
 * have to visit all online cpus and make sum. So, for now, unnecessary
 * synchronization is not implemented. (just implemented for cpu hotplug)
 *
 * If there are kernel internal actions which can make use of some not-exact
 * value, and reading all cpu value can be performance bottleneck in some
 * common workload, threashold and synchonization as vmstat[] should be
 * implemented.
 */
851
static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
852
				 enum mem_cgroup_stat_index idx)
853
{
854
	long val = 0;
855 856
	int cpu;

857 858
	get_online_cpus();
	for_each_online_cpu(cpu)
859
		val += per_cpu(memcg->stat->count[idx], cpu);
860
#ifdef CONFIG_HOTPLUG_CPU
861 862 863
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.count[idx];
	spin_unlock(&memcg->pcp_counter_lock);
864 865
#endif
	put_online_cpus();
866 867 868
	return val;
}

869
static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
870 871 872
					 bool charge)
{
	int val = (charge) ? 1 : -1;
873
	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
874 875
}

876
static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
877 878 879 880 881 882
					    enum mem_cgroup_events_index idx)
{
	unsigned long val = 0;
	int cpu;

	for_each_online_cpu(cpu)
883
		val += per_cpu(memcg->stat->events[idx], cpu);
884
#ifdef CONFIG_HOTPLUG_CPU
885 886 887
	spin_lock(&memcg->pcp_counter_lock);
	val += memcg->nocpu_base.events[idx];
	spin_unlock(&memcg->pcp_counter_lock);
888 889 890 891
#endif
	return val;
}

892
static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
893
					 struct page *page,
894
					 bool anon, int nr_pages)
895
{
896 897
	preempt_disable();

898 899 900 901 902 903
	/*
	 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
	 * counted as CACHE even if it's on ANON LRU.
	 */
	if (anon)
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
904
				nr_pages);
905
	else
906
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
907
				nr_pages);
908

909 910 911 912
	if (PageTransHuge(page))
		__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
				nr_pages);

913 914
	/* pagein of a big page is an event. So, ignore page size */
	if (nr_pages > 0)
915
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
916
	else {
917
		__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
918 919
		nr_pages = -nr_pages; /* for event */
	}
920

921
	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
922

923
	preempt_enable();
924 925
}

926
unsigned long
927
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
928 929 930 931 932 933 934 935
{
	struct mem_cgroup_per_zone *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	return mz->lru_size[lru];
}

static unsigned long
936
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
937
			unsigned int lru_mask)
938 939
{
	struct mem_cgroup_per_zone *mz;
H
Hugh Dickins 已提交
940
	enum lru_list lru;
941 942
	unsigned long ret = 0;

943
	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
944

H
Hugh Dickins 已提交
945 946 947
	for_each_lru(lru) {
		if (BIT(lru) & lru_mask)
			ret += mz->lru_size[lru];
948 949 950 951 952
	}
	return ret;
}

static unsigned long
953
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
954 955
			int nid, unsigned int lru_mask)
{
956 957 958
	u64 total = 0;
	int zid;

959
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
960 961
		total += mem_cgroup_zone_nr_lru_pages(memcg,
						nid, zid, lru_mask);
962

963 964
	return total;
}
965

966
static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
967
			unsigned int lru_mask)
968
{
969
	int nid;
970 971
	u64 total = 0;

972
	for_each_node_state(nid, N_MEMORY)
973
		total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
974
	return total;
975 976
}

977 978
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
				       enum mem_cgroup_events_target target)
979 980 981
{
	unsigned long val, next;

982
	val = __this_cpu_read(memcg->stat->nr_page_events);
983
	next = __this_cpu_read(memcg->stat->targets[target]);
984
	/* from time_after() in jiffies.h */
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
	if ((long)next - (long)val < 0) {
		switch (target) {
		case MEM_CGROUP_TARGET_THRESH:
			next = val + THRESHOLDS_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_SOFTLIMIT:
			next = val + SOFTLIMIT_EVENTS_TARGET;
			break;
		case MEM_CGROUP_TARGET_NUMAINFO:
			next = val + NUMAINFO_EVENTS_TARGET;
			break;
		default:
			break;
		}
		__this_cpu_write(memcg->stat->targets[target], next);
		return true;
1001
	}
1002
	return false;
1003 1004 1005 1006 1007 1008
}

/*
 * Check events in order.
 *
 */
1009
static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1010
{
1011
	preempt_disable();
1012
	/* threshold event is triggered in finer grain than soft limit */
1013 1014
	if (unlikely(mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_THRESH))) {
1015 1016
		bool do_softlimit;
		bool do_numainfo __maybe_unused;
1017 1018 1019 1020 1021 1022 1023 1024 1025

		do_softlimit = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_SOFTLIMIT);
#if MAX_NUMNODES > 1
		do_numainfo = mem_cgroup_event_ratelimit(memcg,
						MEM_CGROUP_TARGET_NUMAINFO);
#endif
		preempt_enable();

1026
		mem_cgroup_threshold(memcg);
1027
		if (unlikely(do_softlimit))
1028
			mem_cgroup_update_tree(memcg, page);
1029
#if MAX_NUMNODES > 1
1030
		if (unlikely(do_numainfo))
1031
			atomic_inc(&memcg->numainfo_events);
1032
#endif
1033 1034
	} else
		preempt_enable();
1035 1036
}

1037
static inline struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
B
Balbir Singh 已提交
1038
{
1039
	return mem_cgroup_from_css(cgroup_css(cont, mem_cgroup_subsys_id));
B
Balbir Singh 已提交
1040 1041
}

1042
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1043
{
1044 1045 1046 1047 1048 1049 1050 1051
	/*
	 * mm_update_next_owner() may clear mm->owner to NULL
	 * if it races with swapoff, page migration, etc.
	 * So this can be called with p == NULL.
	 */
	if (unlikely(!p))
		return NULL;

1052
	return mem_cgroup_from_css(task_css(p, mem_cgroup_subsys_id));
1053 1054
}

1055
struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
1056
{
1057
	struct mem_cgroup *memcg = NULL;
1058 1059 1060

	if (!mm)
		return NULL;
1061 1062 1063 1064 1065 1066 1067
	/*
	 * Because we have no locks, mm->owner's may be being moved to other
	 * cgroup. We use css_tryget() here even if this looks
	 * pessimistic (rather than adding locks here).
	 */
	rcu_read_lock();
	do {
1068 1069
		memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
		if (unlikely(!memcg))
1070
			break;
1071
	} while (!css_tryget(&memcg->css));
1072
	rcu_read_unlock();
1073
	return memcg;
1074 1075
}

1076 1077 1078 1079 1080 1081 1082 1083 1084
/*
 * Returns a next (in a pre-order walk) alive memcg (with elevated css
 * ref. count) or NULL if the whole root's subtree has been visited.
 *
 * helper function to be used by mem_cgroup_iter
 */
static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
		struct mem_cgroup *last_visited)
{
1085
	struct cgroup_subsys_state *prev_css, *next_css;
1086 1087 1088 1089 1090 1091 1092 1093

	/*
	 * Root is not visited by cgroup iterators so it needs an
	 * explicit visit.
	 */
	if (!last_visited)
		return root;

1094
	prev_css = (last_visited == root) ? NULL : &last_visited->css;
1095
skip_node:
1096
	next_css = css_next_descendant_pre(prev_css, &root->css);
1097 1098 1099 1100 1101 1102 1103 1104

	/*
	 * Even if we found a group we have to make sure it is
	 * alive. css && !memcg means that the groups should be
	 * skipped and we should continue the tree walk.
	 * last_visited css is safe to use because it is
	 * protected by css_get and the tree walk is rcu safe.
	 */
1105 1106 1107
	if (next_css) {
		struct mem_cgroup *mem = mem_cgroup_from_css(next_css);

1108 1109 1110
		if (css_tryget(&mem->css))
			return mem;
		else {
1111
			prev_css = next_css;
1112 1113 1114 1115 1116 1117 1118
			goto skip_node;
		}
	}

	return NULL;
}

1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
static void mem_cgroup_iter_invalidate(struct mem_cgroup *root)
{
	/*
	 * When a group in the hierarchy below root is destroyed, the
	 * hierarchy iterator can no longer be trusted since it might
	 * have pointed to the destroyed group.  Invalidate it.
	 */
	atomic_inc(&root->dead_count);
}

static struct mem_cgroup *
mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter,
		     struct mem_cgroup *root,
		     int *sequence)
{
	struct mem_cgroup *position = NULL;
	/*
	 * A cgroup destruction happens in two stages: offlining and
	 * release.  They are separated by a RCU grace period.
	 *
	 * If the iterator is valid, we may still race with an
	 * offlining.  The RCU lock ensures the object won't be
	 * released, tryget will fail if we lost the race.
	 */
	*sequence = atomic_read(&root->dead_count);
	if (iter->last_dead_count == *sequence) {
		smp_rmb();
		position = iter->last_visited;
		if (position && !css_tryget(&position->css))
			position = NULL;
	}
	return position;
}

static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
				   struct mem_cgroup *last_visited,
				   struct mem_cgroup *new_position,
				   int sequence)
{
	if (last_visited)
		css_put(&last_visited->css);
	/*
	 * We store the sequence count from the time @last_visited was
	 * loaded successfully instead of rereading it here so that we
	 * don't lose destruction events in between.  We could have
	 * raced with the destruction of @new_position after all.
	 */
	iter->last_visited = new_position;
	smp_wmb();
	iter->last_dead_count = sequence;
}

1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
/**
 * mem_cgroup_iter - iterate over memory cgroup hierarchy
 * @root: hierarchy root
 * @prev: previously returned memcg, NULL on first invocation
 * @reclaim: cookie for shared reclaim walks, NULL for full walks
 *
 * Returns references to children of the hierarchy below @root, or
 * @root itself, or %NULL after a full round-trip.
 *
 * Caller must pass the return value in @prev on subsequent
 * invocations for reference counting, or use mem_cgroup_iter_break()
 * to cancel a hierarchy walk before the round-trip is complete.
 *
 * Reclaimers can specify a zone and a priority level in @reclaim to
 * divide up the memcgs in the hierarchy among all concurrent
 * reclaimers operating on the same zone and priority.
 */
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
				   struct mem_cgroup *prev,
				   struct mem_cgroup_reclaim_cookie *reclaim)
K
KAMEZAWA Hiroyuki 已提交
1191
{
1192
	struct mem_cgroup *memcg = NULL;
1193
	struct mem_cgroup *last_visited = NULL;
1194

1195 1196 1197
	if (mem_cgroup_disabled())
		return NULL;

1198 1199
	if (!root)
		root = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
1200

1201
	if (prev && !reclaim)
1202
		last_visited = prev;
K
KAMEZAWA Hiroyuki 已提交
1203

1204 1205
	if (!root->use_hierarchy && root != root_mem_cgroup) {
		if (prev)
1206
			goto out_css_put;
1207 1208
		return root;
	}
K
KAMEZAWA Hiroyuki 已提交
1209

1210
	rcu_read_lock();
1211
	while (!memcg) {
1212
		struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1213
		int uninitialized_var(seq);
1214

1215 1216 1217 1218 1219 1220 1221
		if (reclaim) {
			int nid = zone_to_nid(reclaim->zone);
			int zid = zone_idx(reclaim->zone);
			struct mem_cgroup_per_zone *mz;

			mz = mem_cgroup_zoneinfo(root, nid, zid);
			iter = &mz->reclaim_iter[reclaim->priority];
1222
			if (prev && reclaim->generation != iter->generation) {
M
Michal Hocko 已提交
1223
				iter->last_visited = NULL;
1224 1225
				goto out_unlock;
			}
M
Michal Hocko 已提交
1226

1227
			last_visited = mem_cgroup_iter_load(iter, root, &seq);
1228
		}
K
KAMEZAWA Hiroyuki 已提交
1229

1230
		memcg = __mem_cgroup_iter_next(root, last_visited);
K
KAMEZAWA Hiroyuki 已提交
1231

1232
		if (reclaim) {
1233
			mem_cgroup_iter_update(iter, last_visited, memcg, seq);
1234

M
Michal Hocko 已提交
1235
			if (!memcg)
1236 1237 1238 1239
				iter->generation++;
			else if (!prev && memcg)
				reclaim->generation = iter->generation;
		}
1240

M
Michal Hocko 已提交
1241
		if (prev && !memcg)
1242
			goto out_unlock;
1243
	}
1244 1245
out_unlock:
	rcu_read_unlock();
1246 1247 1248 1249
out_css_put:
	if (prev && prev != root)
		css_put(&prev->css);

1250
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
1251
}
K
KAMEZAWA Hiroyuki 已提交
1252

1253 1254 1255 1256 1257 1258 1259
/**
 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
 * @root: hierarchy root
 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
 */
void mem_cgroup_iter_break(struct mem_cgroup *root,
			   struct mem_cgroup *prev)
1260 1261 1262 1263 1264 1265
{
	if (!root)
		root = root_mem_cgroup;
	if (prev && prev != root)
		css_put(&prev->css);
}
K
KAMEZAWA Hiroyuki 已提交
1266

1267 1268 1269 1270 1271 1272
/*
 * Iteration constructs for visiting all cgroups (under a tree).  If
 * loops are exited prematurely (break), mem_cgroup_iter_break() must
 * be used for reference counting.
 */
#define for_each_mem_cgroup_tree(iter, root)		\
1273
	for (iter = mem_cgroup_iter(root, NULL, NULL);	\
1274
	     iter != NULL;				\
1275
	     iter = mem_cgroup_iter(root, iter, NULL))
1276

1277
#define for_each_mem_cgroup(iter)			\
1278
	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
1279
	     iter != NULL;				\
1280
	     iter = mem_cgroup_iter(NULL, iter, NULL))
K
KAMEZAWA Hiroyuki 已提交
1281

1282
void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1283
{
1284
	struct mem_cgroup *memcg;
1285 1286

	rcu_read_lock();
1287 1288
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
1289 1290 1291 1292
		goto out;

	switch (idx) {
	case PGFAULT:
1293 1294 1295 1296
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
		break;
	case PGMAJFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1297 1298 1299 1300 1301 1302 1303
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
}
1304
EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1305

1306 1307 1308
/**
 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
 * @zone: zone of the wanted lruvec
1309
 * @memcg: memcg of the wanted lruvec
1310 1311 1312 1313 1314 1315 1316 1317 1318
 *
 * Returns the lru list vector holding pages for the given @zone and
 * @mem.  This can be the global zone lruvec, if the memory controller
 * is disabled.
 */
struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
				      struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_zone *mz;
1319
	struct lruvec *lruvec;
1320

1321 1322 1323 1324
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1325 1326

	mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
1337 1338
}

K
KAMEZAWA Hiroyuki 已提交
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
/*
 * Following LRU functions are allowed to be used without PCG_LOCK.
 * Operations are called by routine of global LRU independently from memcg.
 * What we have to take care of here is validness of pc->mem_cgroup.
 *
 * Changes to pc->mem_cgroup happens when
 * 1. charge
 * 2. moving account
 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
 * It is added to LRU before charge.
 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
 * When moving account, the page is not on LRU. It's isolated.
 */
1352

1353
/**
1354
 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1355
 * @page: the page
1356
 * @zone: zone of the page
1357
 */
1358
struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
K
KAMEZAWA Hiroyuki 已提交
1359 1360
{
	struct mem_cgroup_per_zone *mz;
1361 1362
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;
1363
	struct lruvec *lruvec;
1364

1365 1366 1367 1368
	if (mem_cgroup_disabled()) {
		lruvec = &zone->lruvec;
		goto out;
	}
1369

K
KAMEZAWA Hiroyuki 已提交
1370
	pc = lookup_page_cgroup(page);
1371
	memcg = pc->mem_cgroup;
1372 1373

	/*
1374
	 * Surreptitiously switch any uncharged offlist page to root:
1375 1376 1377 1378 1379 1380 1381
	 * an uncharged page off lru does nothing to secure
	 * its former mem_cgroup from sudden removal.
	 *
	 * Our caller holds lru_lock, and PageCgroupUsed is updated
	 * under page_cgroup lock: between them, they make all uses
	 * of pc->mem_cgroup safe.
	 */
1382
	if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1383 1384
		pc->mem_cgroup = memcg = root_mem_cgroup;

1385
	mz = page_cgroup_zoneinfo(memcg, page);
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
	 * we have to be prepared to initialize lruvec->zone here;
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
	if (unlikely(lruvec->zone != zone))
		lruvec->zone = zone;
	return lruvec;
K
KAMEZAWA Hiroyuki 已提交
1396
}
1397

1398
/**
1399 1400 1401 1402
 * mem_cgroup_update_lru_size - account for adding or removing an lru page
 * @lruvec: mem_cgroup per zone lru vector
 * @lru: index of lru list the page is sitting on
 * @nr_pages: positive when adding or negative when removing
1403
 *
1404 1405
 * This function must be called when a page is added to or removed from an
 * lru list.
1406
 */
1407 1408
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
				int nr_pages)
1409 1410
{
	struct mem_cgroup_per_zone *mz;
1411
	unsigned long *lru_size;
1412 1413 1414 1415

	if (mem_cgroup_disabled())
		return;

1416 1417 1418 1419
	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	lru_size = mz->lru_size + lru;
	*lru_size += nr_pages;
	VM_BUG_ON((long)(*lru_size) < 0);
K
KAMEZAWA Hiroyuki 已提交
1420
}
1421

1422
/*
1423
 * Checks whether given mem is same or in the root_mem_cgroup's
1424 1425
 * hierarchy subtree
 */
1426 1427
bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				  struct mem_cgroup *memcg)
1428
{
1429 1430
	if (root_memcg == memcg)
		return true;
1431
	if (!root_memcg->use_hierarchy || !memcg)
1432
		return false;
1433 1434 1435 1436 1437 1438 1439 1440
	return css_is_ancestor(&memcg->css, &root_memcg->css);
}

static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
				       struct mem_cgroup *memcg)
{
	bool ret;

1441
	rcu_read_lock();
1442
	ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1443 1444
	rcu_read_unlock();
	return ret;
1445 1446
}

1447 1448
bool task_in_mem_cgroup(struct task_struct *task,
			const struct mem_cgroup *memcg)
1449
{
1450
	struct mem_cgroup *curr = NULL;
1451
	struct task_struct *p;
1452
	bool ret;
1453

1454
	p = find_lock_task_mm(task);
1455 1456 1457 1458 1459 1460 1461 1462 1463
	if (p) {
		curr = try_get_mem_cgroup_from_mm(p->mm);
		task_unlock(p);
	} else {
		/*
		 * All threads may have already detached their mm's, but the oom
		 * killer still needs to detect if they have already been oom
		 * killed to prevent needlessly killing additional tasks.
		 */
1464
		rcu_read_lock();
1465 1466 1467
		curr = mem_cgroup_from_task(task);
		if (curr)
			css_get(&curr->css);
1468
		rcu_read_unlock();
1469
	}
1470
	if (!curr)
1471
		return false;
1472
	/*
1473
	 * We should check use_hierarchy of "memcg" not "curr". Because checking
1474
	 * use_hierarchy of "curr" here make this function true if hierarchy is
1475 1476
	 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
	 * hierarchy(even if use_hierarchy is disabled in "memcg").
1477
	 */
1478
	ret = mem_cgroup_same_or_subtree(memcg, curr);
1479
	css_put(&curr->css);
1480 1481 1482
	return ret;
}

1483
int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1484
{
1485
	unsigned long inactive_ratio;
1486
	unsigned long inactive;
1487
	unsigned long active;
1488
	unsigned long gb;
1489

1490 1491
	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1492

1493 1494 1495 1496 1497 1498
	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

1499
	return inactive * inactive_ratio < active;
1500 1501
}

1502 1503 1504
#define mem_cgroup_from_res_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

1505
/**
1506
 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
W
Wanpeng Li 已提交
1507
 * @memcg: the memory cgroup
1508
 *
1509
 * Returns the maximum amount of memory @mem can be charged with, in
1510
 * pages.
1511
 */
1512
static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1513
{
1514 1515
	unsigned long long margin;

1516
	margin = res_counter_margin(&memcg->res);
1517
	if (do_swap_account)
1518
		margin = min(margin, res_counter_margin(&memcg->memsw));
1519
	return margin >> PAGE_SHIFT;
1520 1521
}

1522
int mem_cgroup_swappiness(struct mem_cgroup *memcg)
K
KOSAKI Motohiro 已提交
1523 1524
{
	/* root ? */
T
Tejun Heo 已提交
1525
	if (!css_parent(&memcg->css))
K
KOSAKI Motohiro 已提交
1526 1527
		return vm_swappiness;

1528
	return memcg->swappiness;
K
KOSAKI Motohiro 已提交
1529 1530
}

1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
/*
 * memcg->moving_account is used for checking possibility that some thread is
 * calling move_account(). When a thread on CPU-A starts moving pages under
 * a memcg, other threads should check memcg->moving_account under
 * rcu_read_lock(), like this:
 *
 *         CPU-A                                    CPU-B
 *                                              rcu_read_lock()
 *         memcg->moving_account+1              if (memcg->mocing_account)
 *                                                   take heavy locks.
 *         synchronize_rcu()                    update something.
 *                                              rcu_read_unlock()
 *         start move here.
 */
1545 1546 1547 1548

/* for quick checking without looking up memcg */
atomic_t memcg_moving __read_mostly;

1549
static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1550
{
1551
	atomic_inc(&memcg_moving);
1552
	atomic_inc(&memcg->moving_account);
1553 1554 1555
	synchronize_rcu();
}

1556
static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1557
{
1558 1559 1560 1561
	/*
	 * Now, mem_cgroup_clear_mc() may call this function with NULL.
	 * We check NULL in callee rather than caller.
	 */
1562 1563
	if (memcg) {
		atomic_dec(&memcg_moving);
1564
		atomic_dec(&memcg->moving_account);
1565
	}
1566
}
1567

1568 1569 1570
/*
 * 2 routines for checking "mem" is under move_account() or not.
 *
1571 1572
 * mem_cgroup_stolen() -  checking whether a cgroup is mc.from or not. This
 *			  is used for avoiding races in accounting.  If true,
1573 1574 1575 1576 1577 1578 1579
 *			  pc->mem_cgroup may be overwritten.
 *
 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
 *			  under hierarchy of moving cgroups. This is for
 *			  waiting at hith-memory prressure caused by "move".
 */

1580
static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1581 1582
{
	VM_BUG_ON(!rcu_read_lock_held());
1583
	return atomic_read(&memcg->moving_account) > 0;
1584
}
1585

1586
static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1587
{
1588 1589
	struct mem_cgroup *from;
	struct mem_cgroup *to;
1590
	bool ret = false;
1591 1592 1593 1594 1595 1596 1597 1598 1599
	/*
	 * Unlike task_move routines, we access mc.to, mc.from not under
	 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
	 */
	spin_lock(&mc.lock);
	from = mc.from;
	to = mc.to;
	if (!from)
		goto unlock;
1600

1601 1602
	ret = mem_cgroup_same_or_subtree(memcg, from)
		|| mem_cgroup_same_or_subtree(memcg, to);
1603 1604
unlock:
	spin_unlock(&mc.lock);
1605 1606 1607
	return ret;
}

1608
static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1609 1610
{
	if (mc.moving_task && current != mc.moving_task) {
1611
		if (mem_cgroup_under_move(memcg)) {
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
			DEFINE_WAIT(wait);
			prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
			/* moving charge context might have finished. */
			if (mc.moving_task)
				schedule();
			finish_wait(&mc.waitq, &wait);
			return true;
		}
	}
	return false;
}

1624 1625 1626 1627
/*
 * Take this lock when
 * - a code tries to modify page's memcg while it's USED.
 * - a code tries to modify page state accounting in a memcg.
1628
 * see mem_cgroup_stolen(), too.
1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
 */
static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
				  unsigned long *flags)
{
	spin_lock_irqsave(&memcg->move_lock, *flags);
}

static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
				unsigned long *flags)
{
	spin_unlock_irqrestore(&memcg->move_lock, *flags);
}

1642
#define K(x) ((x) << (PAGE_SHIFT-10))
1643
/**
1644
 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
 * @memcg: The memory cgroup that went over limit
 * @p: Task that is going to be killed
 *
 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
 * enabled
 */
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
	struct cgroup *task_cgrp;
	struct cgroup *mem_cgrp;
	/*
	 * Need a buffer in BSS, can't rely on allocations. The code relies
	 * on the assumption that OOM is serialized for memory controller.
	 * If this assumption is broken, revisit this code.
	 */
	static char memcg_name[PATH_MAX];
	int ret;
1662 1663
	struct mem_cgroup *iter;
	unsigned int i;
1664

1665
	if (!p)
1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
		return;

	rcu_read_lock();

	mem_cgrp = memcg->css.cgroup;
	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);

	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		/*
		 * Unfortunately, we are unable to convert to a useful name
		 * But we'll still print out the usage information
		 */
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

1684
	pr_info("Task in %s killed", memcg_name);
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696

	rcu_read_lock();
	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
	if (ret < 0) {
		rcu_read_unlock();
		goto done;
	}
	rcu_read_unlock();

	/*
	 * Continues from above, so we don't need an KERN_ level
	 */
1697
	pr_cont(" as a result of limit of %s\n", memcg_name);
1698 1699
done:

1700
	pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1701 1702 1703
		res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->res, RES_FAILCNT));
1704
	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1705 1706 1707
		res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1708
	pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1709 1710 1711
		res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
		res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
		res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735

	for_each_mem_cgroup_tree(iter, memcg) {
		pr_info("Memory cgroup stats");

		rcu_read_lock();
		ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
		if (!ret)
			pr_cont(" for %s", memcg_name);
		rcu_read_unlock();
		pr_cont(":");

		for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
			if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
				continue;
			pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
				K(mem_cgroup_read_stat(iter, i)));
		}

		for (i = 0; i < NR_LRU_LISTS; i++)
			pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
				K(mem_cgroup_nr_lru_pages(iter, BIT(i))));

		pr_cont("\n");
	}
1736 1737
}

1738 1739 1740 1741
/*
 * This function returns the number of memcg under hierarchy tree. Returns
 * 1(self count) if no children.
 */
1742
static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1743 1744
{
	int num = 0;
K
KAMEZAWA Hiroyuki 已提交
1745 1746
	struct mem_cgroup *iter;

1747
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
1748
		num++;
1749 1750 1751
	return num;
}

D
David Rientjes 已提交
1752 1753 1754
/*
 * Return the memory (and swap, if configured) limit for a memcg.
 */
1755
static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
D
David Rientjes 已提交
1756 1757 1758
{
	u64 limit;

1759 1760
	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);

D
David Rientjes 已提交
1761
	/*
1762
	 * Do not consider swap space if we cannot swap due to swappiness
D
David Rientjes 已提交
1763
	 */
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
	if (mem_cgroup_swappiness(memcg)) {
		u64 memsw;

		limit += total_swap_pages << PAGE_SHIFT;
		memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);

		/*
		 * If memsw is finite and limits the amount of swap space
		 * available to this memcg, return that limit.
		 */
		limit = min(limit, memsw);
	}

	return limit;
D
David Rientjes 已提交
1778 1779
}

1780 1781
static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
				     int order)
1782 1783 1784 1785 1786 1787 1788
{
	struct mem_cgroup *iter;
	unsigned long chosen_points = 0;
	unsigned long totalpages;
	unsigned int points = 0;
	struct task_struct *chosen = NULL;

1789
	/*
1790 1791 1792
	 * If current has a pending SIGKILL or is exiting, then automatically
	 * select it.  The goal is to allow it to allocate so that it may
	 * quickly exit and free its memory.
1793
	 */
1794
	if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
1795 1796 1797 1798 1799
		set_thread_flag(TIF_MEMDIE);
		return;
	}

	check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1800 1801 1802
	totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
	for_each_mem_cgroup_tree(iter, memcg) {
		struct cgroup *cgroup = iter->css.cgroup;
1803
		struct cgroup_task_iter it;
1804 1805
		struct task_struct *task;

1806
		cgroup_task_iter_start(cgroup, &it);
1807
		while ((task = cgroup_task_iter_next(&it))) {
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
			switch (oom_scan_process_thread(task, totalpages, NULL,
							false)) {
			case OOM_SCAN_SELECT:
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = ULONG_MAX;
				get_task_struct(chosen);
				/* fall through */
			case OOM_SCAN_CONTINUE:
				continue;
			case OOM_SCAN_ABORT:
1820
				cgroup_task_iter_end(&it);
1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
				mem_cgroup_iter_break(memcg, iter);
				if (chosen)
					put_task_struct(chosen);
				return;
			case OOM_SCAN_OK:
				break;
			};
			points = oom_badness(task, memcg, NULL, totalpages);
			if (points > chosen_points) {
				if (chosen)
					put_task_struct(chosen);
				chosen = task;
				chosen_points = points;
				get_task_struct(chosen);
			}
		}
1837
		cgroup_task_iter_end(&it);
1838 1839 1840 1841 1842 1843 1844 1845 1846
	}

	if (!chosen)
		return;
	points = chosen_points * 1000 / totalpages;
	oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
			 NULL, "Memory cgroup out of memory");
}

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
					gfp_t gfp_mask,
					unsigned long flags)
{
	unsigned long total = 0;
	bool noswap = false;
	int loop;

	if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
		noswap = true;
	if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
		noswap = true;

	for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
		if (loop)
			drain_all_stock_async(memcg);
		total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
		/*
		 * Allow limit shrinkers, which are triggered directly
		 * by userspace, to catch signals and stop reclaim
		 * after minimal progress, regardless of the margin.
		 */
		if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
			break;
		if (mem_cgroup_margin(memcg))
			break;
		/*
		 * If nothing was reclaimed after two attempts, there
		 * may be no reclaimable pages in this hierarchy.
		 */
		if (loop && !total)
			break;
	}
	return total;
}

1883 1884
/**
 * test_mem_cgroup_node_reclaimable
W
Wanpeng Li 已提交
1885
 * @memcg: the target memcg
1886 1887 1888 1889 1890 1891 1892
 * @nid: the node ID to be checked.
 * @noswap : specify true here if the user wants flle only information.
 *
 * This function returns whether the specified memcg contains any
 * reclaimable pages on a node. Returns true if there are any reclaimable
 * pages in the node.
 */
1893
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1894 1895
		int nid, bool noswap)
{
1896
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1897 1898 1899
		return true;
	if (noswap || !total_swap_pages)
		return false;
1900
	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1901 1902 1903 1904
		return true;
	return false;

}
1905 1906 1907 1908 1909 1910 1911 1912
#if MAX_NUMNODES > 1

/*
 * Always updating the nodemask is not very good - even if we have an empty
 * list or the wrong list here, we can start from some node and traverse all
 * nodes based on the zonelist. So update the list loosely once per 10 secs.
 *
 */
1913
static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1914 1915
{
	int nid;
1916 1917 1918 1919
	/*
	 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
	 * pagein/pageout changes since the last update.
	 */
1920
	if (!atomic_read(&memcg->numainfo_events))
1921
		return;
1922
	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1923 1924 1925
		return;

	/* make a nodemask where this memcg uses memory from */
1926
	memcg->scan_nodes = node_states[N_MEMORY];
1927

1928
	for_each_node_mask(nid, node_states[N_MEMORY]) {
1929

1930 1931
		if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
			node_clear(nid, memcg->scan_nodes);
1932
	}
1933

1934 1935
	atomic_set(&memcg->numainfo_events, 0);
	atomic_set(&memcg->numainfo_updating, 0);
1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
}

/*
 * Selecting a node where we start reclaim from. Because what we need is just
 * reducing usage counter, start from anywhere is O,K. Considering
 * memory reclaim from current node, there are pros. and cons.
 *
 * Freeing memory from current node means freeing memory from a node which
 * we'll use or we've used. So, it may make LRU bad. And if several threads
 * hit limits, it will see a contention on a node. But freeing from remote
 * node means more costs for memory reclaim because of memory latency.
 *
 * Now, we use round-robin. Better algorithm is welcomed.
 */
1950
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1951 1952 1953
{
	int node;

1954 1955
	mem_cgroup_may_update_nodemask(memcg);
	node = memcg->last_scanned_node;
1956

1957
	node = next_node(node, memcg->scan_nodes);
1958
	if (node == MAX_NUMNODES)
1959
		node = first_node(memcg->scan_nodes);
1960 1961 1962 1963 1964 1965 1966 1967 1968
	/*
	 * We call this when we hit limit, not when pages are added to LRU.
	 * No LRU may hold pages because all pages are UNEVICTABLE or
	 * memcg is too small and all pages are not on LRU. In that case,
	 * we use curret node.
	 */
	if (unlikely(node == MAX_NUMNODES))
		node = numa_node_id();

1969
	memcg->last_scanned_node = node;
1970 1971 1972
	return node;
}

1973 1974 1975 1976 1977 1978
/*
 * Check all nodes whether it contains reclaimable pages or not.
 * For quick scan, we make use of scan_nodes. This will allow us to skip
 * unused nodes. But scan_nodes is lazily updated and may not cotain
 * enough new information. We need to do double check.
 */
1979
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1980 1981 1982 1983 1984 1985 1986
{
	int nid;

	/*
	 * quick check...making use of scan_node.
	 * We can skip unused nodes.
	 */
1987 1988
	if (!nodes_empty(memcg->scan_nodes)) {
		for (nid = first_node(memcg->scan_nodes);
1989
		     nid < MAX_NUMNODES;
1990
		     nid = next_node(nid, memcg->scan_nodes)) {
1991

1992
			if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1993 1994 1995 1996 1997 1998
				return true;
		}
	}
	/*
	 * Check rest of nodes.
	 */
1999
	for_each_node_state(nid, N_MEMORY) {
2000
		if (node_isset(nid, memcg->scan_nodes))
2001
			continue;
2002
		if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
2003 2004 2005 2006 2007
			return true;
	}
	return false;
}

2008
#else
2009
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
2010 2011 2012
{
	return 0;
}
2013

2014
static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
2015
{
2016
	return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
2017
}
2018 2019
#endif

2020 2021 2022 2023
static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
				   struct zone *zone,
				   gfp_t gfp_mask,
				   unsigned long *total_scanned)
2024
{
2025
	struct mem_cgroup *victim = NULL;
2026
	int total = 0;
K
KAMEZAWA Hiroyuki 已提交
2027
	int loop = 0;
2028
	unsigned long excess;
2029
	unsigned long nr_scanned;
2030 2031 2032 2033
	struct mem_cgroup_reclaim_cookie reclaim = {
		.zone = zone,
		.priority = 0,
	};
2034

2035
	excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
K
KAMEZAWA Hiroyuki 已提交
2036

2037
	while (1) {
2038
		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
2039
		if (!victim) {
K
KAMEZAWA Hiroyuki 已提交
2040
			loop++;
2041 2042 2043 2044 2045 2046
			if (loop >= 2) {
				/*
				 * If we have not been able to reclaim
				 * anything, it might because there are
				 * no reclaimable pages under this hierarchy
				 */
2047
				if (!total)
2048 2049
					break;
				/*
L
Lucas De Marchi 已提交
2050
				 * We want to do more targeted reclaim.
2051 2052 2053 2054 2055
				 * excess >> 2 is not to excessive so as to
				 * reclaim too much, nor too less that we keep
				 * coming back to reclaim from this cgroup
				 */
				if (total >= (excess >> 2) ||
2056
					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
2057 2058
					break;
			}
2059
			continue;
2060
		}
2061
		if (!mem_cgroup_reclaimable(victim, false))
2062
			continue;
2063 2064 2065 2066
		total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
						     zone, &nr_scanned);
		*total_scanned += nr_scanned;
		if (!res_counter_soft_limit_excess(&root_memcg->res))
2067
			break;
2068
	}
2069
	mem_cgroup_iter_break(root_memcg, victim);
K
KAMEZAWA Hiroyuki 已提交
2070
	return total;
2071 2072
}

K
KAMEZAWA Hiroyuki 已提交
2073 2074 2075
/*
 * Check OOM-Killer is already running under our hierarchy.
 * If someone is running, return false.
2076
 * Has to be called with memcg_oom_lock
K
KAMEZAWA Hiroyuki 已提交
2077
 */
2078
static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
2079
{
2080
	struct mem_cgroup *iter, *failed = NULL;
2081

2082
	for_each_mem_cgroup_tree(iter, memcg) {
2083
		if (iter->oom_lock) {
2084 2085 2086 2087 2088
			/*
			 * this subtree of our hierarchy is already locked
			 * so we cannot give a lock.
			 */
			failed = iter;
2089 2090
			mem_cgroup_iter_break(memcg, iter);
			break;
2091 2092
		} else
			iter->oom_lock = true;
K
KAMEZAWA Hiroyuki 已提交
2093
	}
K
KAMEZAWA Hiroyuki 已提交
2094

2095
	if (!failed)
2096
		return true;
2097 2098 2099 2100 2101

	/*
	 * OK, we failed to lock the whole subtree so we have to clean up
	 * what we set up to the failing subtree
	 */
2102
	for_each_mem_cgroup_tree(iter, memcg) {
2103
		if (iter == failed) {
2104 2105
			mem_cgroup_iter_break(memcg, iter);
			break;
2106 2107 2108
		}
		iter->oom_lock = false;
	}
2109
	return false;
2110
}
2111

2112
/*
2113
 * Has to be called with memcg_oom_lock
2114
 */
2115
static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2116
{
K
KAMEZAWA Hiroyuki 已提交
2117 2118
	struct mem_cgroup *iter;

2119
	for_each_mem_cgroup_tree(iter, memcg)
2120 2121 2122 2123
		iter->oom_lock = false;
	return 0;
}

2124
static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2125 2126 2127
{
	struct mem_cgroup *iter;

2128
	for_each_mem_cgroup_tree(iter, memcg)
2129 2130 2131
		atomic_inc(&iter->under_oom);
}

2132
static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2133 2134 2135
{
	struct mem_cgroup *iter;

K
KAMEZAWA Hiroyuki 已提交
2136 2137 2138 2139 2140
	/*
	 * When a new child is created while the hierarchy is under oom,
	 * mem_cgroup_oom_lock() may not be called. We have to use
	 * atomic_add_unless() here.
	 */
2141
	for_each_mem_cgroup_tree(iter, memcg)
2142
		atomic_add_unless(&iter->under_oom, -1, 0);
2143 2144
}

2145
static DEFINE_SPINLOCK(memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2146 2147
static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);

K
KAMEZAWA Hiroyuki 已提交
2148
struct oom_wait_info {
2149
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
2150 2151 2152 2153 2154 2155
	wait_queue_t	wait;
};

static int memcg_oom_wake_function(wait_queue_t *wait,
	unsigned mode, int sync, void *arg)
{
2156 2157
	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
	struct mem_cgroup *oom_wait_memcg;
K
KAMEZAWA Hiroyuki 已提交
2158 2159 2160
	struct oom_wait_info *oom_wait_info;

	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2161
	oom_wait_memcg = oom_wait_info->memcg;
K
KAMEZAWA Hiroyuki 已提交
2162 2163

	/*
2164
	 * Both of oom_wait_info->memcg and wake_memcg are stable under us.
K
KAMEZAWA Hiroyuki 已提交
2165 2166
	 * Then we can use css_is_ancestor without taking care of RCU.
	 */
2167 2168
	if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
		&& !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
K
KAMEZAWA Hiroyuki 已提交
2169 2170 2171 2172
		return 0;
	return autoremove_wake_function(wait, mode, sync, arg);
}

2173
static void memcg_wakeup_oom(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
2174
{
2175 2176
	/* for filtering, pass "memcg" as argument. */
	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
K
KAMEZAWA Hiroyuki 已提交
2177 2178
}

2179
static void memcg_oom_recover(struct mem_cgroup *memcg)
2180
{
2181 2182
	if (memcg && atomic_read(&memcg->under_oom))
		memcg_wakeup_oom(memcg);
2183 2184
}

K
KAMEZAWA Hiroyuki 已提交
2185 2186 2187
/*
 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
 */
2188 2189
static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
				  int order)
2190
{
K
KAMEZAWA Hiroyuki 已提交
2191
	struct oom_wait_info owait;
2192
	bool locked, need_to_kill;
K
KAMEZAWA Hiroyuki 已提交
2193

2194
	owait.memcg = memcg;
K
KAMEZAWA Hiroyuki 已提交
2195 2196 2197 2198
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.task_list);
2199
	need_to_kill = true;
2200
	mem_cgroup_mark_under_oom(memcg);
2201

2202
	/* At first, try to OOM lock hierarchy under memcg.*/
2203
	spin_lock(&memcg_oom_lock);
2204
	locked = mem_cgroup_oom_lock(memcg);
K
KAMEZAWA Hiroyuki 已提交
2205 2206 2207 2208 2209
	/*
	 * Even if signal_pending(), we can't quit charge() loop without
	 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
	 * under OOM is always welcomed, use TASK_KILLABLE here.
	 */
2210
	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2211
	if (!locked || memcg->oom_kill_disable)
2212 2213
		need_to_kill = false;
	if (locked)
2214
		mem_cgroup_oom_notify(memcg);
2215
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2216

2217 2218
	if (need_to_kill) {
		finish_wait(&memcg_oom_waitq, &owait.wait);
2219
		mem_cgroup_out_of_memory(memcg, mask, order);
2220
	} else {
K
KAMEZAWA Hiroyuki 已提交
2221
		schedule();
K
KAMEZAWA Hiroyuki 已提交
2222
		finish_wait(&memcg_oom_waitq, &owait.wait);
K
KAMEZAWA Hiroyuki 已提交
2223
	}
2224
	spin_lock(&memcg_oom_lock);
2225
	if (locked)
2226 2227
		mem_cgroup_oom_unlock(memcg);
	memcg_wakeup_oom(memcg);
2228
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
2229

2230
	mem_cgroup_unmark_under_oom(memcg);
2231

K
KAMEZAWA Hiroyuki 已提交
2232 2233 2234
	if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
		return false;
	/* Give chance to dying process */
2235
	schedule_timeout_uninterruptible(1);
K
KAMEZAWA Hiroyuki 已提交
2236
	return true;
2237 2238
}

2239 2240 2241
/*
 * Currently used to update mapped file statistics, but the routine can be
 * generalized to update other statistics as well.
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258
 *
 * Notes: Race condition
 *
 * We usually use page_cgroup_lock() for accessing page_cgroup member but
 * it tends to be costly. But considering some conditions, we doesn't need
 * to do so _always_.
 *
 * Considering "charge", lock_page_cgroup() is not required because all
 * file-stat operations happen after a page is attached to radix-tree. There
 * are no race with "charge".
 *
 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
 * if there are race with "uncharge". Statistics itself is properly handled
 * by flags.
 *
 * Considering "move", this is an only case we see a race. To make the race
2259 2260
 * small, we check mm->moving_account and detect there are possibility of race
 * If there is, we take a lock.
2261
 */
2262

2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275
void __mem_cgroup_begin_update_page_stat(struct page *page,
				bool *locked, unsigned long *flags)
{
	struct mem_cgroup *memcg;
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
again:
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
		return;
	/*
	 * If this memory cgroup is not under account moving, we don't
2276
	 * need to take move_lock_mem_cgroup(). Because we already hold
2277
	 * rcu_read_lock(), any calls to move_account will be delayed until
2278
	 * rcu_read_unlock() if mem_cgroup_stolen() == true.
2279
	 */
2280
	if (!mem_cgroup_stolen(memcg))
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
		return;

	move_lock_mem_cgroup(memcg, flags);
	if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
		move_unlock_mem_cgroup(memcg, flags);
		goto again;
	}
	*locked = true;
}

void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
{
	struct page_cgroup *pc = lookup_page_cgroup(page);

	/*
	 * It's guaranteed that pc->mem_cgroup never changes while
	 * lock is held because a routine modifies pc->mem_cgroup
2298
	 * should take move_lock_mem_cgroup().
2299 2300 2301 2302
	 */
	move_unlock_mem_cgroup(pc->mem_cgroup, flags);
}

2303 2304
void mem_cgroup_update_page_stat(struct page *page,
				 enum mem_cgroup_page_stat_item idx, int val)
2305
{
2306
	struct mem_cgroup *memcg;
2307
	struct page_cgroup *pc = lookup_page_cgroup(page);
2308
	unsigned long uninitialized_var(flags);
2309

2310
	if (mem_cgroup_disabled())
2311
		return;
2312

2313 2314
	memcg = pc->mem_cgroup;
	if (unlikely(!memcg || !PageCgroupUsed(pc)))
2315
		return;
2316 2317

	switch (idx) {
2318 2319
	case MEMCG_NR_FILE_MAPPED:
		idx = MEM_CGROUP_STAT_FILE_MAPPED;
2320 2321 2322
		break;
	default:
		BUG();
2323
	}
2324

2325
	this_cpu_add(memcg->stat->count[idx], val);
2326
}
2327

2328 2329 2330 2331
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
2332
#define CHARGE_BATCH	32U
2333 2334
struct memcg_stock_pcp {
	struct mem_cgroup *cached; /* this never be root cgroup */
2335
	unsigned int nr_pages;
2336
	struct work_struct work;
2337
	unsigned long flags;
2338
#define FLUSHING_CACHED_CHARGE	0
2339 2340
};
static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2341
static DEFINE_MUTEX(percpu_charge_mutex);
2342

2343 2344 2345 2346 2347 2348 2349 2350 2351 2352
/**
 * consume_stock: Try to consume stocked charge on this cpu.
 * @memcg: memcg to consume from.
 * @nr_pages: how many pages to charge.
 *
 * The charges will only happen if @memcg matches the current cpu's memcg
 * stock, and at least @nr_pages are available in that stock.  Failure to
 * service an allocation will refill the stock.
 *
 * returns true if successful, false otherwise.
2353
 */
2354
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2355 2356 2357 2358
{
	struct memcg_stock_pcp *stock;
	bool ret = true;

2359 2360 2361
	if (nr_pages > CHARGE_BATCH)
		return false;

2362
	stock = &get_cpu_var(memcg_stock);
2363 2364
	if (memcg == stock->cached && stock->nr_pages >= nr_pages)
		stock->nr_pages -= nr_pages;
2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
	else /* need to call res_counter_charge */
		ret = false;
	put_cpu_var(memcg_stock);
	return ret;
}

/*
 * Returns stocks cached in percpu to res_counter and reset cached information.
 */
static void drain_stock(struct memcg_stock_pcp *stock)
{
	struct mem_cgroup *old = stock->cached;

2378 2379 2380 2381
	if (stock->nr_pages) {
		unsigned long bytes = stock->nr_pages * PAGE_SIZE;

		res_counter_uncharge(&old->res, bytes);
2382
		if (do_swap_account)
2383 2384
			res_counter_uncharge(&old->memsw, bytes);
		stock->nr_pages = 0;
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
	}
	stock->cached = NULL;
}

/*
 * This must be called under preempt disabled or must be called by
 * a thread which is pinned to local cpu.
 */
static void drain_local_stock(struct work_struct *dummy)
{
	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
	drain_stock(stock);
2397
	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2398 2399
}

2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
static void __init memcg_stock_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		struct memcg_stock_pcp *stock =
					&per_cpu(memcg_stock, cpu);
		INIT_WORK(&stock->work, drain_local_stock);
	}
}

2411 2412
/*
 * Cache charges(val) which is from res_counter, to local per_cpu area.
2413
 * This will be consumed by consume_stock() function, later.
2414
 */
2415
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2416 2417 2418
{
	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);

2419
	if (stock->cached != memcg) { /* reset if necessary */
2420
		drain_stock(stock);
2421
		stock->cached = memcg;
2422
	}
2423
	stock->nr_pages += nr_pages;
2424 2425 2426 2427
	put_cpu_var(memcg_stock);
}

/*
2428
 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2429 2430
 * of the hierarchy under it. sync flag says whether we should block
 * until the work is done.
2431
 */
2432
static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2433
{
2434
	int cpu, curcpu;
2435

2436 2437
	/* Notify other cpus that system-wide "drain" is running */
	get_online_cpus();
2438
	curcpu = get_cpu();
2439 2440
	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2441
		struct mem_cgroup *memcg;
2442

2443 2444
		memcg = stock->cached;
		if (!memcg || !stock->nr_pages)
2445
			continue;
2446
		if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2447
			continue;
2448 2449 2450 2451 2452 2453
		if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
			if (cpu == curcpu)
				drain_local_stock(&stock->work);
			else
				schedule_work_on(cpu, &stock->work);
		}
2454
	}
2455
	put_cpu();
2456 2457 2458 2459 2460 2461

	if (!sync)
		goto out;

	for_each_online_cpu(cpu) {
		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2462
		if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2463 2464 2465
			flush_work(&stock->work);
	}
out:
2466
 	put_online_cpus();
2467 2468 2469 2470 2471 2472 2473 2474
}

/*
 * Tries to drain stocked charges in other cpus. This function is asynchronous
 * and just put a work per cpu for draining localy on each cpu. Caller can
 * expects some charges will be back to res_counter later but cannot wait for
 * it.
 */
2475
static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2476
{
2477 2478 2479 2480 2481
	/*
	 * If someone calls draining, avoid adding more kworker runs.
	 */
	if (!mutex_trylock(&percpu_charge_mutex))
		return;
2482
	drain_all_stock(root_memcg, false);
2483
	mutex_unlock(&percpu_charge_mutex);
2484 2485 2486
}

/* This is a synchronous drain interface. */
2487
static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2488 2489
{
	/* called when force_empty is called */
2490
	mutex_lock(&percpu_charge_mutex);
2491
	drain_all_stock(root_memcg, true);
2492
	mutex_unlock(&percpu_charge_mutex);
2493 2494
}

2495 2496 2497 2498
/*
 * This function drains percpu counter value from DEAD cpu and
 * move it to local cpu. Note that this function can be preempted.
 */
2499
static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2500 2501 2502
{
	int i;

2503
	spin_lock(&memcg->pcp_counter_lock);
2504
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2505
		long x = per_cpu(memcg->stat->count[i], cpu);
2506

2507 2508
		per_cpu(memcg->stat->count[i], cpu) = 0;
		memcg->nocpu_base.count[i] += x;
2509
	}
2510
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2511
		unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2512

2513 2514
		per_cpu(memcg->stat->events[i], cpu) = 0;
		memcg->nocpu_base.events[i] += x;
2515
	}
2516
	spin_unlock(&memcg->pcp_counter_lock);
2517 2518 2519
}

static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2520 2521 2522 2523 2524
					unsigned long action,
					void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct memcg_stock_pcp *stock;
2525
	struct mem_cgroup *iter;
2526

2527
	if (action == CPU_ONLINE)
2528 2529
		return NOTIFY_OK;

2530
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2531
		return NOTIFY_OK;
2532

2533
	for_each_mem_cgroup(iter)
2534 2535
		mem_cgroup_drain_pcp_counter(iter, cpu);

2536 2537 2538 2539 2540
	stock = &per_cpu(memcg_stock, cpu);
	drain_stock(stock);
	return NOTIFY_OK;
}

2541 2542 2543 2544 2545 2546 2547 2548 2549 2550

/* See __mem_cgroup_try_charge() for details */
enum {
	CHARGE_OK,		/* success */
	CHARGE_RETRY,		/* need to retry but retry is not bad */
	CHARGE_NOMEM,		/* we can't do more. return -ENOMEM */
	CHARGE_WOULDBLOCK,	/* GFP_WAIT wasn't set and no enough res. */
	CHARGE_OOM_DIE,		/* the current is killed because of OOM */
};

2551
static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2552 2553
				unsigned int nr_pages, unsigned int min_pages,
				bool oom_check)
2554
{
2555
	unsigned long csize = nr_pages * PAGE_SIZE;
2556 2557 2558 2559 2560
	struct mem_cgroup *mem_over_limit;
	struct res_counter *fail_res;
	unsigned long flags = 0;
	int ret;

2561
	ret = res_counter_charge(&memcg->res, csize, &fail_res);
2562 2563 2564 2565

	if (likely(!ret)) {
		if (!do_swap_account)
			return CHARGE_OK;
2566
		ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2567 2568 2569
		if (likely(!ret))
			return CHARGE_OK;

2570
		res_counter_uncharge(&memcg->res, csize);
2571 2572 2573 2574
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
		flags |= MEM_CGROUP_RECLAIM_NOSWAP;
	} else
		mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2575 2576 2577 2578
	/*
	 * Never reclaim on behalf of optional batching, retry with a
	 * single page instead.
	 */
2579
	if (nr_pages > min_pages)
2580 2581 2582 2583 2584
		return CHARGE_RETRY;

	if (!(gfp_mask & __GFP_WAIT))
		return CHARGE_WOULDBLOCK;

2585 2586 2587
	if (gfp_mask & __GFP_NORETRY)
		return CHARGE_NOMEM;

2588
	ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2589
	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2590
		return CHARGE_RETRY;
2591
	/*
2592 2593 2594 2595 2596 2597 2598
	 * Even though the limit is exceeded at this point, reclaim
	 * may have been able to free some pages.  Retry the charge
	 * before killing the task.
	 *
	 * Only for regular pages, though: huge pages are rather
	 * unlikely to succeed so close to the limit, and we fall back
	 * to regular pages anyway in case of failure.
2599
	 */
2600
	if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
		return CHARGE_RETRY;

	/*
	 * At task move, charge accounts can be doubly counted. So, it's
	 * better to wait until the end of task_move if something is going on.
	 */
	if (mem_cgroup_wait_acct_move(mem_over_limit))
		return CHARGE_RETRY;

	/* If we don't need to call oom-killer at el, return immediately */
	if (!oom_check)
		return CHARGE_NOMEM;
	/* check OOM */
2614
	if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
2615 2616 2617 2618 2619
		return CHARGE_OOM_DIE;

	return CHARGE_RETRY;
}

2620
/*
2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639
 * __mem_cgroup_try_charge() does
 * 1. detect memcg to be charged against from passed *mm and *ptr,
 * 2. update res_counter
 * 3. call memory reclaim if necessary.
 *
 * In some special case, if the task is fatal, fatal_signal_pending() or
 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
 * as possible without any hazards. 2: all pages should have a valid
 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
 * pointer, that is treated as a charge to root_mem_cgroup.
 *
 * So __mem_cgroup_try_charge() will return
 *  0       ...  on success, filling *ptr with a valid memcg pointer.
 *  -ENOMEM ...  charge failure because of resource limits.
 *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
 *
 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
 * the oom-killer can be invoked.
2640
 */
2641
static int __mem_cgroup_try_charge(struct mm_struct *mm,
A
Andrea Arcangeli 已提交
2642
				   gfp_t gfp_mask,
2643
				   unsigned int nr_pages,
2644
				   struct mem_cgroup **ptr,
2645
				   bool oom)
2646
{
2647
	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2648
	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2649
	struct mem_cgroup *memcg = NULL;
2650
	int ret;
2651

K
KAMEZAWA Hiroyuki 已提交
2652 2653 2654 2655 2656 2657 2658 2659
	/*
	 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
	 * in system level. So, allow to go ahead dying process in addition to
	 * MEMDIE process.
	 */
	if (unlikely(test_thread_flag(TIF_MEMDIE)
		     || fatal_signal_pending(current)))
		goto bypass;
2660

2661
	/*
2662 2663
	 * We always charge the cgroup the mm_struct belongs to.
	 * The mm_struct's mem_cgroup changes on task migration if the
2664
	 * thread group leader migrates. It's possible that mm is not
2665
	 * set, if so charge the root memcg (happens for pagecache usage).
2666
	 */
2667
	if (!*ptr && !mm)
2668
		*ptr = root_mem_cgroup;
K
KAMEZAWA Hiroyuki 已提交
2669
again:
2670 2671 2672
	if (*ptr) { /* css should be a valid one */
		memcg = *ptr;
		if (mem_cgroup_is_root(memcg))
K
KAMEZAWA Hiroyuki 已提交
2673
			goto done;
2674
		if (consume_stock(memcg, nr_pages))
K
KAMEZAWA Hiroyuki 已提交
2675
			goto done;
2676
		css_get(&memcg->css);
2677
	} else {
K
KAMEZAWA Hiroyuki 已提交
2678
		struct task_struct *p;
2679

K
KAMEZAWA Hiroyuki 已提交
2680 2681 2682
		rcu_read_lock();
		p = rcu_dereference(mm->owner);
		/*
2683
		 * Because we don't have task_lock(), "p" can exit.
2684
		 * In that case, "memcg" can point to root or p can be NULL with
2685 2686 2687 2688 2689 2690
		 * race with swapoff. Then, we have small risk of mis-accouning.
		 * But such kind of mis-account by race always happens because
		 * we don't have cgroup_mutex(). It's overkill and we allo that
		 * small race, here.
		 * (*) swapoff at el will charge against mm-struct not against
		 * task-struct. So, mm->owner can be NULL.
K
KAMEZAWA Hiroyuki 已提交
2691
		 */
2692
		memcg = mem_cgroup_from_task(p);
2693 2694 2695
		if (!memcg)
			memcg = root_mem_cgroup;
		if (mem_cgroup_is_root(memcg)) {
K
KAMEZAWA Hiroyuki 已提交
2696 2697 2698
			rcu_read_unlock();
			goto done;
		}
2699
		if (consume_stock(memcg, nr_pages)) {
K
KAMEZAWA Hiroyuki 已提交
2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711
			/*
			 * It seems dagerous to access memcg without css_get().
			 * But considering how consume_stok works, it's not
			 * necessary. If consume_stock success, some charges
			 * from this memcg are cached on this cpu. So, we
			 * don't need to call css_get()/css_tryget() before
			 * calling consume_stock().
			 */
			rcu_read_unlock();
			goto done;
		}
		/* after here, we may be blocked. we need to get refcnt */
2712
		if (!css_tryget(&memcg->css)) {
K
KAMEZAWA Hiroyuki 已提交
2713 2714 2715 2716 2717
			rcu_read_unlock();
			goto again;
		}
		rcu_read_unlock();
	}
2718

2719 2720
	do {
		bool oom_check;
2721

2722
		/* If killed, bypass charge */
K
KAMEZAWA Hiroyuki 已提交
2723
		if (fatal_signal_pending(current)) {
2724
			css_put(&memcg->css);
2725
			goto bypass;
K
KAMEZAWA Hiroyuki 已提交
2726
		}
2727

2728 2729 2730 2731
		oom_check = false;
		if (oom && !nr_oom_retries) {
			oom_check = true;
			nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2732
		}
2733

2734 2735
		ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, nr_pages,
		    oom_check);
2736 2737 2738 2739
		switch (ret) {
		case CHARGE_OK:
			break;
		case CHARGE_RETRY: /* not in OOM situation but retry */
2740
			batch = nr_pages;
2741 2742
			css_put(&memcg->css);
			memcg = NULL;
K
KAMEZAWA Hiroyuki 已提交
2743
			goto again;
2744
		case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2745
			css_put(&memcg->css);
2746 2747
			goto nomem;
		case CHARGE_NOMEM: /* OOM routine works */
K
KAMEZAWA Hiroyuki 已提交
2748
			if (!oom) {
2749
				css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2750
				goto nomem;
K
KAMEZAWA Hiroyuki 已提交
2751
			}
2752 2753 2754 2755
			/* If oom, we never return -ENOMEM */
			nr_oom_retries--;
			break;
		case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2756
			css_put(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
2757
			goto bypass;
2758
		}
2759 2760
	} while (ret != CHARGE_OK);

2761
	if (batch > nr_pages)
2762 2763
		refill_stock(memcg, batch - nr_pages);
	css_put(&memcg->css);
2764
done:
2765
	*ptr = memcg;
2766 2767
	return 0;
nomem:
2768
	*ptr = NULL;
2769
	return -ENOMEM;
K
KAMEZAWA Hiroyuki 已提交
2770
bypass:
2771 2772
	*ptr = root_mem_cgroup;
	return -EINTR;
2773
}
2774

2775 2776 2777 2778 2779
/*
 * Somemtimes we have to undo a charge we got by try_charge().
 * This function is for that and do uncharge, put css's refcnt.
 * gotten by try_charge().
 */
2780
static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2781
				       unsigned int nr_pages)
2782
{
2783
	if (!mem_cgroup_is_root(memcg)) {
2784 2785
		unsigned long bytes = nr_pages * PAGE_SIZE;

2786
		res_counter_uncharge(&memcg->res, bytes);
2787
		if (do_swap_account)
2788
			res_counter_uncharge(&memcg->memsw, bytes);
2789
	}
2790 2791
}

2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
/*
 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
 * This is useful when moving usage to parent cgroup.
 */
static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
					unsigned int nr_pages)
{
	unsigned long bytes = nr_pages * PAGE_SIZE;

	if (mem_cgroup_is_root(memcg))
		return;

	res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
	if (do_swap_account)
		res_counter_uncharge_until(&memcg->memsw,
						memcg->memsw.parent, bytes);
}

2810 2811
/*
 * A helper function to get mem_cgroup from ID. must be called under
T
Tejun Heo 已提交
2812 2813 2814
 * rcu_read_lock().  The caller is responsible for calling css_tryget if
 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
 * called against removed memcg.)
2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
 */
static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
{
	struct cgroup_subsys_state *css;

	/* ID 0 is unused ID */
	if (!id)
		return NULL;
	css = css_lookup(&mem_cgroup_subsys, id);
	if (!css)
		return NULL;
2826
	return mem_cgroup_from_css(css);
2827 2828
}

2829
struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2830
{
2831
	struct mem_cgroup *memcg = NULL;
2832
	struct page_cgroup *pc;
2833
	unsigned short id;
2834 2835
	swp_entry_t ent;

2836 2837 2838
	VM_BUG_ON(!PageLocked(page));

	pc = lookup_page_cgroup(page);
2839
	lock_page_cgroup(pc);
2840
	if (PageCgroupUsed(pc)) {
2841 2842 2843
		memcg = pc->mem_cgroup;
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2844
	} else if (PageSwapCache(page)) {
2845
		ent.val = page_private(page);
2846
		id = lookup_swap_cgroup_id(ent);
2847
		rcu_read_lock();
2848 2849 2850
		memcg = mem_cgroup_lookup(id);
		if (memcg && !css_tryget(&memcg->css))
			memcg = NULL;
2851
		rcu_read_unlock();
2852
	}
2853
	unlock_page_cgroup(pc);
2854
	return memcg;
2855 2856
}

2857
static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2858
				       struct page *page,
2859
				       unsigned int nr_pages,
2860 2861
				       enum charge_type ctype,
				       bool lrucare)
2862
{
2863
	struct page_cgroup *pc = lookup_page_cgroup(page);
2864
	struct zone *uninitialized_var(zone);
2865
	struct lruvec *lruvec;
2866
	bool was_on_lru = false;
2867
	bool anon;
2868

2869
	lock_page_cgroup(pc);
2870
	VM_BUG_ON(PageCgroupUsed(pc));
2871 2872 2873 2874
	/*
	 * we don't need page_cgroup_lock about tail pages, becase they are not
	 * accessed by any other context at this point.
	 */
2875 2876 2877 2878 2879 2880 2881 2882 2883

	/*
	 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
	 * may already be on some other mem_cgroup's LRU.  Take care of it.
	 */
	if (lrucare) {
		zone = page_zone(page);
		spin_lock_irq(&zone->lru_lock);
		if (PageLRU(page)) {
2884
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2885
			ClearPageLRU(page);
2886
			del_page_from_lru_list(page, lruvec, page_lru(page));
2887 2888 2889 2890
			was_on_lru = true;
		}
	}

2891
	pc->mem_cgroup = memcg;
2892 2893 2894 2895 2896 2897 2898
	/*
	 * We access a page_cgroup asynchronously without lock_page_cgroup().
	 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
	 * is accessed after testing USED bit. To make pc->mem_cgroup visible
	 * before USED bit, we need memory barrier here.
	 * See mem_cgroup_add_lru_list(), etc.
 	 */
K
KAMEZAWA Hiroyuki 已提交
2899
	smp_wmb();
2900
	SetPageCgroupUsed(pc);
2901

2902 2903
	if (lrucare) {
		if (was_on_lru) {
2904
			lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2905 2906
			VM_BUG_ON(PageLRU(page));
			SetPageLRU(page);
2907
			add_page_to_lru_list(page, lruvec, page_lru(page));
2908 2909 2910 2911
		}
		spin_unlock_irq(&zone->lru_lock);
	}

2912
	if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2913 2914 2915 2916
		anon = true;
	else
		anon = false;

2917
	mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
2918
	unlock_page_cgroup(pc);
2919

2920 2921 2922 2923 2924
	/*
	 * "charge_statistics" updated event counter. Then, check it.
	 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
	 * if they exceeds softlimit.
	 */
2925
	memcg_check_events(memcg, page);
2926
}
2927

2928 2929
static DEFINE_MUTEX(set_limit_mutex);

2930 2931 2932 2933 2934 2935 2936
#ifdef CONFIG_MEMCG_KMEM
static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
{
	return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
		(memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
}

G
Glauber Costa 已提交
2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
/*
 * This is a bit cumbersome, but it is rarely used and avoids a backpointer
 * in the memcg_cache_params struct.
 */
static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
{
	struct kmem_cache *cachep;

	VM_BUG_ON(p->is_root_cache);
	cachep = p->root_cache;
	return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
}

2950
#ifdef CONFIG_SLABINFO
2951 2952
static int mem_cgroup_slabinfo_read(struct cgroup_subsys_state *css,
				    struct cftype *cft, struct seq_file *m)
2953
{
2954
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970
	struct memcg_cache_params *params;

	if (!memcg_can_account_kmem(memcg))
		return -EIO;

	print_slabinfo_header(m);

	mutex_lock(&memcg->slab_caches_mutex);
	list_for_each_entry(params, &memcg->memcg_slab_caches, list)
		cache_show(memcg_params_to_cache(params), m);
	mutex_unlock(&memcg->slab_caches_mutex);

	return 0;
}
#endif

2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023
static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
{
	struct res_counter *fail_res;
	struct mem_cgroup *_memcg;
	int ret = 0;
	bool may_oom;

	ret = res_counter_charge(&memcg->kmem, size, &fail_res);
	if (ret)
		return ret;

	/*
	 * Conditions under which we can wait for the oom_killer. Those are
	 * the same conditions tested by the core page allocator
	 */
	may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);

	_memcg = memcg;
	ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
				      &_memcg, may_oom);

	if (ret == -EINTR)  {
		/*
		 * __mem_cgroup_try_charge() chosed to bypass to root due to
		 * OOM kill or fatal signal.  Since our only options are to
		 * either fail the allocation or charge it to this cgroup, do
		 * it as a temporary condition. But we can't fail. From a
		 * kmem/slab perspective, the cache has already been selected,
		 * by mem_cgroup_kmem_get_cache(), so it is too late to change
		 * our minds.
		 *
		 * This condition will only trigger if the task entered
		 * memcg_charge_kmem in a sane state, but was OOM-killed during
		 * __mem_cgroup_try_charge() above. Tasks that were already
		 * dying when the allocation triggers should have been already
		 * directed to the root cgroup in memcontrol.h
		 */
		res_counter_charge_nofail(&memcg->res, size, &fail_res);
		if (do_swap_account)
			res_counter_charge_nofail(&memcg->memsw, size,
						  &fail_res);
		ret = 0;
	} else if (ret)
		res_counter_uncharge(&memcg->kmem, size);

	return ret;
}

static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
{
	res_counter_uncharge(&memcg->res, size);
	if (do_swap_account)
		res_counter_uncharge(&memcg->memsw, size);
3024 3025 3026 3027 3028

	/* Not down to 0 */
	if (res_counter_uncharge(&memcg->kmem, size))
		return;

3029 3030 3031 3032 3033 3034 3035 3036
	/*
	 * Releases a reference taken in kmem_cgroup_css_offline in case
	 * this last uncharge is racing with the offlining code or it is
	 * outliving the memcg existence.
	 *
	 * The memory barrier imposed by test&clear is paired with the
	 * explicit one in memcg_kmem_mark_dead().
	 */
3037
	if (memcg_kmem_test_and_clear_dead(memcg))
3038
		css_put(&memcg->css);
3039 3040
}

3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
{
	if (!memcg)
		return;

	mutex_lock(&memcg->slab_caches_mutex);
	list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
	mutex_unlock(&memcg->slab_caches_mutex);
}

/*
 * helper for acessing a memcg's index. It will be used as an index in the
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}

3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
/*
 * This ends up being protected by the set_limit mutex, during normal
 * operation, because that is its main call site.
 *
 * But when we create a new cache, we can call this as well if its parent
 * is kmem-limited. That will have to hold set_limit_mutex as well.
 */
int memcg_update_cache_sizes(struct mem_cgroup *memcg)
{
	int num, ret;

	num = ida_simple_get(&kmem_limited_groups,
				0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
	if (num < 0)
		return num;
	/*
	 * After this point, kmem_accounted (that we test atomically in
	 * the beginning of this conditional), is no longer 0. This
	 * guarantees only one process will set the following boolean
	 * to true. We don't need test_and_set because we're protected
	 * by the set_limit_mutex anyway.
	 */
	memcg_kmem_set_activated(memcg);

	ret = memcg_update_all_caches(num+1);
	if (ret) {
		ida_simple_remove(&kmem_limited_groups, num);
		memcg_kmem_clear_activated(memcg);
		return ret;
	}

	memcg->kmemcg_id = num;
	INIT_LIST_HEAD(&memcg->memcg_slab_caches);
	mutex_init(&memcg->slab_caches_mutex);
	return 0;
}

static size_t memcg_caches_array_size(int num_groups)
{
	ssize_t size;
	if (num_groups <= 0)
		return 0;

	size = 2 * num_groups;
	if (size < MEMCG_CACHES_MIN_SIZE)
		size = MEMCG_CACHES_MIN_SIZE;
	else if (size > MEMCG_CACHES_MAX_SIZE)
		size = MEMCG_CACHES_MAX_SIZE;

	return size;
}

/*
 * We should update the current array size iff all caches updates succeed. This
 * can only be done from the slab side. The slab mutex needs to be held when
 * calling this.
 */
void memcg_update_array_size(int num)
{
	if (num > memcg_limited_groups_array_size)
		memcg_limited_groups_array_size = memcg_caches_array_size(num);
}

3124 3125
static void kmem_cache_destroy_work_func(struct work_struct *w);

3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176
int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
{
	struct memcg_cache_params *cur_params = s->memcg_params;

	VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);

	if (num_groups > memcg_limited_groups_array_size) {
		int i;
		ssize_t size = memcg_caches_array_size(num_groups);

		size *= sizeof(void *);
		size += sizeof(struct memcg_cache_params);

		s->memcg_params = kzalloc(size, GFP_KERNEL);
		if (!s->memcg_params) {
			s->memcg_params = cur_params;
			return -ENOMEM;
		}

		s->memcg_params->is_root_cache = true;

		/*
		 * There is the chance it will be bigger than
		 * memcg_limited_groups_array_size, if we failed an allocation
		 * in a cache, in which case all caches updated before it, will
		 * have a bigger array.
		 *
		 * But if that is the case, the data after
		 * memcg_limited_groups_array_size is certainly unused
		 */
		for (i = 0; i < memcg_limited_groups_array_size; i++) {
			if (!cur_params->memcg_caches[i])
				continue;
			s->memcg_params->memcg_caches[i] =
						cur_params->memcg_caches[i];
		}

		/*
		 * Ideally, we would wait until all caches succeed, and only
		 * then free the old one. But this is not worth the extra
		 * pointer per-cache we'd have to have for this.
		 *
		 * It is not a big deal if some caches are left with a size
		 * bigger than the others. And all updates will reset this
		 * anyway.
		 */
		kfree(cur_params);
	}
	return 0;
}

G
Glauber Costa 已提交
3177 3178
int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
			 struct kmem_cache *root_cache)
3179 3180 3181 3182 3183 3184
{
	size_t size = sizeof(struct memcg_cache_params);

	if (!memcg_kmem_enabled())
		return 0;

3185 3186 3187
	if (!memcg)
		size += memcg_limited_groups_array_size * sizeof(void *);

3188 3189 3190 3191
	s->memcg_params = kzalloc(size, GFP_KERNEL);
	if (!s->memcg_params)
		return -ENOMEM;

3192 3193
	INIT_WORK(&s->memcg_params->destroy,
			kmem_cache_destroy_work_func);
G
Glauber Costa 已提交
3194
	if (memcg) {
3195
		s->memcg_params->memcg = memcg;
G
Glauber Costa 已提交
3196
		s->memcg_params->root_cache = root_cache;
3197 3198 3199
	} else
		s->memcg_params->is_root_cache = true;

3200 3201 3202 3203 3204
	return 0;
}

void memcg_release_cache(struct kmem_cache *s)
{
3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228
	struct kmem_cache *root;
	struct mem_cgroup *memcg;
	int id;

	/*
	 * This happens, for instance, when a root cache goes away before we
	 * add any memcg.
	 */
	if (!s->memcg_params)
		return;

	if (s->memcg_params->is_root_cache)
		goto out;

	memcg = s->memcg_params->memcg;
	id  = memcg_cache_id(memcg);

	root = s->memcg_params->root_cache;
	root->memcg_params->memcg_caches[id] = NULL;

	mutex_lock(&memcg->slab_caches_mutex);
	list_del(&s->memcg_params->list);
	mutex_unlock(&memcg->slab_caches_mutex);

3229
	css_put(&memcg->css);
3230
out:
3231 3232 3233
	kfree(s->memcg_params);
}

3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264
/*
 * During the creation a new cache, we need to disable our accounting mechanism
 * altogether. This is true even if we are not creating, but rather just
 * enqueing new caches to be created.
 *
 * This is because that process will trigger allocations; some visible, like
 * explicit kmallocs to auxiliary data structures, name strings and internal
 * cache structures; some well concealed, like INIT_WORK() that can allocate
 * objects during debug.
 *
 * If any allocation happens during memcg_kmem_get_cache, we will recurse back
 * to it. This may not be a bounded recursion: since the first cache creation
 * failed to complete (waiting on the allocation), we'll just try to create the
 * cache again, failing at the same point.
 *
 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
 * memcg_kmem_skip_account. So we enclose anything that might allocate memory
 * inside the following two functions.
 */
static inline void memcg_stop_kmem_account(void)
{
	VM_BUG_ON(!current->mm);
	current->memcg_kmem_skip_account++;
}

static inline void memcg_resume_kmem_account(void)
{
	VM_BUG_ON(!current->mm);
	current->memcg_kmem_skip_account--;
}

G
Glauber Costa 已提交
3265 3266 3267 3268 3269 3270 3271 3272 3273
static void kmem_cache_destroy_work_func(struct work_struct *w)
{
	struct kmem_cache *cachep;
	struct memcg_cache_params *p;

	p = container_of(w, struct memcg_cache_params, destroy);

	cachep = memcg_params_to_cache(p);

G
Glauber Costa 已提交
3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294
	/*
	 * If we get down to 0 after shrink, we could delete right away.
	 * However, memcg_release_pages() already puts us back in the workqueue
	 * in that case. If we proceed deleting, we'll get a dangling
	 * reference, and removing the object from the workqueue in that case
	 * is unnecessary complication. We are not a fast path.
	 *
	 * Note that this case is fundamentally different from racing with
	 * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
	 * kmem_cache_shrink, not only we would be reinserting a dead cache
	 * into the queue, but doing so from inside the worker racing to
	 * destroy it.
	 *
	 * So if we aren't down to zero, we'll just schedule a worker and try
	 * again
	 */
	if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
		kmem_cache_shrink(cachep);
		if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
			return;
	} else
G
Glauber Costa 已提交
3295 3296 3297 3298 3299 3300 3301 3302
		kmem_cache_destroy(cachep);
}

void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
{
	if (!cachep->memcg_params->dead)
		return;

G
Glauber Costa 已提交
3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322
	/*
	 * There are many ways in which we can get here.
	 *
	 * We can get to a memory-pressure situation while the delayed work is
	 * still pending to run. The vmscan shrinkers can then release all
	 * cache memory and get us to destruction. If this is the case, we'll
	 * be executed twice, which is a bug (the second time will execute over
	 * bogus data). In this case, cancelling the work should be fine.
	 *
	 * But we can also get here from the worker itself, if
	 * kmem_cache_shrink is enough to shake all the remaining objects and
	 * get the page count to 0. In this case, we'll deadlock if we try to
	 * cancel the work (the worker runs with an internal lock held, which
	 * is the same lock we would hold for cancel_work_sync().)
	 *
	 * Since we can't possibly know who got us here, just refrain from
	 * running if there is already work pending
	 */
	if (work_pending(&cachep->memcg_params->destroy))
		return;
G
Glauber Costa 已提交
3323 3324 3325 3326 3327 3328 3329
	/*
	 * We have to defer the actual destroying to a workqueue, because
	 * we might currently be in a context that cannot sleep.
	 */
	schedule_work(&cachep->memcg_params->destroy);
}

3330 3331 3332 3333 3334 3335 3336 3337 3338
/*
 * This lock protects updaters, not readers. We want readers to be as fast as
 * they can, and they will either see NULL or a valid cache value. Our model
 * allow them to see NULL, in which case the root memcg will be selected.
 *
 * We need this lock because multiple allocations to the same cache from a non
 * will span more than one worker. Only one of them can create the cache.
 */
static DEFINE_MUTEX(memcg_cache_mutex);
3339

3340 3341 3342
/*
 * Called with memcg_cache_mutex held
 */
3343 3344 3345 3346
static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
					 struct kmem_cache *s)
{
	struct kmem_cache *new;
3347
	static char *tmp_name = NULL;
3348

3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366
	lockdep_assert_held(&memcg_cache_mutex);

	/*
	 * kmem_cache_create_memcg duplicates the given name and
	 * cgroup_name for this name requires RCU context.
	 * This static temporary buffer is used to prevent from
	 * pointless shortliving allocation.
	 */
	if (!tmp_name) {
		tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
		if (!tmp_name)
			return NULL;
	}

	rcu_read_lock();
	snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
			 memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
	rcu_read_unlock();
3367

3368
	new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
G
Glauber Costa 已提交
3369
				      (s->flags & ~SLAB_PANIC), s->ctor, s);
3370

3371 3372 3373
	if (new)
		new->allocflags |= __GFP_KMEMCG;

3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388
	return new;
}

static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
						  struct kmem_cache *cachep)
{
	struct kmem_cache *new_cachep;
	int idx;

	BUG_ON(!memcg_can_account_kmem(memcg));

	idx = memcg_cache_id(memcg);

	mutex_lock(&memcg_cache_mutex);
	new_cachep = cachep->memcg_params->memcg_caches[idx];
3389 3390
	if (new_cachep) {
		css_put(&memcg->css);
3391
		goto out;
3392
	}
3393 3394 3395 3396

	new_cachep = kmem_cache_dup(memcg, cachep);
	if (new_cachep == NULL) {
		new_cachep = cachep;
3397
		css_put(&memcg->css);
3398 3399 3400
		goto out;
	}

G
Glauber Costa 已提交
3401
	atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413

	cachep->memcg_params->memcg_caches[idx] = new_cachep;
	/*
	 * the readers won't lock, make sure everybody sees the updated value,
	 * so they won't put stuff in the queue again for no reason
	 */
	wmb();
out:
	mutex_unlock(&memcg_cache_mutex);
	return new_cachep;
}

3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452
void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
{
	struct kmem_cache *c;
	int i;

	if (!s->memcg_params)
		return;
	if (!s->memcg_params->is_root_cache)
		return;

	/*
	 * If the cache is being destroyed, we trust that there is no one else
	 * requesting objects from it. Even if there are, the sanity checks in
	 * kmem_cache_destroy should caught this ill-case.
	 *
	 * Still, we don't want anyone else freeing memcg_caches under our
	 * noses, which can happen if a new memcg comes to life. As usual,
	 * we'll take the set_limit_mutex to protect ourselves against this.
	 */
	mutex_lock(&set_limit_mutex);
	for (i = 0; i < memcg_limited_groups_array_size; i++) {
		c = s->memcg_params->memcg_caches[i];
		if (!c)
			continue;

		/*
		 * We will now manually delete the caches, so to avoid races
		 * we need to cancel all pending destruction workers and
		 * proceed with destruction ourselves.
		 *
		 * kmem_cache_destroy() will call kmem_cache_shrink internally,
		 * and that could spawn the workers again: it is likely that
		 * the cache still have active pages until this very moment.
		 * This would lead us back to mem_cgroup_destroy_cache.
		 *
		 * But that will not execute at all if the "dead" flag is not
		 * set, so flip it down to guarantee we are in control.
		 */
		c->memcg_params->dead = false;
G
Glauber Costa 已提交
3453
		cancel_work_sync(&c->memcg_params->destroy);
3454 3455 3456 3457 3458
		kmem_cache_destroy(c);
	}
	mutex_unlock(&set_limit_mutex);
}

3459 3460 3461 3462 3463 3464
struct create_work {
	struct mem_cgroup *memcg;
	struct kmem_cache *cachep;
	struct work_struct work;
};

G
Glauber Costa 已提交
3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481
static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
{
	struct kmem_cache *cachep;
	struct memcg_cache_params *params;

	if (!memcg_kmem_is_active(memcg))
		return;

	mutex_lock(&memcg->slab_caches_mutex);
	list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
		cachep = memcg_params_to_cache(params);
		cachep->memcg_params->dead = true;
		schedule_work(&cachep->memcg_params->destroy);
	}
	mutex_unlock(&memcg->slab_caches_mutex);
}

3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
static void memcg_create_cache_work_func(struct work_struct *w)
{
	struct create_work *cw;

	cw = container_of(w, struct create_work, work);
	memcg_create_kmem_cache(cw->memcg, cw->cachep);
	kfree(cw);
}

/*
 * Enqueue the creation of a per-memcg kmem_cache.
 */
3494 3495
static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
					 struct kmem_cache *cachep)
3496 3497 3498 3499
{
	struct create_work *cw;

	cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3500 3501
	if (cw == NULL) {
		css_put(&memcg->css);
3502 3503 3504 3505 3506 3507 3508 3509 3510 3511
		return;
	}

	cw->memcg = memcg;
	cw->cachep = cachep;

	INIT_WORK(&cw->work, memcg_create_cache_work_func);
	schedule_work(&cw->work);
}

3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529
static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
				       struct kmem_cache *cachep)
{
	/*
	 * We need to stop accounting when we kmalloc, because if the
	 * corresponding kmalloc cache is not yet created, the first allocation
	 * in __memcg_create_cache_enqueue will recurse.
	 *
	 * However, it is better to enclose the whole function. Depending on
	 * the debugging options enabled, INIT_WORK(), for instance, can
	 * trigger an allocation. This too, will make us recurse. Because at
	 * this point we can't allow ourselves back into memcg_kmem_get_cache,
	 * the safest choice is to do it like this, wrapping the whole function.
	 */
	memcg_stop_kmem_account();
	__memcg_create_cache_enqueue(memcg, cachep);
	memcg_resume_kmem_account();
}
3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
/*
 * Return the kmem_cache we're supposed to use for a slab allocation.
 * We try to use the current memcg's version of the cache.
 *
 * If the cache does not exist yet, if we are the first user of it,
 * we either create it immediately, if possible, or create it asynchronously
 * in a workqueue.
 * In the latter case, we will let the current allocation go through with
 * the original cache.
 *
 * Can't be called in interrupt context or from kernel threads.
 * This function needs to be called with rcu_read_lock() held.
 */
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
					  gfp_t gfp)
{
	struct mem_cgroup *memcg;
	int idx;

	VM_BUG_ON(!cachep->memcg_params);
	VM_BUG_ON(!cachep->memcg_params->is_root_cache);

3552 3553 3554
	if (!current->mm || current->memcg_kmem_skip_account)
		return cachep;

3555 3556 3557 3558
	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));

	if (!memcg_can_account_kmem(memcg))
3559
		goto out;
3560 3561 3562 3563 3564 3565 3566 3567

	idx = memcg_cache_id(memcg);

	/*
	 * barrier to mare sure we're always seeing the up to date value.  The
	 * code updating memcg_caches will issue a write barrier to match this.
	 */
	read_barrier_depends();
3568 3569 3570
	if (likely(cachep->memcg_params->memcg_caches[idx])) {
		cachep = cachep->memcg_params->memcg_caches[idx];
		goto out;
3571 3572
	}

3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599
	/* The corresponding put will be done in the workqueue. */
	if (!css_tryget(&memcg->css))
		goto out;
	rcu_read_unlock();

	/*
	 * If we are in a safe context (can wait, and not in interrupt
	 * context), we could be be predictable and return right away.
	 * This would guarantee that the allocation being performed
	 * already belongs in the new cache.
	 *
	 * However, there are some clashes that can arrive from locking.
	 * For instance, because we acquire the slab_mutex while doing
	 * kmem_cache_dup, this means no further allocation could happen
	 * with the slab_mutex held.
	 *
	 * Also, because cache creation issue get_online_cpus(), this
	 * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
	 * that ends up reversed during cpu hotplug. (cpuset allocates
	 * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
	 * better to defer everything.
	 */
	memcg_create_cache_enqueue(memcg, cachep);
	return cachep;
out:
	rcu_read_unlock();
	return cachep;
3600 3601 3602
}
EXPORT_SYMBOL(__memcg_kmem_get_cache);

3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623
/*
 * We need to verify if the allocation against current->mm->owner's memcg is
 * possible for the given order. But the page is not allocated yet, so we'll
 * need a further commit step to do the final arrangements.
 *
 * It is possible for the task to switch cgroups in this mean time, so at
 * commit time, we can't rely on task conversion any longer.  We'll then use
 * the handle argument to return to the caller which cgroup we should commit
 * against. We could also return the memcg directly and avoid the pointer
 * passing, but a boolean return value gives better semantics considering
 * the compiled-out case as well.
 *
 * Returning true means the allocation is possible.
 */
bool
__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
{
	struct mem_cgroup *memcg;
	int ret;

	*_memcg = NULL;
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651

	/*
	 * Disabling accounting is only relevant for some specific memcg
	 * internal allocations. Therefore we would initially not have such
	 * check here, since direct calls to the page allocator that are marked
	 * with GFP_KMEMCG only happen outside memcg core. We are mostly
	 * concerned with cache allocations, and by having this test at
	 * memcg_kmem_get_cache, we are already able to relay the allocation to
	 * the root cache and bypass the memcg cache altogether.
	 *
	 * There is one exception, though: the SLUB allocator does not create
	 * large order caches, but rather service large kmallocs directly from
	 * the page allocator. Therefore, the following sequence when backed by
	 * the SLUB allocator:
	 *
	 * 	memcg_stop_kmem_account();
	 * 	kmalloc(<large_number>)
	 * 	memcg_resume_kmem_account();
	 *
	 * would effectively ignore the fact that we should skip accounting,
	 * since it will drive us directly to this function without passing
	 * through the cache selector memcg_kmem_get_cache. Such large
	 * allocations are extremely rare but can happen, for instance, for the
	 * cache arrays. We bring this test here.
	 */
	if (!current->mm || current->memcg_kmem_skip_account)
		return true;

3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725
	memcg = try_get_mem_cgroup_from_mm(current->mm);

	/*
	 * very rare case described in mem_cgroup_from_task. Unfortunately there
	 * isn't much we can do without complicating this too much, and it would
	 * be gfp-dependent anyway. Just let it go
	 */
	if (unlikely(!memcg))
		return true;

	if (!memcg_can_account_kmem(memcg)) {
		css_put(&memcg->css);
		return true;
	}

	ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
	if (!ret)
		*_memcg = memcg;

	css_put(&memcg->css);
	return (ret == 0);
}

void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
			      int order)
{
	struct page_cgroup *pc;

	VM_BUG_ON(mem_cgroup_is_root(memcg));

	/* The page allocation failed. Revert */
	if (!page) {
		memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
		return;
	}

	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	pc->mem_cgroup = memcg;
	SetPageCgroupUsed(pc);
	unlock_page_cgroup(pc);
}

void __memcg_kmem_uncharge_pages(struct page *page, int order)
{
	struct mem_cgroup *memcg = NULL;
	struct page_cgroup *pc;


	pc = lookup_page_cgroup(page);
	/*
	 * Fast unlocked return. Theoretically might have changed, have to
	 * check again after locking.
	 */
	if (!PageCgroupUsed(pc))
		return;

	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
		memcg = pc->mem_cgroup;
		ClearPageCgroupUsed(pc);
	}
	unlock_page_cgroup(pc);

	/*
	 * We trust that only if there is a memcg associated with the page, it
	 * is a valid allocation
	 */
	if (!memcg)
		return;

	VM_BUG_ON(mem_cgroup_is_root(memcg));
	memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
}
G
Glauber Costa 已提交
3726 3727 3728 3729
#else
static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
{
}
3730 3731
#endif /* CONFIG_MEMCG_KMEM */

3732 3733
#ifdef CONFIG_TRANSPARENT_HUGEPAGE

3734
#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
3735 3736
/*
 * Because tail pages are not marked as "used", set it. We're under
3737 3738 3739
 * zone->lru_lock, 'splitting on pmd' and compound_lock.
 * charge/uncharge will be never happen and move_account() is done under
 * compound_lock(), so we don't have to take care of races.
3740
 */
3741
void mem_cgroup_split_huge_fixup(struct page *head)
3742 3743
{
	struct page_cgroup *head_pc = lookup_page_cgroup(head);
3744
	struct page_cgroup *pc;
3745
	struct mem_cgroup *memcg;
3746
	int i;
3747

3748 3749
	if (mem_cgroup_disabled())
		return;
3750 3751

	memcg = head_pc->mem_cgroup;
3752 3753
	for (i = 1; i < HPAGE_PMD_NR; i++) {
		pc = head_pc + i;
3754
		pc->mem_cgroup = memcg;
3755 3756 3757
		smp_wmb();/* see __commit_charge() */
		pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
	}
3758 3759
	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
		       HPAGE_PMD_NR);
3760
}
3761
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3762

3763
/**
3764
 * mem_cgroup_move_account - move account of the page
3765
 * @page: the page
3766
 * @nr_pages: number of regular pages (>1 for huge pages)
3767 3768 3769 3770 3771
 * @pc:	page_cgroup of the page.
 * @from: mem_cgroup which the page is moved from.
 * @to:	mem_cgroup which the page is moved to. @from != @to.
 *
 * The caller must confirm following.
K
KAMEZAWA Hiroyuki 已提交
3772
 * - page is not on LRU (isolate_page() is useful.)
3773
 * - compound_lock is held when nr_pages > 1
3774
 *
3775 3776
 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
 * from old cgroup.
3777
 */
3778 3779 3780 3781
static int mem_cgroup_move_account(struct page *page,
				   unsigned int nr_pages,
				   struct page_cgroup *pc,
				   struct mem_cgroup *from,
3782
				   struct mem_cgroup *to)
3783
{
3784 3785
	unsigned long flags;
	int ret;
3786
	bool anon = PageAnon(page);
3787

3788
	VM_BUG_ON(from == to);
3789
	VM_BUG_ON(PageLRU(page));
3790 3791 3792 3793 3794 3795 3796
	/*
	 * The page is isolated from LRU. So, collapse function
	 * will not handle this page. But page splitting can happen.
	 * Do this check under compound_page_lock(). The caller should
	 * hold it.
	 */
	ret = -EBUSY;
3797
	if (nr_pages > 1 && !PageTransHuge(page))
3798 3799 3800 3801 3802 3803 3804 3805
		goto out;

	lock_page_cgroup(pc);

	ret = -EINVAL;
	if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
		goto unlock;

3806
	move_lock_mem_cgroup(from, &flags);
3807

3808
	if (!anon && page_mapped(page)) {
3809 3810 3811 3812 3813
		/* Update mapped_file data for mem_cgroup */
		preempt_disable();
		__this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		__this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
		preempt_enable();
3814
	}
3815
	mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
3816

3817
	/* caller should have done css_get */
K
KAMEZAWA Hiroyuki 已提交
3818
	pc->mem_cgroup = to;
3819
	mem_cgroup_charge_statistics(to, page, anon, nr_pages);
3820
	move_unlock_mem_cgroup(from, &flags);
3821 3822
	ret = 0;
unlock:
3823
	unlock_page_cgroup(pc);
3824 3825 3826
	/*
	 * check events
	 */
3827 3828
	memcg_check_events(to, page);
	memcg_check_events(from, page);
3829
out:
3830 3831 3832
	return ret;
}

3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852
/**
 * mem_cgroup_move_parent - moves page to the parent group
 * @page: the page to move
 * @pc: page_cgroup of the page
 * @child: page's cgroup
 *
 * move charges to its parent or the root cgroup if the group has no
 * parent (aka use_hierarchy==0).
 * Although this might fail (get_page_unless_zero, isolate_lru_page or
 * mem_cgroup_move_account fails) the failure is always temporary and
 * it signals a race with a page removal/uncharge or migration. In the
 * first case the page is on the way out and it will vanish from the LRU
 * on the next attempt and the call should be retried later.
 * Isolation from the LRU fails only if page has been isolated from
 * the LRU since we looked at it and that usually means either global
 * reclaim or migration going on. The page will either get back to the
 * LRU or vanish.
 * Finaly mem_cgroup_move_account fails only if the page got uncharged
 * (!PageCgroupUsed) or moved to a different group. The page will
 * disappear in the next attempt.
3853
 */
3854 3855
static int mem_cgroup_move_parent(struct page *page,
				  struct page_cgroup *pc,
3856
				  struct mem_cgroup *child)
3857 3858
{
	struct mem_cgroup *parent;
3859
	unsigned int nr_pages;
3860
	unsigned long uninitialized_var(flags);
3861 3862
	int ret;

3863
	VM_BUG_ON(mem_cgroup_is_root(child));
3864

3865 3866 3867 3868 3869
	ret = -EBUSY;
	if (!get_page_unless_zero(page))
		goto out;
	if (isolate_lru_page(page))
		goto put;
3870

3871
	nr_pages = hpage_nr_pages(page);
K
KAMEZAWA Hiroyuki 已提交
3872

3873 3874 3875 3876 3877 3878
	parent = parent_mem_cgroup(child);
	/*
	 * If no parent, move charges to root cgroup.
	 */
	if (!parent)
		parent = root_mem_cgroup;
3879

3880 3881
	if (nr_pages > 1) {
		VM_BUG_ON(!PageTransHuge(page));
3882
		flags = compound_lock_irqsave(page);
3883
	}
3884

3885
	ret = mem_cgroup_move_account(page, nr_pages,
3886
				pc, child, parent);
3887 3888
	if (!ret)
		__mem_cgroup_cancel_local_charge(child, nr_pages);
3889

3890
	if (nr_pages > 1)
3891
		compound_unlock_irqrestore(page, flags);
K
KAMEZAWA Hiroyuki 已提交
3892
	putback_lru_page(page);
3893
put:
3894
	put_page(page);
3895
out:
3896 3897 3898
	return ret;
}

3899 3900 3901 3902 3903 3904 3905
/*
 * Charge the memory controller for page usage.
 * Return
 * 0 if the charge was successful
 * < 0 if the cgroup is over its limit
 */
static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
3906
				gfp_t gfp_mask, enum charge_type ctype)
3907
{
3908
	struct mem_cgroup *memcg = NULL;
3909
	unsigned int nr_pages = 1;
3910
	bool oom = true;
3911
	int ret;
A
Andrea Arcangeli 已提交
3912

A
Andrea Arcangeli 已提交
3913
	if (PageTransHuge(page)) {
3914
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
3915
		VM_BUG_ON(!PageTransHuge(page));
3916 3917 3918 3919 3920
		/*
		 * Never OOM-kill a process for a huge page.  The
		 * fault handler will fall back to regular pages.
		 */
		oom = false;
A
Andrea Arcangeli 已提交
3921
	}
3922

3923
	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3924
	if (ret == -ENOMEM)
3925
		return ret;
3926
	__mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3927 3928 3929
	return 0;
}

3930 3931
int mem_cgroup_newpage_charge(struct page *page,
			      struct mm_struct *mm, gfp_t gfp_mask)
3932
{
3933
	if (mem_cgroup_disabled())
3934
		return 0;
3935 3936 3937
	VM_BUG_ON(page_mapped(page));
	VM_BUG_ON(page->mapping && !PageAnon(page));
	VM_BUG_ON(!mm);
3938
	return mem_cgroup_charge_common(page, mm, gfp_mask,
3939
					MEM_CGROUP_CHARGE_TYPE_ANON);
3940 3941
}

3942 3943 3944
/*
 * While swap-in, try_charge -> commit or cancel, the page is locked.
 * And when try_charge() successfully returns, one refcnt to memcg without
3945
 * struct page_cgroup is acquired. This refcnt will be consumed by
3946 3947
 * "commit()" or removed by "cancel()"
 */
3948 3949 3950 3951
static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
					  struct page *page,
					  gfp_t mask,
					  struct mem_cgroup **memcgp)
3952
{
3953
	struct mem_cgroup *memcg;
3954
	struct page_cgroup *pc;
3955
	int ret;
3956

3957 3958 3959 3960 3961 3962 3963 3964 3965 3966
	pc = lookup_page_cgroup(page);
	/*
	 * Every swap fault against a single page tries to charge the
	 * page, bail as early as possible.  shmem_unuse() encounters
	 * already charged pages, too.  The USED bit is protected by
	 * the page lock, which serializes swap cache removal, which
	 * in turn serializes uncharging.
	 */
	if (PageCgroupUsed(pc))
		return 0;
3967 3968
	if (!do_swap_account)
		goto charge_cur_mm;
3969 3970
	memcg = try_get_mem_cgroup_from_page(page);
	if (!memcg)
3971
		goto charge_cur_mm;
3972 3973
	*memcgp = memcg;
	ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
3974
	css_put(&memcg->css);
3975 3976
	if (ret == -EINTR)
		ret = 0;
3977
	return ret;
3978
charge_cur_mm:
3979 3980 3981 3982
	ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
	if (ret == -EINTR)
		ret = 0;
	return ret;
3983 3984
}

3985 3986 3987 3988 3989 3990
int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
				 gfp_t gfp_mask, struct mem_cgroup **memcgp)
{
	*memcgp = NULL;
	if (mem_cgroup_disabled())
		return 0;
3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004
	/*
	 * A racing thread's fault, or swapoff, may have already
	 * updated the pte, and even removed page from swap cache: in
	 * those cases unuse_pte()'s pte_same() test will fail; but
	 * there's also a KSM case which does need to charge the page.
	 */
	if (!PageSwapCache(page)) {
		int ret;

		ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
		if (ret == -EINTR)
			ret = 0;
		return ret;
	}
4005 4006 4007
	return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
}

4008 4009 4010 4011 4012 4013 4014 4015 4016
void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return;
	if (!memcg)
		return;
	__mem_cgroup_cancel_charge(memcg, 1);
}

D
Daisuke Nishimura 已提交
4017
static void
4018
__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
D
Daisuke Nishimura 已提交
4019
					enum charge_type ctype)
4020
{
4021
	if (mem_cgroup_disabled())
4022
		return;
4023
	if (!memcg)
4024
		return;
4025

4026
	__mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
4027 4028 4029
	/*
	 * Now swap is on-memory. This means this page may be
	 * counted both as mem and swap....double count.
4030 4031 4032
	 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
	 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
	 * may call delete_from_swap_cache() before reach here.
4033
	 */
4034
	if (do_swap_account && PageSwapCache(page)) {
4035
		swp_entry_t ent = {.val = page_private(page)};
4036
		mem_cgroup_uncharge_swap(ent);
4037
	}
4038 4039
}

4040 4041
void mem_cgroup_commit_charge_swapin(struct page *page,
				     struct mem_cgroup *memcg)
D
Daisuke Nishimura 已提交
4042
{
4043
	__mem_cgroup_commit_charge_swapin(page, memcg,
4044
					  MEM_CGROUP_CHARGE_TYPE_ANON);
D
Daisuke Nishimura 已提交
4045 4046
}

4047 4048
int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
				gfp_t gfp_mask)
4049
{
4050 4051 4052 4053
	struct mem_cgroup *memcg = NULL;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
	int ret;

4054
	if (mem_cgroup_disabled())
4055 4056 4057 4058 4059 4060 4061
		return 0;
	if (PageCompound(page))
		return 0;

	if (!PageSwapCache(page))
		ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
	else { /* page is swapcache/shmem */
4062 4063
		ret = __mem_cgroup_try_charge_swapin(mm, page,
						     gfp_mask, &memcg);
4064 4065 4066 4067
		if (!ret)
			__mem_cgroup_commit_charge_swapin(page, memcg, type);
	}
	return ret;
4068 4069
}

4070
static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
4071 4072
				   unsigned int nr_pages,
				   const enum charge_type ctype)
4073 4074 4075
{
	struct memcg_batch_info *batch = NULL;
	bool uncharge_memsw = true;
4076

4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087
	/* If swapout, usage of swap doesn't decrease */
	if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
		uncharge_memsw = false;

	batch = &current->memcg_batch;
	/*
	 * In usual, we do css_get() when we remember memcg pointer.
	 * But in this case, we keep res->usage until end of a series of
	 * uncharges. Then, it's ok to ignore memcg's refcnt.
	 */
	if (!batch->memcg)
4088
		batch->memcg = memcg;
4089 4090
	/*
	 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
L
Lucas De Marchi 已提交
4091
	 * In those cases, all pages freed continuously can be expected to be in
4092 4093 4094 4095 4096 4097 4098 4099
	 * the same cgroup and we have chance to coalesce uncharges.
	 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
	 * because we want to do uncharge as soon as possible.
	 */

	if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
		goto direct_uncharge;

4100
	if (nr_pages > 1)
A
Andrea Arcangeli 已提交
4101 4102
		goto direct_uncharge;

4103 4104 4105 4106 4107
	/*
	 * In typical case, batch->memcg == mem. This means we can
	 * merge a series of uncharges to an uncharge of res_counter.
	 * If not, we uncharge res_counter ony by one.
	 */
4108
	if (batch->memcg != memcg)
4109 4110
		goto direct_uncharge;
	/* remember freed charge and uncharge it later */
4111
	batch->nr_pages++;
4112
	if (uncharge_memsw)
4113
		batch->memsw_nr_pages++;
4114 4115
	return;
direct_uncharge:
4116
	res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
4117
	if (uncharge_memsw)
4118 4119 4120
		res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
	if (unlikely(batch->memcg != memcg))
		memcg_oom_recover(memcg);
4121
}
4122

4123
/*
4124
 * uncharge if !page_mapped(page)
4125
 */
4126
static struct mem_cgroup *
4127 4128
__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
			     bool end_migration)
4129
{
4130
	struct mem_cgroup *memcg = NULL;
4131 4132
	unsigned int nr_pages = 1;
	struct page_cgroup *pc;
4133
	bool anon;
4134

4135
	if (mem_cgroup_disabled())
4136
		return NULL;
4137

A
Andrea Arcangeli 已提交
4138
	if (PageTransHuge(page)) {
4139
		nr_pages <<= compound_order(page);
A
Andrea Arcangeli 已提交
4140 4141
		VM_BUG_ON(!PageTransHuge(page));
	}
4142
	/*
4143
	 * Check if our page_cgroup is valid
4144
	 */
4145
	pc = lookup_page_cgroup(page);
4146
	if (unlikely(!PageCgroupUsed(pc)))
4147
		return NULL;
4148

4149
	lock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
4150

4151
	memcg = pc->mem_cgroup;
4152

K
KAMEZAWA Hiroyuki 已提交
4153 4154 4155
	if (!PageCgroupUsed(pc))
		goto unlock_out;

4156 4157
	anon = PageAnon(page);

K
KAMEZAWA Hiroyuki 已提交
4158
	switch (ctype) {
4159
	case MEM_CGROUP_CHARGE_TYPE_ANON:
4160 4161 4162 4163 4164
		/*
		 * Generally PageAnon tells if it's the anon statistics to be
		 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
		 * used before page reached the stage of being marked PageAnon.
		 */
4165 4166
		anon = true;
		/* fallthrough */
K
KAMEZAWA Hiroyuki 已提交
4167
	case MEM_CGROUP_CHARGE_TYPE_DROP:
4168
		/* See mem_cgroup_prepare_migration() */
4169 4170 4171 4172 4173 4174 4175 4176 4177 4178
		if (page_mapped(page))
			goto unlock_out;
		/*
		 * Pages under migration may not be uncharged.  But
		 * end_migration() /must/ be the one uncharging the
		 * unused post-migration page and so it has to call
		 * here with the migration bit still set.  See the
		 * res_counter handling below.
		 */
		if (!end_migration && PageCgroupMigration(pc))
K
KAMEZAWA Hiroyuki 已提交
4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189
			goto unlock_out;
		break;
	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
		if (!PageAnon(page)) {	/* Shared memory */
			if (page->mapping && !page_is_file_cache(page))
				goto unlock_out;
		} else if (page_mapped(page)) /* Anon */
				goto unlock_out;
		break;
	default:
		break;
4190
	}
K
KAMEZAWA Hiroyuki 已提交
4191

4192
	mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
K
KAMEZAWA Hiroyuki 已提交
4193

4194
	ClearPageCgroupUsed(pc);
4195 4196 4197 4198 4199 4200
	/*
	 * pc->mem_cgroup is not cleared here. It will be accessed when it's
	 * freed from LRU. This is safe because uncharged page is expected not
	 * to be reused (freed soon). Exception is SwapCache, it's handled by
	 * special functions.
	 */
4201

4202
	unlock_page_cgroup(pc);
K
KAMEZAWA Hiroyuki 已提交
4203
	/*
4204
	 * even after unlock, we have memcg->res.usage here and this memcg
L
Li Zefan 已提交
4205
	 * will never be freed, so it's safe to call css_get().
K
KAMEZAWA Hiroyuki 已提交
4206
	 */
4207
	memcg_check_events(memcg, page);
K
KAMEZAWA Hiroyuki 已提交
4208
	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4209
		mem_cgroup_swap_statistics(memcg, true);
L
Li Zefan 已提交
4210
		css_get(&memcg->css);
K
KAMEZAWA Hiroyuki 已提交
4211
	}
4212 4213 4214 4215 4216 4217
	/*
	 * Migration does not charge the res_counter for the
	 * replacement page, so leave it alone when phasing out the
	 * page that is unused after the migration.
	 */
	if (!end_migration && !mem_cgroup_is_root(memcg))
4218
		mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4219

4220
	return memcg;
K
KAMEZAWA Hiroyuki 已提交
4221 4222 4223

unlock_out:
	unlock_page_cgroup(pc);
4224
	return NULL;
4225 4226
}

4227 4228
void mem_cgroup_uncharge_page(struct page *page)
{
4229 4230 4231
	/* early check. */
	if (page_mapped(page))
		return;
4232
	VM_BUG_ON(page->mapping && !PageAnon(page));
4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244
	/*
	 * If the page is in swap cache, uncharge should be deferred
	 * to the swap path, which also properly accounts swap usage
	 * and handles memcg lifetime.
	 *
	 * Note that this check is not stable and reclaim may add the
	 * page to swap cache at any time after this.  However, if the
	 * page is not in swap cache by the time page->mapcount hits
	 * 0, there won't be any page table references to the swap
	 * slot, and reclaim will free it and not actually write the
	 * page to disk.
	 */
4245 4246
	if (PageSwapCache(page))
		return;
4247
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4248 4249 4250 4251 4252
}

void mem_cgroup_uncharge_cache_page(struct page *page)
{
	VM_BUG_ON(page_mapped(page));
4253
	VM_BUG_ON(page->mapping);
4254
	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4255 4256
}

4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270
/*
 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
 * In that cases, pages are freed continuously and we can expect pages
 * are in the same memcg. All these calls itself limits the number of
 * pages freed at once, then uncharge_start/end() is called properly.
 * This may be called prural(2) times in a context,
 */

void mem_cgroup_uncharge_start(void)
{
	current->memcg_batch.do_batch++;
	/* We can do nest. */
	if (current->memcg_batch.do_batch == 1) {
		current->memcg_batch.memcg = NULL;
4271 4272
		current->memcg_batch.nr_pages = 0;
		current->memcg_batch.memsw_nr_pages = 0;
4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292
	}
}

void mem_cgroup_uncharge_end(void)
{
	struct memcg_batch_info *batch = &current->memcg_batch;

	if (!batch->do_batch)
		return;

	batch->do_batch--;
	if (batch->do_batch) /* If stacked, do nothing. */
		return;

	if (!batch->memcg)
		return;
	/*
	 * This "batch->memcg" is valid without any css_get/put etc...
	 * bacause we hide charges behind us.
	 */
4293 4294 4295 4296 4297 4298
	if (batch->nr_pages)
		res_counter_uncharge(&batch->memcg->res,
				     batch->nr_pages * PAGE_SIZE);
	if (batch->memsw_nr_pages)
		res_counter_uncharge(&batch->memcg->memsw,
				     batch->memsw_nr_pages * PAGE_SIZE);
4299
	memcg_oom_recover(batch->memcg);
4300 4301 4302 4303
	/* forget this pointer (for sanity check) */
	batch->memcg = NULL;
}

4304
#ifdef CONFIG_SWAP
4305
/*
4306
 * called after __delete_from_swap_cache() and drop "page" account.
4307 4308
 * memcg information is recorded to swap_cgroup of "ent"
 */
K
KAMEZAWA Hiroyuki 已提交
4309 4310
void
mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4311 4312
{
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
4313 4314 4315 4316 4317
	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;

	if (!swapout) /* this was a swap cache but the swap is unused ! */
		ctype = MEM_CGROUP_CHARGE_TYPE_DROP;

4318
	memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4319

K
KAMEZAWA Hiroyuki 已提交
4320 4321
	/*
	 * record memcg information,  if swapout && memcg != NULL,
L
Li Zefan 已提交
4322
	 * css_get() was called in uncharge().
K
KAMEZAWA Hiroyuki 已提交
4323 4324
	 */
	if (do_swap_account && swapout && memcg)
4325
		swap_cgroup_record(ent, css_id(&memcg->css));
4326
}
4327
#endif
4328

A
Andrew Morton 已提交
4329
#ifdef CONFIG_MEMCG_SWAP
4330 4331 4332 4333 4334
/*
 * called from swap_entry_free(). remove record in swap_cgroup and
 * uncharge "memsw" account.
 */
void mem_cgroup_uncharge_swap(swp_entry_t ent)
K
KAMEZAWA Hiroyuki 已提交
4335
{
4336
	struct mem_cgroup *memcg;
4337
	unsigned short id;
4338 4339 4340 4341

	if (!do_swap_account)
		return;

4342 4343 4344
	id = swap_cgroup_record(ent, 0);
	rcu_read_lock();
	memcg = mem_cgroup_lookup(id);
4345
	if (memcg) {
4346 4347 4348 4349
		/*
		 * We uncharge this because swap is freed.
		 * This memcg can be obsolete one. We avoid calling css_tryget
		 */
4350
		if (!mem_cgroup_is_root(memcg))
4351
			res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4352
		mem_cgroup_swap_statistics(memcg, false);
L
Li Zefan 已提交
4353
		css_put(&memcg->css);
4354
	}
4355
	rcu_read_unlock();
K
KAMEZAWA Hiroyuki 已提交
4356
}
4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372

/**
 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
 * @entry: swap entry to be moved
 * @from:  mem_cgroup which the entry is moved from
 * @to:  mem_cgroup which the entry is moved to
 *
 * It succeeds only when the swap_cgroup's record for this entry is the same
 * as the mem_cgroup's id of @from.
 *
 * Returns 0 on success, -EINVAL on failure.
 *
 * The caller must have charged to @to, IOW, called res_counter_charge() about
 * both res and memsw, and called css_get().
 */
static int mem_cgroup_move_swap_account(swp_entry_t entry,
4373
				struct mem_cgroup *from, struct mem_cgroup *to)
4374 4375 4376 4377 4378 4379 4380 4381
{
	unsigned short old_id, new_id;

	old_id = css_id(&from->css);
	new_id = css_id(&to->css);

	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
		mem_cgroup_swap_statistics(from, false);
4382
		mem_cgroup_swap_statistics(to, true);
4383
		/*
4384 4385 4386
		 * This function is only called from task migration context now.
		 * It postpones res_counter and refcount handling till the end
		 * of task migration(mem_cgroup_clear_mc()) for performance
L
Li Zefan 已提交
4387 4388 4389 4390 4391 4392
		 * improvement. But we cannot postpone css_get(to)  because if
		 * the process that has been moved to @to does swap-in, the
		 * refcount of @to might be decreased to 0.
		 *
		 * We are in attach() phase, so the cgroup is guaranteed to be
		 * alive, so we can just call css_get().
4393
		 */
L
Li Zefan 已提交
4394
		css_get(&to->css);
4395 4396 4397 4398 4399 4400
		return 0;
	}
	return -EINVAL;
}
#else
static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4401
				struct mem_cgroup *from, struct mem_cgroup *to)
4402 4403 4404
{
	return -EINVAL;
}
4405
#endif
K
KAMEZAWA Hiroyuki 已提交
4406

4407
/*
4408 4409
 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
 * page belongs to.
4410
 */
4411 4412
void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
				  struct mem_cgroup **memcgp)
4413
{
4414
	struct mem_cgroup *memcg = NULL;
4415
	unsigned int nr_pages = 1;
4416
	struct page_cgroup *pc;
4417
	enum charge_type ctype;
4418

4419
	*memcgp = NULL;
4420

4421
	if (mem_cgroup_disabled())
4422
		return;
4423

4424 4425 4426
	if (PageTransHuge(page))
		nr_pages <<= compound_order(page);

4427 4428 4429
	pc = lookup_page_cgroup(page);
	lock_page_cgroup(pc);
	if (PageCgroupUsed(pc)) {
4430 4431
		memcg = pc->mem_cgroup;
		css_get(&memcg->css);
4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462
		/*
		 * At migrating an anonymous page, its mapcount goes down
		 * to 0 and uncharge() will be called. But, even if it's fully
		 * unmapped, migration may fail and this page has to be
		 * charged again. We set MIGRATION flag here and delay uncharge
		 * until end_migration() is called
		 *
		 * Corner Case Thinking
		 * A)
		 * When the old page was mapped as Anon and it's unmap-and-freed
		 * while migration was ongoing.
		 * If unmap finds the old page, uncharge() of it will be delayed
		 * until end_migration(). If unmap finds a new page, it's
		 * uncharged when it make mapcount to be 1->0. If unmap code
		 * finds swap_migration_entry, the new page will not be mapped
		 * and end_migration() will find it(mapcount==0).
		 *
		 * B)
		 * When the old page was mapped but migraion fails, the kernel
		 * remaps it. A charge for it is kept by MIGRATION flag even
		 * if mapcount goes down to 0. We can do remap successfully
		 * without charging it again.
		 *
		 * C)
		 * The "old" page is under lock_page() until the end of
		 * migration, so, the old page itself will not be swapped-out.
		 * If the new page is swapped out before end_migraton, our
		 * hook to usual swap-out path will catch the event.
		 */
		if (PageAnon(page))
			SetPageCgroupMigration(pc);
4463
	}
4464
	unlock_page_cgroup(pc);
4465 4466 4467 4468
	/*
	 * If the page is not charged at this point,
	 * we return here.
	 */
4469
	if (!memcg)
4470
		return;
4471

4472
	*memcgp = memcg;
4473 4474 4475 4476 4477 4478 4479
	/*
	 * We charge new page before it's used/mapped. So, even if unlock_page()
	 * is called before end_migration, we can catch all events on this new
	 * page. In the case new page is migrated but not remapped, new page's
	 * mapcount will be finally 0 and we call uncharge in end_migration().
	 */
	if (PageAnon(page))
4480
		ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4481
	else
4482
		ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4483 4484 4485 4486 4487
	/*
	 * The page is committed to the memcg, but it's not actually
	 * charged to the res_counter since we plan on replacing the
	 * old one and only one page is going to be left afterwards.
	 */
4488
	__mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4489
}
4490

4491
/* remove redundant charge if migration failed*/
4492
void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4493
	struct page *oldpage, struct page *newpage, bool migration_ok)
4494
{
4495
	struct page *used, *unused;
4496
	struct page_cgroup *pc;
4497
	bool anon;
4498

4499
	if (!memcg)
4500
		return;
4501

4502
	if (!migration_ok) {
4503 4504
		used = oldpage;
		unused = newpage;
4505
	} else {
4506
		used = newpage;
4507 4508
		unused = oldpage;
	}
4509
	anon = PageAnon(used);
4510 4511 4512 4513
	__mem_cgroup_uncharge_common(unused,
				     anon ? MEM_CGROUP_CHARGE_TYPE_ANON
				     : MEM_CGROUP_CHARGE_TYPE_CACHE,
				     true);
4514
	css_put(&memcg->css);
4515
	/*
4516 4517 4518
	 * We disallowed uncharge of pages under migration because mapcount
	 * of the page goes down to zero, temporarly.
	 * Clear the flag and check the page should be charged.
4519
	 */
4520 4521 4522 4523 4524
	pc = lookup_page_cgroup(oldpage);
	lock_page_cgroup(pc);
	ClearPageCgroupMigration(pc);
	unlock_page_cgroup(pc);

4525
	/*
4526 4527 4528 4529 4530 4531
	 * If a page is a file cache, radix-tree replacement is very atomic
	 * and we can skip this check. When it was an Anon page, its mapcount
	 * goes down to 0. But because we added MIGRATION flage, it's not
	 * uncharged yet. There are several case but page->mapcount check
	 * and USED bit check in mem_cgroup_uncharge_page() will do enough
	 * check. (see prepare_charge() also)
4532
	 */
4533
	if (anon)
4534
		mem_cgroup_uncharge_page(used);
4535
}
4536

4537 4538 4539 4540 4541 4542 4543 4544
/*
 * At replace page cache, newpage is not under any memcg but it's on
 * LRU. So, this function doesn't touch res_counter but handles LRU
 * in correct way. Both pages are locked so we cannot race with uncharge.
 */
void mem_cgroup_replace_page_cache(struct page *oldpage,
				  struct page *newpage)
{
4545
	struct mem_cgroup *memcg = NULL;
4546 4547 4548 4549 4550 4551 4552 4553 4554
	struct page_cgroup *pc;
	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;

	if (mem_cgroup_disabled())
		return;

	pc = lookup_page_cgroup(oldpage);
	/* fix accounting on old pages */
	lock_page_cgroup(pc);
4555 4556
	if (PageCgroupUsed(pc)) {
		memcg = pc->mem_cgroup;
4557
		mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
4558 4559
		ClearPageCgroupUsed(pc);
	}
4560 4561
	unlock_page_cgroup(pc);

4562 4563 4564 4565 4566 4567
	/*
	 * When called from shmem_replace_page(), in some cases the
	 * oldpage has already been charged, and in some cases not.
	 */
	if (!memcg)
		return;
4568 4569 4570 4571 4572
	/*
	 * Even if newpage->mapping was NULL before starting replacement,
	 * the newpage may be on LRU(or pagevec for LRU) already. We lock
	 * LRU while we overwrite pc->mem_cgroup.
	 */
4573
	__mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4574 4575
}

4576 4577 4578 4579 4580 4581
#ifdef CONFIG_DEBUG_VM
static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup(page);
4582 4583 4584 4585 4586
	/*
	 * Can be NULL while feeding pages into the page allocator for
	 * the first time, i.e. during boot or memory hotplug;
	 * or when mem_cgroup_disabled().
	 */
4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605
	if (likely(pc) && PageCgroupUsed(pc))
		return pc;
	return NULL;
}

bool mem_cgroup_bad_page_check(struct page *page)
{
	if (mem_cgroup_disabled())
		return false;

	return lookup_page_cgroup_used(page) != NULL;
}

void mem_cgroup_print_bad_page(struct page *page)
{
	struct page_cgroup *pc;

	pc = lookup_page_cgroup_used(page);
	if (pc) {
4606 4607
		pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
			 pc, pc->flags, pc->mem_cgroup);
4608 4609 4610 4611
	}
}
#endif

4612
static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4613
				unsigned long long val)
4614
{
4615
	int retry_count;
4616
	u64 memswlimit, memlimit;
4617
	int ret = 0;
4618 4619
	int children = mem_cgroup_count_children(memcg);
	u64 curusage, oldusage;
4620
	int enlarge;
4621 4622 4623 4624 4625 4626 4627 4628 4629

	/*
	 * For keeping hierarchical_reclaim simple, how long we should retry
	 * is depends on callers. We set our retry-count to be function
	 * of # of children which we should visit in this loop.
	 */
	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;

	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4630

4631
	enlarge = 0;
4632
	while (retry_count) {
4633 4634 4635 4636
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
4637 4638 4639
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
4640
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4641 4642 4643 4644 4645 4646
		 */
		mutex_lock(&set_limit_mutex);
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
4647 4648
			break;
		}
4649 4650 4651 4652 4653

		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit < val)
			enlarge = 1;

4654
		ret = res_counter_set_limit(&memcg->res, val);
4655 4656 4657 4658 4659 4660
		if (!ret) {
			if (memswlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
4661 4662 4663 4664 4665
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

4666 4667
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_SHRINK);
4668 4669 4670 4671 4672 4673
		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
		/* Usage is reduced ? */
  		if (curusage >= oldusage)
			retry_count--;
		else
			oldusage = curusage;
4674
	}
4675 4676
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
4677

4678 4679 4680
	return ret;
}

L
Li Zefan 已提交
4681 4682
static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
					unsigned long long val)
4683
{
4684
	int retry_count;
4685
	u64 memlimit, memswlimit, oldusage, curusage;
4686 4687
	int children = mem_cgroup_count_children(memcg);
	int ret = -EBUSY;
4688
	int enlarge = 0;
4689

4690 4691 4692
	/* see mem_cgroup_resize_res_limit */
 	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4693 4694 4695 4696 4697 4698 4699 4700
	while (retry_count) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		/*
		 * Rather than hide all in some function, I do this in
		 * open coded manner. You see what this really does.
4701
		 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4702 4703 4704 4705 4706 4707 4708 4709
		 */
		mutex_lock(&set_limit_mutex);
		memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
		if (memlimit > val) {
			ret = -EINVAL;
			mutex_unlock(&set_limit_mutex);
			break;
		}
4710 4711 4712
		memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		if (memswlimit < val)
			enlarge = 1;
4713
		ret = res_counter_set_limit(&memcg->memsw, val);
4714 4715 4716 4717 4718 4719
		if (!ret) {
			if (memlimit == val)
				memcg->memsw_is_minimum = true;
			else
				memcg->memsw_is_minimum = false;
		}
4720 4721 4722 4723 4724
		mutex_unlock(&set_limit_mutex);

		if (!ret)
			break;

4725 4726 4727
		mem_cgroup_reclaim(memcg, GFP_KERNEL,
				   MEM_CGROUP_RECLAIM_NOSWAP |
				   MEM_CGROUP_RECLAIM_SHRINK);
4728
		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4729
		/* Usage is reduced ? */
4730
		if (curusage >= oldusage)
4731
			retry_count--;
4732 4733
		else
			oldusage = curusage;
4734
	}
4735 4736
	if (!ret && enlarge)
		memcg_oom_recover(memcg);
4737 4738 4739
	return ret;
}

4740
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4741 4742
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
4743 4744 4745 4746 4747 4748
{
	unsigned long nr_reclaimed = 0;
	struct mem_cgroup_per_zone *mz, *next_mz = NULL;
	unsigned long reclaimed;
	int loop = 0;
	struct mem_cgroup_tree_per_zone *mctz;
4749
	unsigned long long excess;
4750
	unsigned long nr_scanned;
4751 4752 4753 4754

	if (order > 0)
		return 0;

4755
	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768
	/*
	 * This loop can run a while, specially if mem_cgroup's continuously
	 * keep exceeding their soft limit and putting the system under
	 * pressure
	 */
	do {
		if (next_mz)
			mz = next_mz;
		else
			mz = mem_cgroup_largest_soft_limit_node(mctz);
		if (!mz)
			break;

4769
		nr_scanned = 0;
4770
		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4771
						    gfp_mask, &nr_scanned);
4772
		nr_reclaimed += reclaimed;
4773
		*total_scanned += nr_scanned;
4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795
		spin_lock(&mctz->lock);

		/*
		 * If we failed to reclaim anything from this memory cgroup
		 * it is time to move on to the next cgroup
		 */
		next_mz = NULL;
		if (!reclaimed) {
			do {
				/*
				 * Loop until we find yet another one.
				 *
				 * By the time we get the soft_limit lock
				 * again, someone might have aded the
				 * group back on the RB tree. Iterate to
				 * make sure we get a different mem.
				 * mem_cgroup_largest_soft_limit_node returns
				 * NULL if no other cgroup is present on
				 * the tree
				 */
				next_mz =
				__mem_cgroup_largest_soft_limit_node(mctz);
4796
				if (next_mz == mz)
4797
					css_put(&next_mz->memcg->css);
4798
				else /* next_mz == NULL or other memcg */
4799 4800 4801
					break;
			} while (1);
		}
4802 4803
		__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
		excess = res_counter_soft_limit_excess(&mz->memcg->res);
4804 4805 4806 4807 4808 4809 4810 4811
		/*
		 * One school of thought says that we should not add
		 * back the node to the tree if reclaim returns 0.
		 * But our reclaim could return 0, simply because due
		 * to priority we are exposing a smaller subset of
		 * memory to reclaim from. Consider this as a longer
		 * term TODO.
		 */
4812
		/* If excess == 0, no tree ops */
4813
		__mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4814
		spin_unlock(&mctz->lock);
4815
		css_put(&mz->memcg->css);
4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827
		loop++;
		/*
		 * Could not reclaim anything and there are no more
		 * mem cgroups to try or we seem to be looping without
		 * reclaiming anything.
		 */
		if (!nr_reclaimed &&
			(next_mz == NULL ||
			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
			break;
	} while (!nr_reclaimed);
	if (next_mz)
4828
		css_put(&next_mz->memcg->css);
4829 4830 4831
	return nr_reclaimed;
}

4832 4833 4834 4835 4836 4837 4838
/**
 * mem_cgroup_force_empty_list - clears LRU of a group
 * @memcg: group to clear
 * @node: NUMA node
 * @zid: zone id
 * @lru: lru to to clear
 *
4839
 * Traverse a specified page_cgroup list and try to drop them all.  This doesn't
4840 4841
 * reclaim the pages page themselves - pages are moved to the parent (or root)
 * group.
4842
 */
4843
static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
K
KAMEZAWA Hiroyuki 已提交
4844
				int node, int zid, enum lru_list lru)
4845
{
4846
	struct lruvec *lruvec;
4847
	unsigned long flags;
4848
	struct list_head *list;
4849 4850
	struct page *busy;
	struct zone *zone;
4851

K
KAMEZAWA Hiroyuki 已提交
4852
	zone = &NODE_DATA(node)->node_zones[zid];
4853 4854
	lruvec = mem_cgroup_zone_lruvec(zone, memcg);
	list = &lruvec->lists[lru];
4855

4856
	busy = NULL;
4857
	do {
4858
		struct page_cgroup *pc;
4859 4860
		struct page *page;

K
KAMEZAWA Hiroyuki 已提交
4861
		spin_lock_irqsave(&zone->lru_lock, flags);
4862
		if (list_empty(list)) {
K
KAMEZAWA Hiroyuki 已提交
4863
			spin_unlock_irqrestore(&zone->lru_lock, flags);
4864
			break;
4865
		}
4866 4867 4868
		page = list_entry(list->prev, struct page, lru);
		if (busy == page) {
			list_move(&page->lru, list);
4869
			busy = NULL;
K
KAMEZAWA Hiroyuki 已提交
4870
			spin_unlock_irqrestore(&zone->lru_lock, flags);
4871 4872
			continue;
		}
K
KAMEZAWA Hiroyuki 已提交
4873
		spin_unlock_irqrestore(&zone->lru_lock, flags);
4874

4875
		pc = lookup_page_cgroup(page);
4876

4877
		if (mem_cgroup_move_parent(page, pc, memcg)) {
4878
			/* found lock contention or "pc" is obsolete. */
4879
			busy = page;
4880 4881 4882
			cond_resched();
		} else
			busy = NULL;
4883
	} while (!list_empty(list));
4884 4885 4886
}

/*
4887 4888
 * make mem_cgroup's charge to be 0 if there is no task by moving
 * all the charges and pages to the parent.
4889
 * This enables deleting this mem_cgroup.
4890 4891
 *
 * Caller is responsible for holding css reference on the memcg.
4892
 */
4893
static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4894
{
4895
	int node, zid;
4896
	u64 usage;
4897

4898
	do {
4899 4900
		/* This is for making all *used* pages to be on LRU. */
		lru_add_drain_all();
4901 4902
		drain_all_stock_sync(memcg);
		mem_cgroup_start_move(memcg);
4903
		for_each_node_state(node, N_MEMORY) {
4904
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
H
Hugh Dickins 已提交
4905 4906
				enum lru_list lru;
				for_each_lru(lru) {
4907
					mem_cgroup_force_empty_list(memcg,
H
Hugh Dickins 已提交
4908
							node, zid, lru);
4909
				}
4910
			}
4911
		}
4912 4913
		mem_cgroup_end_move(memcg);
		memcg_oom_recover(memcg);
4914
		cond_resched();
4915

4916
		/*
4917 4918 4919 4920 4921
		 * Kernel memory may not necessarily be trackable to a specific
		 * process. So they are not migrated, and therefore we can't
		 * expect their value to drop to 0 here.
		 * Having res filled up with kmem only is enough.
		 *
4922 4923 4924 4925 4926 4927
		 * This is a safety check because mem_cgroup_force_empty_list
		 * could have raced with mem_cgroup_replace_page_cache callers
		 * so the lru seemed empty but the page could have been added
		 * right after the check. RES_USAGE should be safe as we always
		 * charge before adding to the LRU.
		 */
4928 4929 4930
		usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
			res_counter_read_u64(&memcg->kmem, RES_USAGE);
	} while (usage > 0);
4931 4932
}

4933 4934 4935 4936 4937 4938 4939
/*
 * This mainly exists for tests during the setting of set of use_hierarchy.
 * Since this is the very setting we are changing, the current hierarchy value
 * is meaningless
 */
static inline bool __memcg_has_children(struct mem_cgroup *memcg)
{
4940
	struct cgroup_subsys_state *pos;
4941 4942

	/* bounce at first found */
4943
	css_for_each_child(pos, &memcg->css)
4944 4945 4946 4947 4948
		return true;
	return false;
}

/*
4949 4950
 * Must be called with memcg_create_mutex held, unless the cgroup is guaranteed
 * to be already dead (as in mem_cgroup_force_empty, for instance).  This is
4951 4952 4953 4954 4955 4956 4957 4958 4959
 * from mem_cgroup_count_children(), in the sense that we don't really care how
 * many children we have; we only need to know if we have any.  It also counts
 * any memcg without hierarchy as infertile.
 */
static inline bool memcg_has_children(struct mem_cgroup *memcg)
{
	return memcg->use_hierarchy && __memcg_has_children(memcg);
}

4960 4961 4962 4963 4964 4965 4966 4967 4968 4969
/*
 * Reclaims as many pages from the given memcg as possible and moves
 * the rest to the parent.
 *
 * Caller is responsible for holding css reference for memcg.
 */
static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
{
	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
	struct cgroup *cgrp = memcg->css.cgroup;
4970

4971
	/* returns EBUSY if there is a task or if we come here twice. */
4972 4973 4974
	if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
		return -EBUSY;

4975 4976
	/* we call try-to-free pages for make this cgroup empty */
	lru_add_drain_all();
4977
	/* try to free all pages in this cgroup */
4978
	while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
4979
		int progress;
4980

4981 4982 4983
		if (signal_pending(current))
			return -EINTR;

4984
		progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
4985
						false);
4986
		if (!progress) {
4987
			nr_retries--;
4988
			/* maybe some writeback is necessary */
4989
			congestion_wait(BLK_RW_ASYNC, HZ/10);
4990
		}
4991 4992

	}
K
KAMEZAWA Hiroyuki 已提交
4993
	lru_add_drain();
4994 4995 4996
	mem_cgroup_reparent_charges(memcg);

	return 0;
4997 4998
}

4999 5000
static int mem_cgroup_force_empty_write(struct cgroup_subsys_state *css,
					unsigned int event)
5001
{
5002
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5003 5004
	int ret;

5005 5006
	if (mem_cgroup_is_root(memcg))
		return -EINVAL;
5007 5008 5009 5010 5011
	css_get(&memcg->css);
	ret = mem_cgroup_force_empty(memcg);
	css_put(&memcg->css);

	return ret;
5012 5013 5014
}


5015 5016
static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
				     struct cftype *cft)
5017
{
5018
	return mem_cgroup_from_css(css)->use_hierarchy;
5019 5020
}

5021 5022
static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
				      struct cftype *cft, u64 val)
5023 5024
{
	int retval = 0;
5025
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
5026
	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5027

5028
	mutex_lock(&memcg_create_mutex);
5029 5030 5031 5032

	if (memcg->use_hierarchy == val)
		goto out;

5033
	/*
5034
	 * If parent's use_hierarchy is set, we can't make any modifications
5035 5036 5037 5038 5039 5040
	 * in the child subtrees. If it is unset, then the change can
	 * occur, provided the current cgroup has no children.
	 *
	 * For the root cgroup, parent_mem is NULL, we allow value to be
	 * set if there are no children.
	 */
5041
	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
5042
				(val == 1 || val == 0)) {
5043
		if (!__memcg_has_children(memcg))
5044
			memcg->use_hierarchy = val;
5045 5046 5047 5048
		else
			retval = -EBUSY;
	} else
		retval = -EINVAL;
5049 5050

out:
5051
	mutex_unlock(&memcg_create_mutex);
5052 5053 5054 5055

	return retval;
}

5056

5057
static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
5058
					       enum mem_cgroup_stat_index idx)
5059
{
K
KAMEZAWA Hiroyuki 已提交
5060
	struct mem_cgroup *iter;
5061
	long val = 0;
5062

5063
	/* Per-cpu values can be negative, use a signed accumulator */
5064
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
5065 5066 5067 5068 5069
		val += mem_cgroup_read_stat(iter, idx);

	if (val < 0) /* race ? */
		val = 0;
	return val;
5070 5071
}

5072
static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
5073
{
K
KAMEZAWA Hiroyuki 已提交
5074
	u64 val;
5075

5076
	if (!mem_cgroup_is_root(memcg)) {
5077
		if (!swap)
5078
			return res_counter_read_u64(&memcg->res, RES_USAGE);
5079
		else
5080
			return res_counter_read_u64(&memcg->memsw, RES_USAGE);
5081 5082
	}

5083 5084 5085 5086
	/*
	 * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
	 * as well as in MEM_CGROUP_STAT_RSS_HUGE.
	 */
5087 5088
	val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
5089

K
KAMEZAWA Hiroyuki 已提交
5090
	if (swap)
5091
		val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
5092 5093 5094 5095

	return val << PAGE_SHIFT;
}

5096 5097 5098
static ssize_t mem_cgroup_read(struct cgroup_subsys_state *css,
			       struct cftype *cft, struct file *file,
			       char __user *buf, size_t nbytes, loff_t *ppos)
B
Balbir Singh 已提交
5099
{
5100
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5101
	char str[64];
5102
	u64 val;
G
Glauber Costa 已提交
5103 5104
	int name, len;
	enum res_type type;
5105 5106 5107

	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
5108

5109 5110
	switch (type) {
	case _MEM:
5111
		if (name == RES_USAGE)
5112
			val = mem_cgroup_usage(memcg, false);
5113
		else
5114
			val = res_counter_read_u64(&memcg->res, name);
5115 5116
		break;
	case _MEMSWAP:
5117
		if (name == RES_USAGE)
5118
			val = mem_cgroup_usage(memcg, true);
5119
		else
5120
			val = res_counter_read_u64(&memcg->memsw, name);
5121
		break;
5122 5123 5124
	case _KMEM:
		val = res_counter_read_u64(&memcg->kmem, name);
		break;
5125 5126 5127
	default:
		BUG();
	}
5128 5129 5130

	len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
	return simple_read_from_buffer(buf, nbytes, ppos, str, len);
B
Balbir Singh 已提交
5131
}
5132

5133
static int memcg_update_kmem_limit(struct cgroup_subsys_state *css, u64 val)
5134 5135 5136
{
	int ret = -EINVAL;
#ifdef CONFIG_MEMCG_KMEM
5137
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149
	/*
	 * For simplicity, we won't allow this to be disabled.  It also can't
	 * be changed if the cgroup has children already, or if tasks had
	 * already joined.
	 *
	 * If tasks join before we set the limit, a person looking at
	 * kmem.usage_in_bytes will have no way to determine when it took
	 * place, which makes the value quite meaningless.
	 *
	 * After it first became limited, changes in the value of the limit are
	 * of course permitted.
	 */
5150
	mutex_lock(&memcg_create_mutex);
5151 5152
	mutex_lock(&set_limit_mutex);
	if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
5153
		if (cgroup_task_count(css->cgroup) || memcg_has_children(memcg)) {
5154 5155 5156 5157 5158 5159
			ret = -EBUSY;
			goto out;
		}
		ret = res_counter_set_limit(&memcg->kmem, val);
		VM_BUG_ON(ret);

5160 5161 5162 5163 5164
		ret = memcg_update_cache_sizes(memcg);
		if (ret) {
			res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
			goto out;
		}
5165 5166 5167 5168 5169 5170
		static_key_slow_inc(&memcg_kmem_enabled_key);
		/*
		 * setting the active bit after the inc will guarantee no one
		 * starts accounting before all call sites are patched
		 */
		memcg_kmem_set_active(memcg);
5171 5172 5173 5174
	} else
		ret = res_counter_set_limit(&memcg->kmem, val);
out:
	mutex_unlock(&set_limit_mutex);
5175
	mutex_unlock(&memcg_create_mutex);
5176 5177 5178 5179
#endif
	return ret;
}

5180
#ifdef CONFIG_MEMCG_KMEM
5181
static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5182
{
5183
	int ret = 0;
5184 5185
	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
	if (!parent)
5186 5187
		goto out;

5188
	memcg->kmem_account_flags = parent->kmem_account_flags;
5189 5190 5191 5192 5193 5194 5195 5196 5197 5198
	/*
	 * When that happen, we need to disable the static branch only on those
	 * memcgs that enabled it. To achieve this, we would be forced to
	 * complicate the code by keeping track of which memcgs were the ones
	 * that actually enabled limits, and which ones got it from its
	 * parents.
	 *
	 * It is a lot simpler just to do static_key_slow_inc() on every child
	 * that is accounted.
	 */
5199 5200 5201 5202
	if (!memcg_kmem_is_active(memcg))
		goto out;

	/*
5203 5204 5205
	 * __mem_cgroup_free() will issue static_key_slow_dec() because this
	 * memcg is active already. If the later initialization fails then the
	 * cgroup core triggers the cleanup so we do not have to do it here.
5206 5207 5208 5209
	 */
	static_key_slow_inc(&memcg_kmem_enabled_key);

	mutex_lock(&set_limit_mutex);
5210
	memcg_stop_kmem_account();
5211
	ret = memcg_update_cache_sizes(memcg);
5212
	memcg_resume_kmem_account();
5213 5214 5215
	mutex_unlock(&set_limit_mutex);
out:
	return ret;
5216
}
5217
#endif /* CONFIG_MEMCG_KMEM */
5218

5219 5220 5221 5222
/*
 * The user of this function is...
 * RES_LIMIT.
 */
5223
static int mem_cgroup_write(struct cgroup_subsys_state *css, struct cftype *cft,
5224
			    const char *buffer)
B
Balbir Singh 已提交
5225
{
5226
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
G
Glauber Costa 已提交
5227 5228
	enum res_type type;
	int name;
5229 5230 5231
	unsigned long long val;
	int ret;

5232 5233
	type = MEMFILE_TYPE(cft->private);
	name = MEMFILE_ATTR(cft->private);
5234

5235
	switch (name) {
5236
	case RES_LIMIT:
5237 5238 5239 5240
		if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
			ret = -EINVAL;
			break;
		}
5241 5242
		/* This function does all necessary parse...reuse it */
		ret = res_counter_memparse_write_strategy(buffer, &val);
5243 5244 5245
		if (ret)
			break;
		if (type == _MEM)
5246
			ret = mem_cgroup_resize_limit(memcg, val);
5247
		else if (type == _MEMSWAP)
5248
			ret = mem_cgroup_resize_memsw_limit(memcg, val);
5249
		else if (type == _KMEM)
5250
			ret = memcg_update_kmem_limit(css, val);
5251 5252
		else
			return -EINVAL;
5253
		break;
5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267
	case RES_SOFT_LIMIT:
		ret = res_counter_memparse_write_strategy(buffer, &val);
		if (ret)
			break;
		/*
		 * For memsw, soft limits are hard to implement in terms
		 * of semantics, for now, we support soft limits for
		 * control without swap
		 */
		if (type == _MEM)
			ret = res_counter_set_soft_limit(&memcg->res, val);
		else
			ret = -EINVAL;
		break;
5268 5269 5270 5271 5272
	default:
		ret = -EINVAL; /* should be BUG() ? */
		break;
	}
	return ret;
B
Balbir Singh 已提交
5273 5274
}

5275 5276 5277 5278 5279 5280 5281 5282 5283 5284
static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
		unsigned long long *mem_limit, unsigned long long *memsw_limit)
{
	unsigned long long min_limit, min_memsw_limit, tmp;

	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
	if (!memcg->use_hierarchy)
		goto out;

T
Tejun Heo 已提交
5285 5286
	while (css_parent(&memcg->css)) {
		memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298
		if (!memcg->use_hierarchy)
			break;
		tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
		min_limit = min(min_limit, tmp);
		tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
		min_memsw_limit = min(min_memsw_limit, tmp);
	}
out:
	*mem_limit = min_limit;
	*memsw_limit = min_memsw_limit;
}

5299
static int mem_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
5300
{
5301
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
G
Glauber Costa 已提交
5302 5303
	int name;
	enum res_type type;
5304

5305 5306
	type = MEMFILE_TYPE(event);
	name = MEMFILE_ATTR(event);
5307

5308
	switch (name) {
5309
	case RES_MAX_USAGE:
5310
		if (type == _MEM)
5311
			res_counter_reset_max(&memcg->res);
5312
		else if (type == _MEMSWAP)
5313
			res_counter_reset_max(&memcg->memsw);
5314 5315 5316 5317
		else if (type == _KMEM)
			res_counter_reset_max(&memcg->kmem);
		else
			return -EINVAL;
5318 5319
		break;
	case RES_FAILCNT:
5320
		if (type == _MEM)
5321
			res_counter_reset_failcnt(&memcg->res);
5322
		else if (type == _MEMSWAP)
5323
			res_counter_reset_failcnt(&memcg->memsw);
5324 5325 5326 5327
		else if (type == _KMEM)
			res_counter_reset_failcnt(&memcg->kmem);
		else
			return -EINVAL;
5328 5329
		break;
	}
5330

5331
	return 0;
5332 5333
}

5334
static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
5335 5336
					struct cftype *cft)
{
5337
	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
5338 5339
}

5340
#ifdef CONFIG_MMU
5341
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5342 5343
					struct cftype *cft, u64 val)
{
5344
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5345 5346 5347

	if (val >= (1 << NR_MOVE_TYPE))
		return -EINVAL;
5348

5349
	/*
5350 5351 5352 5353
	 * No kind of locking is needed in here, because ->can_attach() will
	 * check this value once in the beginning of the process, and then carry
	 * on with stale data. This means that changes to this value will only
	 * affect task migrations starting after the change.
5354
	 */
5355
	memcg->move_charge_at_immigrate = val;
5356 5357
	return 0;
}
5358
#else
5359
static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5360 5361 5362 5363 5364
					struct cftype *cft, u64 val)
{
	return -ENOSYS;
}
#endif
5365

5366
#ifdef CONFIG_NUMA
5367 5368
static int memcg_numa_stat_show(struct cgroup_subsys_state *css,
				struct cftype *cft, struct seq_file *m)
5369 5370 5371 5372
{
	int nid;
	unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
	unsigned long node_nr;
5373
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5374

5375
	total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5376
	seq_printf(m, "total=%lu", total_nr);
5377
	for_each_node_state(nid, N_MEMORY) {
5378
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5379 5380 5381 5382
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5383
	file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5384
	seq_printf(m, "file=%lu", file_nr);
5385
	for_each_node_state(nid, N_MEMORY) {
5386
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5387
				LRU_ALL_FILE);
5388 5389 5390 5391
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5392
	anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5393
	seq_printf(m, "anon=%lu", anon_nr);
5394
	for_each_node_state(nid, N_MEMORY) {
5395
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5396
				LRU_ALL_ANON);
5397 5398 5399 5400
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');

5401
	unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5402
	seq_printf(m, "unevictable=%lu", unevictable_nr);
5403
	for_each_node_state(nid, N_MEMORY) {
5404
		node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5405
				BIT(LRU_UNEVICTABLE));
5406 5407 5408 5409 5410 5411 5412
		seq_printf(m, " N%d=%lu", nid, node_nr);
	}
	seq_putc(m, '\n');
	return 0;
}
#endif /* CONFIG_NUMA */

5413 5414 5415 5416 5417
static inline void mem_cgroup_lru_names_not_uptodate(void)
{
	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
}

5418
static int memcg_stat_show(struct cgroup_subsys_state *css, struct cftype *cft,
5419
				 struct seq_file *m)
5420
{
5421
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5422 5423
	struct mem_cgroup *mi;
	unsigned int i;
5424

5425
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5426
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5427
			continue;
5428 5429
		seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
			   mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5430
	}
L
Lee Schermerhorn 已提交
5431

5432 5433 5434 5435 5436 5437 5438 5439
	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
			   mem_cgroup_read_events(memcg, i));

	for (i = 0; i < NR_LRU_LISTS; i++)
		seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
			   mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);

K
KAMEZAWA Hiroyuki 已提交
5440
	/* Hierarchical information */
5441 5442
	{
		unsigned long long limit, memsw_limit;
5443
		memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5444
		seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5445
		if (do_swap_account)
5446 5447
			seq_printf(m, "hierarchical_memsw_limit %llu\n",
				   memsw_limit);
5448
	}
K
KOSAKI Motohiro 已提交
5449

5450 5451 5452
	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
		long long val = 0;

5453
		if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5454
			continue;
5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474
		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
		seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
	}

	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_read_events(mi, i);
		seq_printf(m, "total_%s %llu\n",
			   mem_cgroup_events_names[i], val);
	}

	for (i = 0; i < NR_LRU_LISTS; i++) {
		unsigned long long val = 0;

		for_each_mem_cgroup_tree(mi, memcg)
			val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
		seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5475
	}
K
KAMEZAWA Hiroyuki 已提交
5476

K
KOSAKI Motohiro 已提交
5477 5478 5479 5480
#ifdef CONFIG_DEBUG_VM
	{
		int nid, zid;
		struct mem_cgroup_per_zone *mz;
5481
		struct zone_reclaim_stat *rstat;
K
KOSAKI Motohiro 已提交
5482 5483 5484 5485 5486
		unsigned long recent_rotated[2] = {0, 0};
		unsigned long recent_scanned[2] = {0, 0};

		for_each_online_node(nid)
			for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5487
				mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5488
				rstat = &mz->lruvec.reclaim_stat;
K
KOSAKI Motohiro 已提交
5489

5490 5491 5492 5493
				recent_rotated[0] += rstat->recent_rotated[0];
				recent_rotated[1] += rstat->recent_rotated[1];
				recent_scanned[0] += rstat->recent_scanned[0];
				recent_scanned[1] += rstat->recent_scanned[1];
K
KOSAKI Motohiro 已提交
5494
			}
5495 5496 5497 5498
		seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
		seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
		seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
		seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
K
KOSAKI Motohiro 已提交
5499 5500 5501
	}
#endif

5502 5503 5504
	return 0;
}

5505 5506
static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
				      struct cftype *cft)
K
KOSAKI Motohiro 已提交
5507
{
5508
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
K
KOSAKI Motohiro 已提交
5509

5510
	return mem_cgroup_swappiness(memcg);
K
KOSAKI Motohiro 已提交
5511 5512
}

5513 5514
static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
				       struct cftype *cft, u64 val)
K
KOSAKI Motohiro 已提交
5515
{
5516
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
5517
	struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
K
KOSAKI Motohiro 已提交
5518

T
Tejun Heo 已提交
5519
	if (val > 100 || !parent)
K
KOSAKI Motohiro 已提交
5520 5521
		return -EINVAL;

5522
	mutex_lock(&memcg_create_mutex);
5523

K
KOSAKI Motohiro 已提交
5524
	/* If under hierarchy, only empty-root can set this value */
5525
	if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5526
		mutex_unlock(&memcg_create_mutex);
K
KOSAKI Motohiro 已提交
5527
		return -EINVAL;
5528
	}
K
KOSAKI Motohiro 已提交
5529 5530 5531

	memcg->swappiness = val;

5532
	mutex_unlock(&memcg_create_mutex);
5533

K
KOSAKI Motohiro 已提交
5534 5535 5536
	return 0;
}

5537 5538 5539 5540 5541 5542 5543 5544
static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
{
	struct mem_cgroup_threshold_ary *t;
	u64 usage;
	int i;

	rcu_read_lock();
	if (!swap)
5545
		t = rcu_dereference(memcg->thresholds.primary);
5546
	else
5547
		t = rcu_dereference(memcg->memsw_thresholds.primary);
5548 5549 5550 5551 5552 5553 5554

	if (!t)
		goto unlock;

	usage = mem_cgroup_usage(memcg, swap);

	/*
5555
	 * current_threshold points to threshold just below or equal to usage.
5556 5557 5558
	 * If it's not true, a threshold was crossed after last
	 * call of __mem_cgroup_threshold().
	 */
5559
	i = t->current_threshold;
5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582

	/*
	 * Iterate backward over array of thresholds starting from
	 * current_threshold and check if a threshold is crossed.
	 * If none of thresholds below usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* i = current_threshold + 1 */
	i++;

	/*
	 * Iterate forward over array of thresholds starting from
	 * current_threshold+1 and check if a threshold is crossed.
	 * If none of thresholds above usage is crossed, we read
	 * only one element of the array here.
	 */
	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
		eventfd_signal(t->entries[i].eventfd, 1);

	/* Update current_threshold */
5583
	t->current_threshold = i - 1;
5584 5585 5586 5587 5588 5589
unlock:
	rcu_read_unlock();
}

static void mem_cgroup_threshold(struct mem_cgroup *memcg)
{
5590 5591 5592 5593 5594 5595 5596
	while (memcg) {
		__mem_cgroup_threshold(memcg, false);
		if (do_swap_account)
			__mem_cgroup_threshold(memcg, true);

		memcg = parent_mem_cgroup(memcg);
	}
5597 5598 5599 5600 5601 5602 5603 5604 5605 5606
}

static int compare_thresholds(const void *a, const void *b)
{
	const struct mem_cgroup_threshold *_a = a;
	const struct mem_cgroup_threshold *_b = b;

	return _a->threshold - _b->threshold;
}

5607
static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
5608 5609 5610
{
	struct mem_cgroup_eventfd_list *ev;

5611
	list_for_each_entry(ev, &memcg->oom_notify, list)
K
KAMEZAWA Hiroyuki 已提交
5612 5613 5614 5615
		eventfd_signal(ev->eventfd, 1);
	return 0;
}

5616
static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
5617
{
K
KAMEZAWA Hiroyuki 已提交
5618 5619
	struct mem_cgroup *iter;

5620
	for_each_mem_cgroup_tree(iter, memcg)
K
KAMEZAWA Hiroyuki 已提交
5621
		mem_cgroup_oom_notify_cb(iter);
K
KAMEZAWA Hiroyuki 已提交
5622 5623 5624 5625
}

static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5626 5627
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5628 5629
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
G
Glauber Costa 已提交
5630
	enum res_type type = MEMFILE_TYPE(cft->private);
5631
	u64 threshold, usage;
5632
	int i, size, ret;
5633 5634 5635 5636 5637 5638

	ret = res_counter_memparse_write_strategy(args, &threshold);
	if (ret)
		return ret;

	mutex_lock(&memcg->thresholds_lock);
5639

5640
	if (type == _MEM)
5641
		thresholds = &memcg->thresholds;
5642
	else if (type == _MEMSWAP)
5643
		thresholds = &memcg->memsw_thresholds;
5644 5645 5646 5647 5648 5649
	else
		BUG();

	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before adding a new one */
5650
	if (thresholds->primary)
5651 5652
		__mem_cgroup_threshold(memcg, type == _MEMSWAP);

5653
	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5654 5655

	/* Allocate memory for new array of thresholds */
5656
	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
5657
			GFP_KERNEL);
5658
	if (!new) {
5659 5660 5661
		ret = -ENOMEM;
		goto unlock;
	}
5662
	new->size = size;
5663 5664

	/* Copy thresholds (if any) to new array */
5665 5666
	if (thresholds->primary) {
		memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5667
				sizeof(struct mem_cgroup_threshold));
5668 5669
	}

5670
	/* Add new threshold */
5671 5672
	new->entries[size - 1].eventfd = eventfd;
	new->entries[size - 1].threshold = threshold;
5673 5674

	/* Sort thresholds. Registering of new threshold isn't time-critical */
5675
	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5676 5677 5678
			compare_thresholds, NULL);

	/* Find current threshold */
5679
	new->current_threshold = -1;
5680
	for (i = 0; i < size; i++) {
5681
		if (new->entries[i].threshold <= usage) {
5682
			/*
5683 5684
			 * new->current_threshold will not be used until
			 * rcu_assign_pointer(), so it's safe to increment
5685 5686
			 * it here.
			 */
5687
			++new->current_threshold;
5688 5689
		} else
			break;
5690 5691
	}

5692 5693 5694 5695 5696
	/* Free old spare buffer and save old primary buffer as spare */
	kfree(thresholds->spare);
	thresholds->spare = thresholds->primary;

	rcu_assign_pointer(thresholds->primary, new);
5697

5698
	/* To be sure that nobody uses thresholds */
5699 5700 5701 5702 5703 5704 5705 5706
	synchronize_rcu();

unlock:
	mutex_unlock(&memcg->thresholds_lock);

	return ret;
}

5707
static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
5708
	struct cftype *cft, struct eventfd_ctx *eventfd)
5709 5710
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5711 5712
	struct mem_cgroup_thresholds *thresholds;
	struct mem_cgroup_threshold_ary *new;
G
Glauber Costa 已提交
5713
	enum res_type type = MEMFILE_TYPE(cft->private);
5714
	u64 usage;
5715
	int i, j, size;
5716 5717 5718

	mutex_lock(&memcg->thresholds_lock);
	if (type == _MEM)
5719
		thresholds = &memcg->thresholds;
5720
	else if (type == _MEMSWAP)
5721
		thresholds = &memcg->memsw_thresholds;
5722 5723 5724
	else
		BUG();

5725 5726 5727
	if (!thresholds->primary)
		goto unlock;

5728 5729 5730 5731 5732 5733
	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);

	/* Check if a threshold crossed before removing */
	__mem_cgroup_threshold(memcg, type == _MEMSWAP);

	/* Calculate new number of threshold */
5734 5735 5736
	size = 0;
	for (i = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd != eventfd)
5737 5738 5739
			size++;
	}

5740
	new = thresholds->spare;
5741

5742 5743
	/* Set thresholds array to NULL if we don't have thresholds */
	if (!size) {
5744 5745
		kfree(new);
		new = NULL;
5746
		goto swap_buffers;
5747 5748
	}

5749
	new->size = size;
5750 5751

	/* Copy thresholds and find current threshold */
5752 5753 5754
	new->current_threshold = -1;
	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
		if (thresholds->primary->entries[i].eventfd == eventfd)
5755 5756
			continue;

5757
		new->entries[j] = thresholds->primary->entries[i];
5758
		if (new->entries[j].threshold <= usage) {
5759
			/*
5760
			 * new->current_threshold will not be used
5761 5762 5763
			 * until rcu_assign_pointer(), so it's safe to increment
			 * it here.
			 */
5764
			++new->current_threshold;
5765 5766 5767 5768
		}
		j++;
	}

5769
swap_buffers:
5770 5771
	/* Swap primary and spare array */
	thresholds->spare = thresholds->primary;
5772 5773 5774 5775 5776 5777
	/* If all events are unregistered, free the spare array */
	if (!new) {
		kfree(thresholds->spare);
		thresholds->spare = NULL;
	}

5778
	rcu_assign_pointer(thresholds->primary, new);
5779

5780
	/* To be sure that nobody uses thresholds */
5781
	synchronize_rcu();
5782
unlock:
5783 5784
	mutex_unlock(&memcg->thresholds_lock);
}
5785

K
KAMEZAWA Hiroyuki 已提交
5786 5787 5788 5789 5790
static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
	struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
{
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
	struct mem_cgroup_eventfd_list *event;
G
Glauber Costa 已提交
5791
	enum res_type type = MEMFILE_TYPE(cft->private);
K
KAMEZAWA Hiroyuki 已提交
5792 5793 5794 5795 5796 5797

	BUG_ON(type != _OOM_TYPE);
	event = kmalloc(sizeof(*event),	GFP_KERNEL);
	if (!event)
		return -ENOMEM;

5798
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5799 5800 5801 5802 5803

	event->eventfd = eventfd;
	list_add(&event->list, &memcg->oom_notify);

	/* already in OOM ? */
5804
	if (atomic_read(&memcg->under_oom))
K
KAMEZAWA Hiroyuki 已提交
5805
		eventfd_signal(eventfd, 1);
5806
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5807 5808 5809 5810

	return 0;
}

5811
static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
K
KAMEZAWA Hiroyuki 已提交
5812 5813
	struct cftype *cft, struct eventfd_ctx *eventfd)
{
5814
	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
K
KAMEZAWA Hiroyuki 已提交
5815
	struct mem_cgroup_eventfd_list *ev, *tmp;
G
Glauber Costa 已提交
5816
	enum res_type type = MEMFILE_TYPE(cft->private);
K
KAMEZAWA Hiroyuki 已提交
5817 5818 5819

	BUG_ON(type != _OOM_TYPE);

5820
	spin_lock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5821

5822
	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
K
KAMEZAWA Hiroyuki 已提交
5823 5824 5825 5826 5827 5828
		if (ev->eventfd == eventfd) {
			list_del(&ev->list);
			kfree(ev);
		}
	}

5829
	spin_unlock(&memcg_oom_lock);
K
KAMEZAWA Hiroyuki 已提交
5830 5831
}

5832
static int mem_cgroup_oom_control_read(struct cgroup_subsys_state *css,
5833 5834
	struct cftype *cft,  struct cgroup_map_cb *cb)
{
5835
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5836

5837
	cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5838

5839
	if (atomic_read(&memcg->under_oom))
5840 5841 5842 5843 5844 5845
		cb->fill(cb, "under_oom", 1);
	else
		cb->fill(cb, "under_oom", 0);
	return 0;
}

5846
static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
5847 5848
	struct cftype *cft, u64 val)
{
5849
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
T
Tejun Heo 已提交
5850
	struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
5851 5852

	/* cannot set to root cgroup and only 0 and 1 are allowed */
T
Tejun Heo 已提交
5853
	if (!parent || !((val == 0) || (val == 1)))
5854 5855
		return -EINVAL;

5856
	mutex_lock(&memcg_create_mutex);
5857
	/* oom-kill-disable is a flag for subhierarchy. */
5858
	if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5859
		mutex_unlock(&memcg_create_mutex);
5860 5861
		return -EINVAL;
	}
5862
	memcg->oom_kill_disable = val;
5863
	if (!val)
5864
		memcg_oom_recover(memcg);
5865
	mutex_unlock(&memcg_create_mutex);
5866 5867 5868
	return 0;
}

A
Andrew Morton 已提交
5869
#ifdef CONFIG_MEMCG_KMEM
5870
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5871
{
5872 5873
	int ret;

5874
	memcg->kmemcg_id = -1;
5875 5876 5877
	ret = memcg_propagate_kmem(memcg);
	if (ret)
		return ret;
5878

5879
	return mem_cgroup_sockets_init(memcg, ss);
5880
}
5881

5882
static void memcg_destroy_kmem(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
5883
{
5884
	mem_cgroup_sockets_destroy(memcg);
5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910
}

static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
{
	if (!memcg_kmem_is_active(memcg))
		return;

	/*
	 * kmem charges can outlive the cgroup. In the case of slab
	 * pages, for instance, a page contain objects from various
	 * processes. As we prevent from taking a reference for every
	 * such allocation we have to be careful when doing uncharge
	 * (see memcg_uncharge_kmem) and here during offlining.
	 *
	 * The idea is that that only the _last_ uncharge which sees
	 * the dead memcg will drop the last reference. An additional
	 * reference is taken here before the group is marked dead
	 * which is then paired with css_put during uncharge resp. here.
	 *
	 * Although this might sound strange as this path is called from
	 * css_offline() when the referencemight have dropped down to 0
	 * and shouldn't be incremented anymore (css_tryget would fail)
	 * we do not have other options because of the kmem allocations
	 * lifetime.
	 */
	css_get(&memcg->css);
5911 5912 5913 5914 5915 5916 5917

	memcg_kmem_mark_dead(memcg);

	if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
		return;

	if (memcg_kmem_test_and_clear_dead(memcg))
5918
		css_put(&memcg->css);
G
Glauber Costa 已提交
5919
}
5920
#else
5921
static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5922 5923 5924
{
	return 0;
}
G
Glauber Costa 已提交
5925

5926 5927 5928 5929 5930
static void memcg_destroy_kmem(struct mem_cgroup *memcg)
{
}

static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
G
Glauber Costa 已提交
5931 5932
{
}
5933 5934
#endif

B
Balbir Singh 已提交
5935 5936
static struct cftype mem_cgroup_files[] = {
	{
5937
		.name = "usage_in_bytes",
5938
		.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
5939
		.read = mem_cgroup_read,
K
KAMEZAWA Hiroyuki 已提交
5940 5941
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
B
Balbir Singh 已提交
5942
	},
5943 5944
	{
		.name = "max_usage_in_bytes",
5945
		.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
5946
		.trigger = mem_cgroup_reset,
5947
		.read = mem_cgroup_read,
5948
	},
B
Balbir Singh 已提交
5949
	{
5950
		.name = "limit_in_bytes",
5951
		.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
5952
		.write_string = mem_cgroup_write,
5953
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
5954
	},
5955 5956 5957 5958
	{
		.name = "soft_limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
		.write_string = mem_cgroup_write,
5959
		.read = mem_cgroup_read,
5960
	},
B
Balbir Singh 已提交
5961 5962
	{
		.name = "failcnt",
5963
		.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
5964
		.trigger = mem_cgroup_reset,
5965
		.read = mem_cgroup_read,
B
Balbir Singh 已提交
5966
	},
5967 5968
	{
		.name = "stat",
5969
		.read_seq_string = memcg_stat_show,
5970
	},
5971 5972 5973 5974
	{
		.name = "force_empty",
		.trigger = mem_cgroup_force_empty_write,
	},
5975 5976
	{
		.name = "use_hierarchy",
5977
		.flags = CFTYPE_INSANE,
5978 5979 5980
		.write_u64 = mem_cgroup_hierarchy_write,
		.read_u64 = mem_cgroup_hierarchy_read,
	},
K
KOSAKI Motohiro 已提交
5981 5982 5983 5984 5985
	{
		.name = "swappiness",
		.read_u64 = mem_cgroup_swappiness_read,
		.write_u64 = mem_cgroup_swappiness_write,
	},
5986 5987 5988 5989 5990
	{
		.name = "move_charge_at_immigrate",
		.read_u64 = mem_cgroup_move_charge_read,
		.write_u64 = mem_cgroup_move_charge_write,
	},
K
KAMEZAWA Hiroyuki 已提交
5991 5992
	{
		.name = "oom_control",
5993 5994
		.read_map = mem_cgroup_oom_control_read,
		.write_u64 = mem_cgroup_oom_control_write,
K
KAMEZAWA Hiroyuki 已提交
5995 5996 5997 5998
		.register_event = mem_cgroup_oom_register_event,
		.unregister_event = mem_cgroup_oom_unregister_event,
		.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
	},
5999 6000 6001 6002 6003
	{
		.name = "pressure_level",
		.register_event = vmpressure_register_event,
		.unregister_event = vmpressure_unregister_event,
	},
6004 6005 6006
#ifdef CONFIG_NUMA
	{
		.name = "numa_stat",
6007
		.read_seq_string = memcg_numa_stat_show,
6008 6009
	},
#endif
6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033
#ifdef CONFIG_MEMCG_KMEM
	{
		.name = "kmem.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.failcnt",
		.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{
		.name = "kmem.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
6034 6035 6036 6037 6038 6039
#ifdef CONFIG_SLABINFO
	{
		.name = "kmem.slabinfo",
		.read_seq_string = mem_cgroup_slabinfo_read,
	},
#endif
6040
#endif
6041
	{ },	/* terminate */
6042
};
6043

6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073
#ifdef CONFIG_MEMCG_SWAP
static struct cftype memsw_cgroup_files[] = {
	{
		.name = "memsw.usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
		.read = mem_cgroup_read,
		.register_event = mem_cgroup_usage_register_event,
		.unregister_event = mem_cgroup_usage_unregister_event,
	},
	{
		.name = "memsw.max_usage_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{
		.name = "memsw.limit_in_bytes",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
		.write_string = mem_cgroup_write,
		.read = mem_cgroup_read,
	},
	{
		.name = "memsw.failcnt",
		.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
		.trigger = mem_cgroup_reset,
		.read = mem_cgroup_read,
	},
	{ },	/* terminate */
};
#endif
6074
static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6075 6076
{
	struct mem_cgroup_per_node *pn;
6077
	struct mem_cgroup_per_zone *mz;
6078
	int zone, tmp = node;
6079 6080 6081 6082 6083 6084 6085 6086
	/*
	 * This routine is called against possible nodes.
	 * But it's BUG to call kmalloc() against offline node.
	 *
	 * TODO: this routine can waste much memory for nodes which will
	 *       never be onlined. It's better to use memory hotplug callback
	 *       function.
	 */
6087 6088
	if (!node_state(node, N_NORMAL_MEMORY))
		tmp = -1;
6089
	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
6090 6091
	if (!pn)
		return 1;
6092 6093 6094

	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
		mz = &pn->zoneinfo[zone];
6095
		lruvec_init(&mz->lruvec);
6096
		mz->usage_in_excess = 0;
6097
		mz->on_tree = false;
6098
		mz->memcg = memcg;
6099
	}
6100
	memcg->nodeinfo[node] = pn;
6101 6102 6103
	return 0;
}

6104
static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
6105
{
6106
	kfree(memcg->nodeinfo[node]);
6107 6108
}

6109 6110
static struct mem_cgroup *mem_cgroup_alloc(void)
{
6111
	struct mem_cgroup *memcg;
6112
	size_t size = memcg_size();
6113

6114
	/* Can be very big if nr_node_ids is very big */
6115
	if (size < PAGE_SIZE)
6116
		memcg = kzalloc(size, GFP_KERNEL);
6117
	else
6118
		memcg = vzalloc(size);
6119

6120
	if (!memcg)
6121 6122
		return NULL;

6123 6124
	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
	if (!memcg->stat)
6125
		goto out_free;
6126 6127
	spin_lock_init(&memcg->pcp_counter_lock);
	return memcg;
6128 6129 6130

out_free:
	if (size < PAGE_SIZE)
6131
		kfree(memcg);
6132
	else
6133
		vfree(memcg);
6134
	return NULL;
6135 6136
}

6137
/*
6138 6139 6140 6141 6142 6143 6144 6145
 * At destroying mem_cgroup, references from swap_cgroup can remain.
 * (scanning all at force_empty is too costly...)
 *
 * Instead of clearing all references at force_empty, we remember
 * the number of reference from swap_cgroup and free mem_cgroup when
 * it goes down to 0.
 *
 * Removal of cgroup itself succeeds regardless of refs from swap.
6146
 */
6147 6148

static void __mem_cgroup_free(struct mem_cgroup *memcg)
6149
{
6150
	int node;
6151
	size_t size = memcg_size();
6152

6153 6154 6155 6156 6157 6158 6159 6160
	mem_cgroup_remove_from_trees(memcg);
	free_css_id(&mem_cgroup_subsys, &memcg->css);

	for_each_node(node)
		free_mem_cgroup_per_zone_info(memcg, node);

	free_percpu(memcg->stat);

6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171
	/*
	 * We need to make sure that (at least for now), the jump label
	 * destruction code runs outside of the cgroup lock. This is because
	 * get_online_cpus(), which is called from the static_branch update,
	 * can't be called inside the cgroup_lock. cpusets are the ones
	 * enforcing this dependency, so if they ever change, we might as well.
	 *
	 * schedule_work() will guarantee this happens. Be careful if you need
	 * to move this code around, and make sure it is outside
	 * the cgroup_lock.
	 */
6172
	disarm_static_keys(memcg);
6173 6174 6175 6176
	if (size < PAGE_SIZE)
		kfree(memcg);
	else
		vfree(memcg);
6177
}
6178

6179 6180 6181
/*
 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
 */
G
Glauber Costa 已提交
6182
struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
6183
{
6184
	if (!memcg->res.parent)
6185
		return NULL;
6186
	return mem_cgroup_from_res_counter(memcg->res.parent, res);
6187
}
G
Glauber Costa 已提交
6188
EXPORT_SYMBOL(parent_mem_cgroup);
6189

6190
static void __init mem_cgroup_soft_limit_tree_init(void)
6191 6192 6193 6194 6195
{
	struct mem_cgroup_tree_per_node *rtpn;
	struct mem_cgroup_tree_per_zone *rtpz;
	int tmp, node, zone;

B
Bob Liu 已提交
6196
	for_each_node(node) {
6197 6198 6199 6200
		tmp = node;
		if (!node_state(node, N_NORMAL_MEMORY))
			tmp = -1;
		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6201
		BUG_ON(!rtpn);
6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212

		soft_limit_tree.rb_tree_per_node[node] = rtpn;

		for (zone = 0; zone < MAX_NR_ZONES; zone++) {
			rtpz = &rtpn->rb_tree_per_zone[zone];
			rtpz->rb_root = RB_ROOT;
			spin_lock_init(&rtpz->lock);
		}
	}
}

L
Li Zefan 已提交
6213
static struct cgroup_subsys_state * __ref
6214
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
B
Balbir Singh 已提交
6215
{
6216
	struct mem_cgroup *memcg;
K
KAMEZAWA Hiroyuki 已提交
6217
	long error = -ENOMEM;
6218
	int node;
B
Balbir Singh 已提交
6219

6220 6221
	memcg = mem_cgroup_alloc();
	if (!memcg)
K
KAMEZAWA Hiroyuki 已提交
6222
		return ERR_PTR(error);
6223

B
Bob Liu 已提交
6224
	for_each_node(node)
6225
		if (alloc_mem_cgroup_per_zone_info(memcg, node))
6226
			goto free_out;
6227

6228
	/* root ? */
6229
	if (parent_css == NULL) {
6230
		root_mem_cgroup = memcg;
6231 6232 6233
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
		res_counter_init(&memcg->kmem, NULL);
6234
	}
6235

6236 6237 6238 6239 6240
	memcg->last_scanned_node = MAX_NUMNODES;
	INIT_LIST_HEAD(&memcg->oom_notify);
	memcg->move_charge_at_immigrate = 0;
	mutex_init(&memcg->thresholds_lock);
	spin_lock_init(&memcg->move_lock);
6241
	vmpressure_init(&memcg->vmpressure);
6242 6243 6244 6245 6246 6247 6248 6249 6250

	return &memcg->css;

free_out:
	__mem_cgroup_free(memcg);
	return ERR_PTR(error);
}

static int
6251
mem_cgroup_css_online(struct cgroup_subsys_state *css)
6252
{
6253 6254
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
	struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(css));
6255 6256
	int error = 0;

T
Tejun Heo 已提交
6257
	if (!parent)
6258 6259
		return 0;

6260
	mutex_lock(&memcg_create_mutex);
6261 6262 6263 6264 6265 6266

	memcg->use_hierarchy = parent->use_hierarchy;
	memcg->oom_kill_disable = parent->oom_kill_disable;
	memcg->swappiness = mem_cgroup_swappiness(parent);

	if (parent->use_hierarchy) {
6267 6268
		res_counter_init(&memcg->res, &parent->res);
		res_counter_init(&memcg->memsw, &parent->memsw);
6269
		res_counter_init(&memcg->kmem, &parent->kmem);
6270

6271
		/*
6272 6273
		 * No need to take a reference to the parent because cgroup
		 * core guarantees its existence.
6274
		 */
6275
	} else {
6276 6277
		res_counter_init(&memcg->res, NULL);
		res_counter_init(&memcg->memsw, NULL);
6278
		res_counter_init(&memcg->kmem, NULL);
6279 6280 6281 6282 6283
		/*
		 * Deeper hierachy with use_hierarchy == false doesn't make
		 * much sense so let cgroup subsystem know about this
		 * unfortunate state in our controller.
		 */
6284
		if (parent != root_mem_cgroup)
6285
			mem_cgroup_subsys.broken_hierarchy = true;
6286
	}
6287 6288

	error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
6289
	mutex_unlock(&memcg_create_mutex);
6290
	return error;
B
Balbir Singh 已提交
6291 6292
}

M
Michal Hocko 已提交
6293 6294 6295 6296 6297 6298 6299 6300
/*
 * Announce all parents that a group from their hierarchy is gone.
 */
static void mem_cgroup_invalidate_reclaim_iterators(struct mem_cgroup *memcg)
{
	struct mem_cgroup *parent = memcg;

	while ((parent = parent_mem_cgroup(parent)))
6301
		mem_cgroup_iter_invalidate(parent);
M
Michal Hocko 已提交
6302 6303 6304 6305 6306 6307

	/*
	 * if the root memcg is not hierarchical we have to check it
	 * explicitely.
	 */
	if (!root_mem_cgroup->use_hierarchy)
6308
		mem_cgroup_iter_invalidate(root_mem_cgroup);
M
Michal Hocko 已提交
6309 6310
}

6311
static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
6312
{
6313
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6314

6315 6316
	kmem_cgroup_css_offline(memcg);

M
Michal Hocko 已提交
6317
	mem_cgroup_invalidate_reclaim_iterators(memcg);
6318
	mem_cgroup_reparent_charges(memcg);
G
Glauber Costa 已提交
6319
	mem_cgroup_destroy_all_caches(memcg);
6320 6321
}

6322
static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
B
Balbir Singh 已提交
6323
{
6324
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6325

6326
	memcg_destroy_kmem(memcg);
6327
	__mem_cgroup_free(memcg);
B
Balbir Singh 已提交
6328 6329
}

6330
#ifdef CONFIG_MMU
6331
/* Handlers for move charge at task migration. */
6332 6333
#define PRECHARGE_COUNT_AT_ONCE	256
static int mem_cgroup_do_precharge(unsigned long count)
6334
{
6335 6336
	int ret = 0;
	int batch_count = PRECHARGE_COUNT_AT_ONCE;
6337
	struct mem_cgroup *memcg = mc.to;
6338

6339
	if (mem_cgroup_is_root(memcg)) {
6340 6341 6342 6343 6344 6345 6346 6347
		mc.precharge += count;
		/* we don't need css_get for root */
		return ret;
	}
	/* try to charge at once */
	if (count > 1) {
		struct res_counter *dummy;
		/*
6348
		 * "memcg" cannot be under rmdir() because we've already checked
6349 6350 6351 6352
		 * by cgroup_lock_live_cgroup() that it is not removed and we
		 * are still under the same cgroup_mutex. So we can postpone
		 * css_get().
		 */
6353
		if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
6354
			goto one_by_one;
6355
		if (do_swap_account && res_counter_charge(&memcg->memsw,
6356
						PAGE_SIZE * count, &dummy)) {
6357
			res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373
			goto one_by_one;
		}
		mc.precharge += count;
		return ret;
	}
one_by_one:
	/* fall back to one by one charge */
	while (count--) {
		if (signal_pending(current)) {
			ret = -EINTR;
			break;
		}
		if (!batch_count--) {
			batch_count = PRECHARGE_COUNT_AT_ONCE;
			cond_resched();
		}
6374 6375
		ret = __mem_cgroup_try_charge(NULL,
					GFP_KERNEL, 1, &memcg, false);
6376
		if (ret)
6377
			/* mem_cgroup_clear_mc() will do uncharge later */
6378
			return ret;
6379 6380
		mc.precharge++;
	}
6381 6382 6383 6384
	return ret;
}

/**
6385
 * get_mctgt_type - get target type of moving charge
6386 6387 6388
 * @vma: the vma the pte to be checked belongs
 * @addr: the address corresponding to the pte to be checked
 * @ptent: the pte to be checked
6389
 * @target: the pointer the target page or swap ent will be stored(can be NULL)
6390 6391 6392 6393 6394 6395
 *
 * Returns
 *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
 *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
 *     move charge. if @target is not NULL, the page is stored in target->page
 *     with extra refcnt got(Callers should handle it).
6396 6397 6398
 *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
 *     target for charge migration. if @target is not NULL, the entry is stored
 *     in target->ent.
6399 6400 6401 6402 6403
 *
 * Called with pte lock held.
 */
union mc_target {
	struct page	*page;
6404
	swp_entry_t	ent;
6405 6406 6407
};

enum mc_target_type {
6408
	MC_TARGET_NONE = 0,
6409
	MC_TARGET_PAGE,
6410
	MC_TARGET_SWAP,
6411 6412
};

D
Daisuke Nishimura 已提交
6413 6414
static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
						unsigned long addr, pte_t ptent)
6415
{
D
Daisuke Nishimura 已提交
6416
	struct page *page = vm_normal_page(vma, addr, ptent);
6417

D
Daisuke Nishimura 已提交
6418 6419 6420 6421
	if (!page || !page_mapped(page))
		return NULL;
	if (PageAnon(page)) {
		/* we don't move shared anon */
6422
		if (!move_anon())
D
Daisuke Nishimura 已提交
6423
			return NULL;
6424 6425
	} else if (!move_file())
		/* we ignore mapcount for file pages */
D
Daisuke Nishimura 已提交
6426 6427 6428 6429 6430 6431 6432
		return NULL;
	if (!get_page_unless_zero(page))
		return NULL;

	return page;
}

6433
#ifdef CONFIG_SWAP
D
Daisuke Nishimura 已提交
6434 6435 6436 6437 6438 6439 6440 6441
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	swp_entry_t ent = pte_to_swp_entry(ptent);

	if (!move_anon() || non_swap_entry(ent))
		return NULL;
6442 6443 6444 6445
	/*
	 * Because lookup_swap_cache() updates some statistics counter,
	 * we call find_get_page() with swapper_space directly.
	 */
6446
	page = find_get_page(swap_address_space(ent), ent.val);
D
Daisuke Nishimura 已提交
6447 6448 6449 6450 6451
	if (do_swap_account)
		entry->val = ent.val;

	return page;
}
6452 6453 6454 6455 6456 6457 6458
#else
static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	return NULL;
}
#endif
D
Daisuke Nishimura 已提交
6459

6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478
static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
			unsigned long addr, pte_t ptent, swp_entry_t *entry)
{
	struct page *page = NULL;
	struct address_space *mapping;
	pgoff_t pgoff;

	if (!vma->vm_file) /* anonymous vma */
		return NULL;
	if (!move_file())
		return NULL;

	mapping = vma->vm_file->f_mapping;
	if (pte_none(ptent))
		pgoff = linear_page_index(vma, addr);
	else /* pte_file(ptent) is true */
		pgoff = pte_to_pgoff(ptent);

	/* page is moved even if it's not RSS of this task(page-faulted). */
6479 6480 6481 6482 6483 6484
	page = find_get_page(mapping, pgoff);

#ifdef CONFIG_SWAP
	/* shmem/tmpfs may report page out on swap: account for that too. */
	if (radix_tree_exceptional_entry(page)) {
		swp_entry_t swap = radix_to_swp_entry(page);
6485
		if (do_swap_account)
6486
			*entry = swap;
6487
		page = find_get_page(swap_address_space(swap), swap.val);
6488
	}
6489
#endif
6490 6491 6492
	return page;
}

6493
static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
D
Daisuke Nishimura 已提交
6494 6495 6496 6497
		unsigned long addr, pte_t ptent, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
6498
	enum mc_target_type ret = MC_TARGET_NONE;
D
Daisuke Nishimura 已提交
6499 6500 6501 6502 6503 6504
	swp_entry_t ent = { .val = 0 };

	if (pte_present(ptent))
		page = mc_handle_present_pte(vma, addr, ptent);
	else if (is_swap_pte(ptent))
		page = mc_handle_swap_pte(vma, addr, ptent, &ent);
6505 6506
	else if (pte_none(ptent) || pte_file(ptent))
		page = mc_handle_file_pte(vma, addr, ptent, &ent);
D
Daisuke Nishimura 已提交
6507 6508

	if (!page && !ent.val)
6509
		return ret;
6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524
	if (page) {
		pc = lookup_page_cgroup(page);
		/*
		 * Do only loose check w/o page_cgroup lock.
		 * mem_cgroup_move_account() checks the pc is valid or not under
		 * the lock.
		 */
		if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
			ret = MC_TARGET_PAGE;
			if (target)
				target->page = page;
		}
		if (!ret || !target)
			put_page(page);
	}
D
Daisuke Nishimura 已提交
6525 6526
	/* There is a swap entry and a page doesn't exist or isn't charged */
	if (ent.val && !ret &&
6527
			css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
6528 6529 6530
		ret = MC_TARGET_SWAP;
		if (target)
			target->ent = ent;
6531 6532 6533 6534
	}
	return ret;
}

6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
 * We don't consider swapping or file mapped pages because THP does not
 * support them for now.
 * Caller should make sure that pmd_trans_huge(pmd) is true.
 */
static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	struct page *page = NULL;
	struct page_cgroup *pc;
	enum mc_target_type ret = MC_TARGET_NONE;

	page = pmd_page(pmd);
	VM_BUG_ON(!page || !PageHead(page));
	if (!move_anon())
		return ret;
	pc = lookup_page_cgroup(page);
	if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
		ret = MC_TARGET_PAGE;
		if (target) {
			get_page(page);
			target->page = page;
		}
	}
	return ret;
}
#else
static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
		unsigned long addr, pmd_t pmd, union mc_target *target)
{
	return MC_TARGET_NONE;
}
#endif

6570 6571 6572 6573 6574 6575 6576 6577
static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
					unsigned long addr, unsigned long end,
					struct mm_walk *walk)
{
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;

6578 6579 6580 6581
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
		if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
			mc.precharge += HPAGE_PMD_NR;
		spin_unlock(&vma->vm_mm->page_table_lock);
6582
		return 0;
6583
	}
6584

6585 6586
	if (pmd_trans_unstable(pmd))
		return 0;
6587 6588
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; pte++, addr += PAGE_SIZE)
6589
		if (get_mctgt_type(vma, addr, *pte, NULL))
6590 6591 6592 6593
			mc.precharge++;	/* increment precharge temporarily */
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

6594 6595 6596
	return 0;
}

6597 6598 6599 6600 6601
static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
{
	unsigned long precharge;
	struct vm_area_struct *vma;

6602
	down_read(&mm->mmap_sem);
6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		struct mm_walk mem_cgroup_count_precharge_walk = {
			.pmd_entry = mem_cgroup_count_precharge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		walk_page_range(vma->vm_start, vma->vm_end,
					&mem_cgroup_count_precharge_walk);
	}
6614
	up_read(&mm->mmap_sem);
6615 6616 6617 6618 6619 6620 6621 6622 6623

	precharge = mc.precharge;
	mc.precharge = 0;

	return precharge;
}

static int mem_cgroup_precharge_mc(struct mm_struct *mm)
{
6624 6625 6626 6627 6628
	unsigned long precharge = mem_cgroup_count_precharge(mm);

	VM_BUG_ON(mc.moving_task);
	mc.moving_task = current;
	return mem_cgroup_do_precharge(precharge);
6629 6630
}

6631 6632
/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
static void __mem_cgroup_clear_mc(void)
6633
{
6634 6635
	struct mem_cgroup *from = mc.from;
	struct mem_cgroup *to = mc.to;
L
Li Zefan 已提交
6636
	int i;
6637

6638
	/* we must uncharge all the leftover precharges from mc.to */
6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649
	if (mc.precharge) {
		__mem_cgroup_cancel_charge(mc.to, mc.precharge);
		mc.precharge = 0;
	}
	/*
	 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
	 * we must uncharge here.
	 */
	if (mc.moved_charge) {
		__mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
		mc.moved_charge = 0;
6650
	}
6651 6652 6653 6654 6655 6656
	/* we must fixup refcnts and charges */
	if (mc.moved_swap) {
		/* uncharge swap account from the old cgroup */
		if (!mem_cgroup_is_root(mc.from))
			res_counter_uncharge(&mc.from->memsw,
						PAGE_SIZE * mc.moved_swap);
L
Li Zefan 已提交
6657 6658 6659

		for (i = 0; i < mc.moved_swap; i++)
			css_put(&mc.from->css);
6660 6661 6662 6663 6664 6665 6666 6667 6668

		if (!mem_cgroup_is_root(mc.to)) {
			/*
			 * we charged both to->res and to->memsw, so we should
			 * uncharge to->res.
			 */
			res_counter_uncharge(&mc.to->res,
						PAGE_SIZE * mc.moved_swap);
		}
L
Li Zefan 已提交
6669
		/* we've already done css_get(mc.to) */
6670 6671
		mc.moved_swap = 0;
	}
6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686
	memcg_oom_recover(from);
	memcg_oom_recover(to);
	wake_up_all(&mc.waitq);
}

static void mem_cgroup_clear_mc(void)
{
	struct mem_cgroup *from = mc.from;

	/*
	 * we must clear moving_task before waking up waiters at the end of
	 * task migration.
	 */
	mc.moving_task = NULL;
	__mem_cgroup_clear_mc();
6687
	spin_lock(&mc.lock);
6688 6689
	mc.from = NULL;
	mc.to = NULL;
6690
	spin_unlock(&mc.lock);
6691
	mem_cgroup_end_move(from);
6692 6693
}

6694
static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
6695
				 struct cgroup_taskset *tset)
6696
{
6697
	struct task_struct *p = cgroup_taskset_first(tset);
6698
	int ret = 0;
6699
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6700
	unsigned long move_charge_at_immigrate;
6701

6702 6703 6704 6705 6706 6707 6708
	/*
	 * We are now commited to this value whatever it is. Changes in this
	 * tunable will only affect upcoming migrations, not the current one.
	 * So we need to save it, and keep it going.
	 */
	move_charge_at_immigrate  = memcg->move_charge_at_immigrate;
	if (move_charge_at_immigrate) {
6709 6710 6711
		struct mm_struct *mm;
		struct mem_cgroup *from = mem_cgroup_from_task(p);

6712
		VM_BUG_ON(from == memcg);
6713 6714 6715 6716 6717

		mm = get_task_mm(p);
		if (!mm)
			return 0;
		/* We move charges only when we move a owner of the mm */
6718 6719 6720 6721
		if (mm->owner == p) {
			VM_BUG_ON(mc.from);
			VM_BUG_ON(mc.to);
			VM_BUG_ON(mc.precharge);
6722
			VM_BUG_ON(mc.moved_charge);
6723
			VM_BUG_ON(mc.moved_swap);
6724
			mem_cgroup_start_move(from);
6725
			spin_lock(&mc.lock);
6726
			mc.from = from;
6727
			mc.to = memcg;
6728
			mc.immigrate_flags = move_charge_at_immigrate;
6729
			spin_unlock(&mc.lock);
6730
			/* We set mc.moving_task later */
6731 6732 6733 6734

			ret = mem_cgroup_precharge_mc(mm);
			if (ret)
				mem_cgroup_clear_mc();
6735 6736
		}
		mmput(mm);
6737 6738 6739 6740
	}
	return ret;
}

6741
static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
6742
				     struct cgroup_taskset *tset)
6743
{
6744
	mem_cgroup_clear_mc();
6745 6746
}

6747 6748 6749
static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
				unsigned long addr, unsigned long end,
				struct mm_walk *walk)
6750
{
6751 6752 6753 6754
	int ret = 0;
	struct vm_area_struct *vma = walk->private;
	pte_t *pte;
	spinlock_t *ptl;
6755 6756 6757 6758
	enum mc_target_type target_type;
	union mc_target target;
	struct page *page;
	struct page_cgroup *pc;
6759

6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770
	/*
	 * We don't take compound_lock() here but no race with splitting thp
	 * happens because:
	 *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
	 *    under splitting, which means there's no concurrent thp split,
	 *  - if another thread runs into split_huge_page() just after we
	 *    entered this if-block, the thread must wait for page table lock
	 *    to be unlocked in __split_huge_page_splitting(), where the main
	 *    part of thp split is not executed yet.
	 */
	if (pmd_trans_huge_lock(pmd, vma) == 1) {
6771
		if (mc.precharge < HPAGE_PMD_NR) {
6772 6773 6774 6775 6776 6777 6778 6779 6780
			spin_unlock(&vma->vm_mm->page_table_lock);
			return 0;
		}
		target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
		if (target_type == MC_TARGET_PAGE) {
			page = target.page;
			if (!isolate_lru_page(page)) {
				pc = lookup_page_cgroup(page);
				if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
6781
							pc, mc.from, mc.to)) {
6782 6783 6784 6785 6786 6787 6788 6789
					mc.precharge -= HPAGE_PMD_NR;
					mc.moved_charge += HPAGE_PMD_NR;
				}
				putback_lru_page(page);
			}
			put_page(page);
		}
		spin_unlock(&vma->vm_mm->page_table_lock);
6790
		return 0;
6791 6792
	}

6793 6794
	if (pmd_trans_unstable(pmd))
		return 0;
6795 6796 6797 6798
retry:
	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
	for (; addr != end; addr += PAGE_SIZE) {
		pte_t ptent = *(pte++);
6799
		swp_entry_t ent;
6800 6801 6802 6803

		if (!mc.precharge)
			break;

6804
		switch (get_mctgt_type(vma, addr, ptent, &target)) {
6805 6806 6807 6808 6809
		case MC_TARGET_PAGE:
			page = target.page;
			if (isolate_lru_page(page))
				goto put;
			pc = lookup_page_cgroup(page);
6810
			if (!mem_cgroup_move_account(page, 1, pc,
6811
						     mc.from, mc.to)) {
6812
				mc.precharge--;
6813 6814
				/* we uncharge from mc.from later. */
				mc.moved_charge++;
6815 6816
			}
			putback_lru_page(page);
6817
put:			/* get_mctgt_type() gets the page */
6818 6819
			put_page(page);
			break;
6820 6821
		case MC_TARGET_SWAP:
			ent = target.ent;
6822
			if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
6823
				mc.precharge--;
6824 6825 6826
				/* we fixup refcnts and charges later. */
				mc.moved_swap++;
			}
6827
			break;
6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841
		default:
			break;
		}
	}
	pte_unmap_unlock(pte - 1, ptl);
	cond_resched();

	if (addr != end) {
		/*
		 * We have consumed all precharges we got in can_attach().
		 * We try charge one by one, but don't do any additional
		 * charges to mc.to if we have failed in charge once in attach()
		 * phase.
		 */
6842
		ret = mem_cgroup_do_precharge(1);
6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854
		if (!ret)
			goto retry;
	}

	return ret;
}

static void mem_cgroup_move_charge(struct mm_struct *mm)
{
	struct vm_area_struct *vma;

	lru_add_drain_all();
6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867
retry:
	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
		/*
		 * Someone who are holding the mmap_sem might be waiting in
		 * waitq. So we cancel all extra charges, wake up all waiters,
		 * and retry. Because we cancel precharges, we might not be able
		 * to move enough charges, but moving charge is a best-effort
		 * feature anyway, so it wouldn't be a big problem.
		 */
		__mem_cgroup_clear_mc();
		cond_resched();
		goto retry;
	}
6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		int ret;
		struct mm_walk mem_cgroup_move_charge_walk = {
			.pmd_entry = mem_cgroup_move_charge_pte_range,
			.mm = mm,
			.private = vma,
		};
		if (is_vm_hugetlb_page(vma))
			continue;
		ret = walk_page_range(vma->vm_start, vma->vm_end,
						&mem_cgroup_move_charge_walk);
		if (ret)
			/*
			 * means we have consumed all precharges and failed in
			 * doing additional charge. Just abandon here.
			 */
			break;
	}
6886
	up_read(&mm->mmap_sem);
6887 6888
}

6889
static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
6890
				 struct cgroup_taskset *tset)
B
Balbir Singh 已提交
6891
{
6892
	struct task_struct *p = cgroup_taskset_first(tset);
6893
	struct mm_struct *mm = get_task_mm(p);
6894 6895

	if (mm) {
6896 6897
		if (mc.to)
			mem_cgroup_move_charge(mm);
6898 6899
		mmput(mm);
	}
6900 6901
	if (mc.to)
		mem_cgroup_clear_mc();
B
Balbir Singh 已提交
6902
}
6903
#else	/* !CONFIG_MMU */
6904
static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
6905
				 struct cgroup_taskset *tset)
6906 6907 6908
{
	return 0;
}
6909
static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
6910
				     struct cgroup_taskset *tset)
6911 6912
{
}
6913
static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
6914
				 struct cgroup_taskset *tset)
6915 6916 6917
{
}
#endif
B
Balbir Singh 已提交
6918

6919 6920 6921 6922
/*
 * Cgroup retains root cgroups across [un]mount cycles making it necessary
 * to verify sane_behavior flag on each mount attempt.
 */
6923
static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
6924 6925 6926 6927 6928 6929
{
	/*
	 * use_hierarchy is forced with sane_behavior.  cgroup core
	 * guarantees that @root doesn't have any children, so turning it
	 * on for the root memcg is enough.
	 */
6930 6931
	if (cgroup_sane_behavior(root_css->cgroup))
		mem_cgroup_from_css(root_css)->use_hierarchy = true;
6932 6933
}

B
Balbir Singh 已提交
6934 6935 6936
struct cgroup_subsys mem_cgroup_subsys = {
	.name = "memory",
	.subsys_id = mem_cgroup_subsys_id,
6937
	.css_alloc = mem_cgroup_css_alloc,
6938
	.css_online = mem_cgroup_css_online,
6939 6940
	.css_offline = mem_cgroup_css_offline,
	.css_free = mem_cgroup_css_free,
6941 6942
	.can_attach = mem_cgroup_can_attach,
	.cancel_attach = mem_cgroup_cancel_attach,
B
Balbir Singh 已提交
6943
	.attach = mem_cgroup_move_task,
6944
	.bind = mem_cgroup_bind,
6945
	.base_cftypes = mem_cgroup_files,
6946
	.early_init = 0,
K
KAMEZAWA Hiroyuki 已提交
6947
	.use_id = 1,
B
Balbir Singh 已提交
6948
};
6949

A
Andrew Morton 已提交
6950
#ifdef CONFIG_MEMCG_SWAP
6951 6952 6953
static int __init enable_swap_account(char *s)
{
	/* consider enabled if no parameter or 1 is given */
6954
	if (!strcmp(s, "1"))
6955
		really_do_swap_account = 1;
6956
	else if (!strcmp(s, "0"))
6957 6958 6959
		really_do_swap_account = 0;
	return 1;
}
6960
__setup("swapaccount=", enable_swap_account);
6961

6962 6963
static void __init memsw_file_init(void)
{
6964 6965 6966 6967 6968 6969 6970 6971 6972
	WARN_ON(cgroup_add_cftypes(&mem_cgroup_subsys, memsw_cgroup_files));
}

static void __init enable_swap_cgroup(void)
{
	if (!mem_cgroup_disabled() && really_do_swap_account) {
		do_swap_account = 1;
		memsw_file_init();
	}
6973
}
6974

6975
#else
6976
static void __init enable_swap_cgroup(void)
6977 6978
{
}
6979
#endif
6980 6981

/*
6982 6983 6984 6985 6986 6987
 * subsys_initcall() for memory controller.
 *
 * Some parts like hotcpu_notifier() have to be initialized from this context
 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
 * everything that doesn't depend on a specific mem_cgroup structure should
 * be initialized from here.
6988 6989 6990 6991
 */
static int __init mem_cgroup_init(void)
{
	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
6992
	enable_swap_cgroup();
6993
	mem_cgroup_soft_limit_tree_init();
6994
	memcg_stock_init();
6995 6996 6997
	return 0;
}
subsys_initcall(mem_cgroup_init);