memcontrol.h 29.1 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
22
#include <linux/cgroup.h>
23
#include <linux/vm_event_item.h>
24
#include <linux/hardirq.h>
25
#include <linux/jump_label.h>
M
Michal Hocko 已提交
26 27 28
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
29 30
#include <linux/mm.h>
#include <linux/vmstat.h>
M
Michal Hocko 已提交
31
#include <linux/writeback.h>
32
#include <linux/page-flags.h>
33

34
struct mem_cgroup;
35 36
struct page;
struct mm_struct;
37
struct kmem_cache;
38

39 40 41 42 43 44 45 46 47
/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
	MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	MEMCG_SWAP,
	MEMCG_SOCK,
	/* XXX: why are these zone and not node counters? */
	MEMCG_KERNEL_STACK_KB,
48
	MEMCG_NR_STAT,
49 50
};

51 52 53 54 55 56 57 58 59
/* Cgroup-specific events, on top of universal VM events */
enum memcg_event_item {
	MEMCG_LOW = NR_VM_EVENT_ITEMS,
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
	MEMCG_NR_EVENTS,
};

60
struct mem_cgroup_reclaim_cookie {
61
	pg_data_t *pgdat;
62 63 64 65
	int priority;
	unsigned int generation;
};

66 67 68 69 70 71 72 73 74 75
#ifdef CONFIG_MEMCG

#define MEM_CGROUP_ID_SHIFT	16
#define MEM_CGROUP_ID_MAX	USHRT_MAX

struct mem_cgroup_id {
	int id;
	atomic_t ref;
};

M
Michal Hocko 已提交
76 77 78 79 80 81 82 83 84 85 86 87 88 89
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_TARGET_NUMAINFO,
	MEM_CGROUP_NTARGETS,
};

struct mem_cgroup_stat_cpu {
90
	long count[MEMCG_NR_STAT];
M
Michal Hocko 已提交
91 92 93 94 95 96 97 98 99 100 101
	unsigned long events[MEMCG_NR_EVENTS];
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

102 103 104 105
struct lruvec_stat {
	long count[NR_VM_NODE_STAT_ITEMS];
};

M
Michal Hocko 已提交
106 107 108
/*
 * per-zone information in memory controller.
 */
109
struct mem_cgroup_per_node {
M
Michal Hocko 已提交
110
	struct lruvec		lruvec;
111 112 113 114

	struct lruvec_stat __percpu *lruvec_stat_cpu;
	atomic_long_t		lruvec_stat[NR_VM_NODE_STAT_ITEMS];

115
	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
M
Michal Hocko 已提交
116 117 118 119 120 121 122

	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];

	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
123 124 125
	bool			congested;	/* memcg has many dirty pages */
						/* backed by a congested BDI */

M
Michal Hocko 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

156 157 158 159 160 161
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

M
Michal Hocko 已提交
162 163 164 165 166 167 168 169 170
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

171 172 173
	/* Private memcg ID. Used to ID objects that outlive the cgroup */
	struct mem_cgroup_id id;

M
Michal Hocko 已提交
174 175
	/* Accounted resources */
	struct page_counter memory;
176
	struct page_counter swap;
177 178

	/* Legacy consumer-oriented counters */
M
Michal Hocko 已提交
179 180
	struct page_counter memsw;
	struct page_counter kmem;
181
	struct page_counter tcpmem;
M
Michal Hocko 已提交
182 183 184 185 186

	/* Normal memory consumption range */
	unsigned long low;
	unsigned long high;

187 188 189
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

M
Michal Hocko 已提交
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

208 209 210
	/* handle for "memory.events" */
	struct cgroup_file events_file;

M
Michal Hocko 已提交
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	struct task_struct	*move_lock_task;
	unsigned long		move_lock_flags;
236 237 238 239

	struct mem_cgroup_stat_cpu __percpu *stat_cpu;
	atomic_long_t		stat[MEMCG_NR_STAT];
	atomic_long_t		events[MEMCG_NR_EVENTS];
M
Michal Hocko 已提交
240

241 242 243
	unsigned long		socket_pressure;

	/* Legacy tcp memory accounting */
244 245
	bool			tcpmem_active;
	int			tcpmem_pressure;
246

247
#ifndef CONFIG_SLOB
M
Michal Hocko 已提交
248 249
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
250
	enum memcg_kmem_state kmem_state;
251
	struct list_head kmem_caches;
M
Michal Hocko 已提交
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
#endif

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
#endif

	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
273

274 275 276 277 278 279
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
#define MEMCG_CHARGE_BATCH 32U

280
extern struct mem_cgroup *root_mem_cgroup;
T
Tejun Heo 已提交
281

282 283 284 285 286
static inline bool mem_cgroup_disabled(void)
{
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

287 288
bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);

289
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
290 291
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
292
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
293 294 295
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
296
void mem_cgroup_uncharge(struct page *page);
297
void mem_cgroup_uncharge_list(struct list_head *page_list);
298

299
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
300

301 302
static struct mem_cgroup_per_node *
mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
303
{
304
	return memcg->nodeinfo[nid];
305 306 307
}

/**
308 309
 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
 * @node: node of the wanted lruvec
310 311
 * @memcg: memcg of the wanted lruvec
 *
312 313
 * Returns the lru list vector holding pages for a given @node or a given
 * @memcg and @zone. This can be the node lruvec, if the memory controller
314 315
 * is disabled.
 */
316
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
317
				struct mem_cgroup *memcg)
318
{
319
	struct mem_cgroup_per_node *mz;
320 321 322
	struct lruvec *lruvec;

	if (mem_cgroup_disabled()) {
323
		lruvec = node_lruvec(pgdat);
324 325 326
		goto out;
	}

327
	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
328 329 330 331
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
M
Mel Gorman 已提交
332
	 * we have to be prepared to initialize lruvec->pgdat here;
333 334
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
335 336
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
337 338 339
	return lruvec;
}

M
Mel Gorman 已提交
340
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
341

342
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
343
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
344

M
Michal Hocko 已提交
345 346 347 348 349
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

350 351 352
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

M
Michal Hocko 已提交
353 354 355 356
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
357 358
int mem_cgroup_scan_tasks(struct mem_cgroup *,
			  int (*)(struct task_struct *, void *), void *);
M
Michal Hocko 已提交
359

360 361 362 363 364
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return 0;

365
	return memcg->id.id;
366
}
367
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
368

369 370 371 372 373 374 375 376 377 378 379
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	struct mem_cgroup_per_node *mz;

	if (mem_cgroup_disabled())
		return NULL;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->memcg;
}

380 381 382 383 384 385 386 387 388 389 390 391 392 393
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

M
Michal Hocko 已提交
394 395 396 397 398 399 400 401 402
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
G
Glauber Costa 已提交
403

404 405
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
406
{
407
	struct mem_cgroup *task_memcg;
408
	bool match = false;
409

410
	rcu_read_lock();
411
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
412
	if (task_memcg)
413
		match = mem_cgroup_is_descendant(task_memcg, memcg);
414
	rcu_read_unlock();
415
	return match;
416
}
417

418
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
419
ino_t page_cgroup_ino(struct page *page);
420

421 422 423 424 425 426 427
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return true;
	return !!(memcg->css.flags & CSS_ONLINE);
}

428 429 430
/*
 * For memory reclaim.
 */
431
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
M
Michal Hocko 已提交
432 433

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
434
		int zid, int nr_pages);
M
Michal Hocko 已提交
435

436 437 438
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
					   int nid, unsigned int lru_mask);

M
Michal Hocko 已提交
439 440 441
static inline
unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
{
442
	struct mem_cgroup_per_node *mz;
443 444
	unsigned long nr_pages = 0;
	int zid;
M
Michal Hocko 已提交
445

446
	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
447 448 449 450 451 452 453 454 455 456 457 458 459
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
		nr_pages += mz->lru_zone_size[zid][lru];
	return nr_pages;
}

static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	struct mem_cgroup_per_node *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->lru_zone_size[zone_idx][lru];
M
Michal Hocko 已提交
460 461
}

462 463
void mem_cgroup_handle_over_high(void);

464 465
unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg);

466 467
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
				struct task_struct *p);
468

469
static inline void mem_cgroup_oom_enable(void)
470
{
T
Tejun Heo 已提交
471 472
	WARN_ON(current->memcg_may_oom);
	current->memcg_may_oom = 1;
473 474
}

475
static inline void mem_cgroup_oom_disable(void)
476
{
T
Tejun Heo 已提交
477 478
	WARN_ON(!current->memcg_may_oom);
	current->memcg_may_oom = 0;
479 480
}

481 482
static inline bool task_in_memcg_oom(struct task_struct *p)
{
T
Tejun Heo 已提交
483
	return p->memcg_in_oom;
484 485
}

486
bool mem_cgroup_oom_synchronize(bool wait);
487

A
Andrew Morton 已提交
488
#ifdef CONFIG_MEMCG_SWAP
489 490
extern int do_swap_account;
#endif
491

492 493
struct mem_cgroup *lock_page_memcg(struct page *page);
void __unlock_page_memcg(struct mem_cgroup *memcg);
J
Johannes Weiner 已提交
494
void unlock_page_memcg(struct page *page);
495

496
/* idx can be of type enum memcg_stat_item or node_stat_item */
497
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
498
					     int idx)
499
{
500 501 502 503 504 505
	long x = atomic_long_read(&memcg->stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
506 507
}

508
/* idx can be of type enum memcg_stat_item or node_stat_item */
509
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
510
				     int idx, int val)
511
{
512 513 514 515 516 517 518 519 520 521 522
	long x;

	if (mem_cgroup_disabled())
		return;

	x = val + __this_cpu_read(memcg->stat_cpu->count[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &memcg->stat[idx]);
		x = 0;
	}
	__this_cpu_write(memcg->stat_cpu->count[idx], x);
523 524
}

525
/* idx can be of type enum memcg_stat_item or node_stat_item */
526
static inline void mod_memcg_state(struct mem_cgroup *memcg,
527
				   int idx, int val)
528
{
529 530 531
	unsigned long flags;

	local_irq_save(flags);
532
	__mod_memcg_state(memcg, idx, val);
533
	local_irq_restore(flags);
534 535
}

M
Michal Hocko 已提交
536
/**
537
 * mod_memcg_page_state - update page state statistics
J
Johannes Weiner 已提交
538
 * @page: the page
M
Michal Hocko 已提交
539 540 541
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
542 543 544
 * The @page must be locked or the caller must use lock_page_memcg()
 * to prevent double accounting when the page is concurrently being
 * moved to another memcg:
545
 *
546
 *   lock_page(page) or lock_page_memcg(page)
547
 *   if (TestClearPageState(page))
548
 *     mod_memcg_page_state(page, state, -1);
549
 *   unlock_page(page) or unlock_page_memcg(page)
550 551
 *
 * Kernel pages are an exception to this, since they'll never move.
M
Michal Hocko 已提交
552
 */
553
static inline void __mod_memcg_page_state(struct page *page,
554
					  int idx, int val)
555 556 557 558 559
{
	if (page->mem_cgroup)
		__mod_memcg_state(page->mem_cgroup, idx, val);
}

560
static inline void mod_memcg_page_state(struct page *page,
561
					int idx, int val)
M
Michal Hocko 已提交
562
{
J
Johannes Weiner 已提交
563
	if (page->mem_cgroup)
564
		mod_memcg_state(page->mem_cgroup, idx, val);
M
Michal Hocko 已提交
565 566
}

567 568
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
569
{
570
	struct mem_cgroup_per_node *pn;
571
	long x;
572 573 574 575 576

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
577 578 579 580 581 582
	x = atomic_long_read(&pn->lruvec_stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
583 584
}

585 586
static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
587
{
588
	struct mem_cgroup_per_node *pn;
589
	long x;
590

591
	/* Update node */
592
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
593

594 595
	if (mem_cgroup_disabled())
		return;
596

597
	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
598 599

	/* Update memcg */
600
	__mod_memcg_state(pn->memcg, idx, val);
601 602

	/* Update lruvec */
603 604 605 606 607 608
	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &pn->lruvec_stat[idx]);
		x = 0;
	}
	__this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
609 610 611 612 613
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
614 615 616
	unsigned long flags;

	local_irq_save(flags);
617
	__mod_lruvec_state(lruvec, idx, val);
618
	local_irq_restore(flags);
619 620 621 622 623
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
624 625
	pg_data_t *pgdat = page_pgdat(page);
	struct lruvec *lruvec;
626

627 628 629
	/* Untracked pages have no memcg, no lruvec. Update only the node */
	if (!page->mem_cgroup) {
		__mod_node_page_state(pgdat, idx, val);
630
		return;
631 632 633 634
	}

	lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
	__mod_lruvec_state(lruvec, idx, val);
635 636 637 638 639
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
640 641 642
	unsigned long flags;

	local_irq_save(flags);
643
	__mod_lruvec_page_state(page, idx, val);
644
	local_irq_restore(flags);
645 646
}

647
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
648 649
						gfp_t gfp_mask,
						unsigned long *total_scanned);
D
David Rientjes 已提交
650

651 652 653 654
/* idx can be of type enum memcg_event_item or vm_event_item */
static inline void __count_memcg_events(struct mem_cgroup *memcg,
					int idx, unsigned long count)
{
655 656 657 658 659 660 661 662 663 664 665
	unsigned long x;

	if (mem_cgroup_disabled())
		return;

	x = count + __this_cpu_read(memcg->stat_cpu->events[idx]);
	if (unlikely(x > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &memcg->events[idx]);
		x = 0;
	}
	__this_cpu_write(memcg->stat_cpu->events[idx], x);
666 667
}

668
static inline void count_memcg_events(struct mem_cgroup *memcg,
669
				      int idx, unsigned long count)
670
{
671 672 673
	unsigned long flags;

	local_irq_save(flags);
674
	__count_memcg_events(memcg, idx, count);
675
	local_irq_restore(flags);
676 677
}

678
/* idx can be of type enum memcg_event_item or vm_event_item */
679
static inline void count_memcg_page_event(struct page *page,
680
					  int idx)
681 682 683 684 685 686 687
{
	if (page->mem_cgroup)
		count_memcg_events(page->mem_cgroup, idx, 1);
}

static inline void count_memcg_event_mm(struct mm_struct *mm,
					enum vm_event_item idx)
688
{
M
Michal Hocko 已提交
689 690
	struct mem_cgroup *memcg;

691 692
	if (mem_cgroup_disabled())
		return;
M
Michal Hocko 已提交
693 694 695

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
696
	if (likely(memcg)) {
697
		count_memcg_events(memcg, idx, 1);
698 699 700
		if (idx == OOM_KILL)
			cgroup_file_notify(&memcg->events_file);
	}
M
Michal Hocko 已提交
701
	rcu_read_unlock();
702
}
703 704 705 706 707 708 709 710

static inline void mem_cgroup_event(struct mem_cgroup *memcg,
				    enum memcg_event_item event)
{
	count_memcg_events(memcg, event, 1);
	cgroup_file_notify(&memcg->events_file);
}

711
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
712
void mem_cgroup_split_huge_fixup(struct page *head);
713 714
#endif

A
Andrew Morton 已提交
715
#else /* CONFIG_MEMCG */
716 717 718 719

#define MEM_CGROUP_ID_SHIFT	0
#define MEM_CGROUP_ID_MAX	0

720 721
struct mem_cgroup;

722 723 724 725 726
static inline bool mem_cgroup_disabled(void)
{
	return true;
}

727
static inline void mem_cgroup_event(struct mem_cgroup *memcg,
728
				    enum memcg_event_item event)
729 730 731 732 733 734 735 736 737
{
}

static inline bool mem_cgroup_low(struct mem_cgroup *root,
				  struct mem_cgroup *memcg)
{
	return false;
}

738 739
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
740 741
					struct mem_cgroup **memcgp,
					bool compound)
742
{
743
	*memcgp = NULL;
744 745 746
	return 0;
}

747 748
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
749
					    bool lrucare, bool compound)
750 751 752
{
}

753
static inline void mem_cgroup_cancel_charge(struct page *page,
754 755
					    struct mem_cgroup *memcg,
					    bool compound)
756 757 758
{
}

759
static inline void mem_cgroup_uncharge(struct page *page)
760 761 762
{
}

763
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
764 765 766
{
}

767
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
768 769 770
{
}

771
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
772
				struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
773
{
774
	return node_lruvec(pgdat);
K
KAMEZAWA Hiroyuki 已提交
775 776
}

777
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
M
Mel Gorman 已提交
778
						    struct pglist_data *pgdat)
779
{
M
Mel Gorman 已提交
780
	return &pgdat->lruvec;
781 782
}

783
static inline bool mm_match_cgroup(struct mm_struct *mm,
784
		struct mem_cgroup *memcg)
785
{
786
	return true;
787 788
}

789 790
static inline bool task_in_mem_cgroup(struct task_struct *task,
				      const struct mem_cgroup *memcg)
791
{
792
	return true;
793 794
}

795 796 797 798 799 800 801 802 803 804 805 806 807
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

808 809 810 811 812 813
static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
		int (*fn)(struct task_struct *, void *), void *arg)
{
	return 0;
}

814
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
815
{
816 817 818 819 820 821 822 823
	return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(id);
	/* XXX: This should always return root_mem_cgroup */
	return NULL;
824
}
825

826 827 828 829 830
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	return NULL;
}

831
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
832
{
833
	return true;
834 835
}

836
static inline unsigned long
837
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
838 839 840
{
	return 0;
}
841 842 843 844 845 846
static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	return 0;
}
847

848 849 850 851 852 853 854
static inline unsigned long
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
			     int nid, unsigned int lru_mask)
{
	return 0;
}

855 856 857 858 859
static inline unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
{
	return 0;
}

860 861 862 863 864
static inline void
mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
}

865 866 867 868 869 870
static inline struct mem_cgroup *lock_page_memcg(struct page *page)
{
	return NULL;
}

static inline void __unlock_page_memcg(struct mem_cgroup *memcg)
871 872 873
{
}

J
Johannes Weiner 已提交
874
static inline void unlock_page_memcg(struct page *page)
875 876 877
{
}

878 879 880 881
static inline void mem_cgroup_handle_over_high(void)
{
}

882
static inline void mem_cgroup_oom_enable(void)
883 884 885
{
}

886
static inline void mem_cgroup_oom_disable(void)
887 888 889
{
}

890 891 892 893 894
static inline bool task_in_memcg_oom(struct task_struct *p)
{
	return false;
}

895
static inline bool mem_cgroup_oom_synchronize(bool wait)
896 897 898 899
{
	return false;
}

900
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
901
					     int idx)
902 903 904 905
{
	return 0;
}

906
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
907
				     int idx,
908
				     int nr)
909 910 911
{
}

912
static inline void mod_memcg_state(struct mem_cgroup *memcg,
913
				   int idx,
914
				   int nr)
915 916 917
{
}

918
static inline void __mod_memcg_page_state(struct page *page,
919
					  int idx,
920
					  int nr)
921 922 923
{
}

924
static inline void mod_memcg_page_state(struct page *page,
925
					int idx,
926
					int nr)
927 928 929
{
}

930 931
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
932
{
933
	return node_page_state(lruvec_pgdat(lruvec), idx);
934 935
}

936 937
static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
938
{
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
	mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
	__mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
	mod_node_page_state(page_pgdat(page), idx, val);
958 959
}

960
static inline
961
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
962 963
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
964
{
965
	return 0;
966 967
}

968
static inline void mem_cgroup_split_huge_fixup(struct page *head)
969 970 971
{
}

972 973 974 975 976 977 978
static inline void count_memcg_events(struct mem_cgroup *memcg,
				      enum vm_event_item idx,
				      unsigned long count)
{
}

static inline void count_memcg_page_event(struct page *page,
979
					  int idx)
980 981 982
{
}

983
static inline
984
void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
985 986
{
}
A
Andrew Morton 已提交
987
#endif /* CONFIG_MEMCG */
988

989
/* idx can be of type enum memcg_stat_item or node_stat_item */
990
static inline void __inc_memcg_state(struct mem_cgroup *memcg,
991
				     int idx)
992 993 994 995
{
	__mod_memcg_state(memcg, idx, 1);
}

996
/* idx can be of type enum memcg_stat_item or node_stat_item */
997
static inline void __dec_memcg_state(struct mem_cgroup *memcg,
998
				     int idx)
999 1000 1001 1002
{
	__mod_memcg_state(memcg, idx, -1);
}

1003
/* idx can be of type enum memcg_stat_item or node_stat_item */
1004
static inline void __inc_memcg_page_state(struct page *page,
1005
					  int idx)
1006 1007 1008 1009
{
	__mod_memcg_page_state(page, idx, 1);
}

1010
/* idx can be of type enum memcg_stat_item or node_stat_item */
1011
static inline void __dec_memcg_page_state(struct page *page,
1012
					  int idx)
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
{
	__mod_memcg_page_state(page, idx, -1);
}

static inline void __inc_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx)
{
	__mod_lruvec_state(lruvec, idx, 1);
}

static inline void __dec_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx)
{
	__mod_lruvec_state(lruvec, idx, -1);
}

static inline void __inc_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, 1);
}

static inline void __dec_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, -1);
}

1041
/* idx can be of type enum memcg_stat_item or node_stat_item */
1042
static inline void inc_memcg_state(struct mem_cgroup *memcg,
1043
				   int idx)
1044 1045 1046 1047
{
	mod_memcg_state(memcg, idx, 1);
}

1048
/* idx can be of type enum memcg_stat_item or node_stat_item */
1049
static inline void dec_memcg_state(struct mem_cgroup *memcg,
1050
				   int idx)
1051 1052 1053 1054
{
	mod_memcg_state(memcg, idx, -1);
}

1055
/* idx can be of type enum memcg_stat_item or node_stat_item */
1056
static inline void inc_memcg_page_state(struct page *page,
1057
					int idx)
1058 1059 1060 1061
{
	mod_memcg_page_state(page, idx, 1);
}

1062
/* idx can be of type enum memcg_stat_item or node_stat_item */
1063
static inline void dec_memcg_page_state(struct page *page,
1064
					int idx)
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
{
	mod_memcg_page_state(page, idx, -1);
}

static inline void inc_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx)
{
	mod_lruvec_state(lruvec, idx, 1);
}

static inline void dec_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx)
{
	mod_lruvec_state(lruvec, idx, -1);
}

static inline void inc_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, 1);
}

static inline void dec_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, -1);
}

1093
#ifdef CONFIG_CGROUP_WRITEBACK
T
Tejun Heo 已提交
1094

1095
struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
T
Tejun Heo 已提交
1096
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
1097 1098 1099
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback);
T
Tejun Heo 已提交
1100 1101 1102 1103 1104 1105 1106 1107

#else	/* CONFIG_CGROUP_WRITEBACK */

static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	return NULL;
}

1108
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
1109 1110
				       unsigned long *pfilepages,
				       unsigned long *pheadroom,
1111 1112 1113 1114 1115
				       unsigned long *pdirty,
				       unsigned long *pwriteback)
{
}

T
Tejun Heo 已提交
1116
#endif	/* CONFIG_CGROUP_WRITEBACK */
1117

G
Glauber Costa 已提交
1118
struct sock;
1119 1120
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1121
#ifdef CONFIG_MEMCG
1122 1123
extern struct static_key_false memcg_sockets_enabled_key;
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
1124 1125
void mem_cgroup_sk_alloc(struct sock *sk);
void mem_cgroup_sk_free(struct sock *sk);
1126
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1127
{
1128
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
1129 1130 1131 1132 1133 1134
		return true;
	do {
		if (time_before(jiffies, memcg->socket_pressure))
			return true;
	} while ((memcg = parent_mem_cgroup(memcg)));
	return false;
1135 1136
}
#else
1137
#define mem_cgroup_sockets_enabled 0
1138 1139
static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
static inline void mem_cgroup_sk_free(struct sock *sk) { };
1140
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1141 1142 1143 1144
{
	return false;
}
#endif
1145

1146 1147 1148 1149 1150 1151 1152
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
void memcg_kmem_put_cache(struct kmem_cache *cachep);
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			    struct mem_cgroup *memcg);
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
void memcg_kmem_uncharge(struct page *page, int order);

1153
#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
1154
extern struct static_key_false memcg_kmem_enabled_key;
1155
extern struct workqueue_struct *memcg_kmem_cache_wq;
1156

1157
extern int memcg_nr_cache_ids;
1158 1159
void memcg_get_cache_ids(void);
void memcg_put_cache_ids(void);
1160 1161 1162 1163 1164 1165

/*
 * Helper macro to loop through all memcg-specific caches. Callers must still
 * check if the cache is valid (it is either valid or NULL).
 * the slab_mutex must be held when looping through those caches
 */
1166
#define for_each_memcg_cache_index(_idx)	\
1167
	for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
1168

1169 1170
static inline bool memcg_kmem_enabled(void)
{
1171
	return static_branch_unlikely(&memcg_kmem_enabled_key);
1172 1173
}

M
Michal Hocko 已提交
1174
/*
J
Jesper Dangaard Brouer 已提交
1175
 * helper for accessing a memcg's index. It will be used as an index in the
M
Michal Hocko 已提交
1176 1177 1178 1179 1180 1181 1182
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}
1183

1184
#else
1185 1186 1187
#define for_each_memcg_cache_index(_idx)	\
	for (; NULL; )

1188 1189 1190 1191 1192
static inline bool memcg_kmem_enabled(void)
{
	return false;
}

1193 1194 1195 1196 1197
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return -1;
}

1198 1199 1200 1201 1202 1203 1204 1205
static inline void memcg_get_cache_ids(void)
{
}

static inline void memcg_put_cache_ids(void)
{
}

1206 1207
#endif /* CONFIG_MEMCG && !CONFIG_SLOB */

B
Balbir Singh 已提交
1208
#endif /* _LINUX_MEMCONTROL_H */