memcontrol.h 32.9 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
22
#include <linux/cgroup.h>
23
#include <linux/vm_event_item.h>
24
#include <linux/hardirq.h>
25
#include <linux/jump_label.h>
M
Michal Hocko 已提交
26 27 28
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
29 30
#include <linux/mm.h>
#include <linux/vmstat.h>
M
Michal Hocko 已提交
31
#include <linux/writeback.h>
32
#include <linux/page-flags.h>
33

34
struct mem_cgroup;
35 36
struct page;
struct mm_struct;
37
struct kmem_cache;
38

39 40 41 42 43 44 45 46 47
/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
	MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	MEMCG_SWAP,
	MEMCG_SOCK,
	/* XXX: why are these zone and not node counters? */
	MEMCG_KERNEL_STACK_KB,
48
	MEMCG_NR_STAT,
49 50
};

51 52
enum memcg_memory_event {
	MEMCG_LOW,
53 54 55
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
R
Roman Gushchin 已提交
56
	MEMCG_OOM_KILL,
57 58
	MEMCG_SWAP_MAX,
	MEMCG_SWAP_FAIL,
59
	MEMCG_NR_MEMORY_EVENTS,
60 61
};

R
Roman Gushchin 已提交
62 63 64 65 66 67
enum mem_cgroup_protection {
	MEMCG_PROT_NONE,
	MEMCG_PROT_LOW,
	MEMCG_PROT_MIN,
};

68
struct mem_cgroup_reclaim_cookie {
69
	pg_data_t *pgdat;
70 71 72 73
	int priority;
	unsigned int generation;
};

74 75 76 77 78 79 80
#ifdef CONFIG_MEMCG

#define MEM_CGROUP_ID_SHIFT	16
#define MEM_CGROUP_ID_MAX	USHRT_MAX

struct mem_cgroup_id {
	int id;
81
	refcount_t ref;
82 83
};

M
Michal Hocko 已提交
84 85 86 87 88 89 90 91 92 93 94 95 96 97
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_TARGET_NUMAINFO,
	MEM_CGROUP_NTARGETS,
};

struct mem_cgroup_stat_cpu {
98
	long count[MEMCG_NR_STAT];
99
	unsigned long events[NR_VM_EVENT_ITEMS];
M
Michal Hocko 已提交
100 101 102 103 104 105 106 107 108 109
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

110 111 112 113
struct lruvec_stat {
	long count[NR_VM_NODE_STAT_ITEMS];
};

114 115 116 117 118 119 120 121 122
/*
 * Bitmap of shrinker::id corresponding to memcg-aware shrinkers,
 * which have elements charged to this memcg.
 */
struct memcg_shrinker_map {
	struct rcu_head rcu;
	unsigned long map[0];
};

M
Michal Hocko 已提交
123 124 125
/*
 * per-zone information in memory controller.
 */
126
struct mem_cgroup_per_node {
M
Michal Hocko 已提交
127
	struct lruvec		lruvec;
128 129 130 131

	struct lruvec_stat __percpu *lruvec_stat_cpu;
	atomic_long_t		lruvec_stat[NR_VM_NODE_STAT_ITEMS];

132
	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
M
Michal Hocko 已提交
133 134 135

	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];

136 137 138
#ifdef CONFIG_MEMCG_KMEM
	struct memcg_shrinker_map __rcu	*shrinker_map;
#endif
M
Michal Hocko 已提交
139 140 141 142
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
143 144 145
	bool			congested;	/* memcg has many dirty pages */
						/* backed by a congested BDI */

M
Michal Hocko 已提交
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

176 177 178 179 180 181
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

182 183 184 185 186 187 188 189 190
#if defined(CONFIG_SMP)
struct memcg_padding {
	char x[0];
} ____cacheline_internodealigned_in_smp;
#define MEMCG_PADDING(name)      struct memcg_padding name;
#else
#define MEMCG_PADDING(name)
#endif

M
Michal Hocko 已提交
191 192 193 194 195 196 197 198 199
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

200 201 202
	/* Private memcg ID. Used to ID objects that outlive the cgroup */
	struct mem_cgroup_id id;

M
Michal Hocko 已提交
203 204
	/* Accounted resources */
	struct page_counter memory;
205
	struct page_counter swap;
206 207

	/* Legacy consumer-oriented counters */
M
Michal Hocko 已提交
208 209
	struct page_counter memsw;
	struct page_counter kmem;
210
	struct page_counter tcpmem;
M
Michal Hocko 已提交
211

212
	/* Upper bound of normal memory consumption range */
M
Michal Hocko 已提交
213 214
	unsigned long high;

215 216 217
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

M
Michal Hocko 已提交
218 219 220 221 222 223 224 225 226 227
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

228 229 230 231 232
	/*
	 * Should the OOM killer kill all belonging tasks, had it kill one?
	 */
	bool oom_group;

M
Michal Hocko 已提交
233 234 235 236 237 238 239 240
	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

241
	/* memory.events */
242 243
	struct cgroup_file events_file;

244 245 246
	/* handle for "memory.swap.events" */
	struct cgroup_file swap_events_file;

M
Michal Hocko 已提交
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
264 265 266 267 268 269
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	unsigned long		move_lock_flags;

	MEMCG_PADDING(_pad1_);

M
Michal Hocko 已提交
270 271 272 273 274
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	struct task_struct	*move_lock_task;
275

276
	/* memory.stat */
277
	struct mem_cgroup_stat_cpu __percpu *stat_cpu;
278 279 280

	MEMCG_PADDING(_pad2_);

281
	atomic_long_t		stat[MEMCG_NR_STAT];
282
	atomic_long_t		events[NR_VM_EVENT_ITEMS];
283
	atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
M
Michal Hocko 已提交
284

285 286 287
	unsigned long		socket_pressure;

	/* Legacy tcp memory accounting */
288 289
	bool			tcpmem_active;
	int			tcpmem_pressure;
290

291
#ifdef CONFIG_MEMCG_KMEM
M
Michal Hocko 已提交
292 293
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
294
	enum memcg_kmem_state kmem_state;
295
	struct list_head kmem_caches;
M
Michal Hocko 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
#endif

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
#endif

	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
317

318 319 320 321 322 323
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
#define MEMCG_CHARGE_BATCH 32U

324
extern struct mem_cgroup *root_mem_cgroup;
T
Tejun Heo 已提交
325

326 327 328 329 330
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

331 332 333 334 335
static inline bool mem_cgroup_disabled(void)
{
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

R
Roman Gushchin 已提交
336 337
enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
						struct mem_cgroup *memcg);
338

339
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
340 341
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
342 343 344
int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
345
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
346 347 348
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
349
void mem_cgroup_uncharge(struct page *page);
350
void mem_cgroup_uncharge_list(struct list_head *page_list);
351

352
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
353

354 355
static struct mem_cgroup_per_node *
mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
356
{
357
	return memcg->nodeinfo[nid];
358 359 360
}

/**
361 362
 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
 * @node: node of the wanted lruvec
363 364
 * @memcg: memcg of the wanted lruvec
 *
365 366
 * Returns the lru list vector holding pages for a given @node or a given
 * @memcg and @zone. This can be the node lruvec, if the memory controller
367 368
 * is disabled.
 */
369
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
370
				struct mem_cgroup *memcg)
371
{
372
	struct mem_cgroup_per_node *mz;
373 374 375
	struct lruvec *lruvec;

	if (mem_cgroup_disabled()) {
376
		lruvec = node_lruvec(pgdat);
377 378 379
		goto out;
	}

380
	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
381 382 383 384
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
M
Mel Gorman 已提交
385
	 * we have to be prepared to initialize lruvec->pgdat here;
386 387
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
388 389
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
390 391 392
	return lruvec;
}

M
Mel Gorman 已提交
393
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
394

395
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
396
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
397

398 399
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);

400 401
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page);

M
Michal Hocko 已提交
402 403 404 405 406
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

407 408
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
409 410
	if (memcg)
		css_put(&memcg->css);
411 412
}

413 414 415
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

M
Michal Hocko 已提交
416 417 418 419
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
420 421
int mem_cgroup_scan_tasks(struct mem_cgroup *,
			  int (*)(struct task_struct *, void *), void *);
M
Michal Hocko 已提交
422

423 424 425 426 427
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return 0;

428
	return memcg->id.id;
429
}
430
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
431

432 433 434 435 436 437 438 439 440 441 442
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	struct mem_cgroup_per_node *mz;

	if (mem_cgroup_disabled())
		return NULL;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->memcg;
}

443 444 445 446 447 448 449 450 451 452 453 454 455 456
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

M
Michal Hocko 已提交
457 458 459 460 461 462 463 464 465
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
G
Glauber Costa 已提交
466

467 468
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
469
{
470
	struct mem_cgroup *task_memcg;
471
	bool match = false;
472

473
	rcu_read_lock();
474
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
475
	if (task_memcg)
476
		match = mem_cgroup_is_descendant(task_memcg, memcg);
477
	rcu_read_unlock();
478
	return match;
479
}
480

481
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
482
ino_t page_cgroup_ino(struct page *page);
483

484 485 486 487 488 489 490
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return true;
	return !!(memcg->css.flags & CSS_ONLINE);
}

491 492 493
/*
 * For memory reclaim.
 */
494
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
M
Michal Hocko 已提交
495 496

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
497
		int zid, int nr_pages);
M
Michal Hocko 已提交
498

499 500 501
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
					   int nid, unsigned int lru_mask);

M
Michal Hocko 已提交
502 503 504
static inline
unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
{
505
	struct mem_cgroup_per_node *mz;
506 507
	unsigned long nr_pages = 0;
	int zid;
M
Michal Hocko 已提交
508

509
	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
510 511 512 513 514 515 516 517 518 519 520 521 522
	for (zid = 0; zid < MAX_NR_ZONES; zid++)
		nr_pages += mz->lru_zone_size[zid][lru];
	return nr_pages;
}

static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	struct mem_cgroup_per_node *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->lru_zone_size[zone_idx][lru];
M
Michal Hocko 已提交
523 524
}

525 526
void mem_cgroup_handle_over_high(void);

527
unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
528

529
void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
530
				struct task_struct *p);
531

532 533
void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);

534
static inline void mem_cgroup_enter_user_fault(void)
535
{
536 537
	WARN_ON(current->in_user_fault);
	current->in_user_fault = 1;
538 539
}

540
static inline void mem_cgroup_exit_user_fault(void)
541
{
542 543
	WARN_ON(!current->in_user_fault);
	current->in_user_fault = 0;
544 545
}

546 547
static inline bool task_in_memcg_oom(struct task_struct *p)
{
T
Tejun Heo 已提交
548
	return p->memcg_in_oom;
549 550
}

551
bool mem_cgroup_oom_synchronize(bool wait);
552 553 554
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
					    struct mem_cgroup *oom_domain);
void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
555

A
Andrew Morton 已提交
556
#ifdef CONFIG_MEMCG_SWAP
557 558
extern int do_swap_account;
#endif
559

560 561
struct mem_cgroup *lock_page_memcg(struct page *page);
void __unlock_page_memcg(struct mem_cgroup *memcg);
J
Johannes Weiner 已提交
562
void unlock_page_memcg(struct page *page);
563

564
/* idx can be of type enum memcg_stat_item or node_stat_item */
565
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
566
					     int idx)
567
{
568 569 570 571 572 573
	long x = atomic_long_read(&memcg->stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
574 575
}

576
/* idx can be of type enum memcg_stat_item or node_stat_item */
577
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
578
				     int idx, int val)
579
{
580 581 582 583 584 585 586 587 588 589 590
	long x;

	if (mem_cgroup_disabled())
		return;

	x = val + __this_cpu_read(memcg->stat_cpu->count[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &memcg->stat[idx]);
		x = 0;
	}
	__this_cpu_write(memcg->stat_cpu->count[idx], x);
591 592
}

593
/* idx can be of type enum memcg_stat_item or node_stat_item */
594
static inline void mod_memcg_state(struct mem_cgroup *memcg,
595
				   int idx, int val)
596
{
597 598 599
	unsigned long flags;

	local_irq_save(flags);
600
	__mod_memcg_state(memcg, idx, val);
601
	local_irq_restore(flags);
602 603
}

M
Michal Hocko 已提交
604
/**
605
 * mod_memcg_page_state - update page state statistics
J
Johannes Weiner 已提交
606
 * @page: the page
M
Michal Hocko 已提交
607 608 609
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
610 611 612
 * The @page must be locked or the caller must use lock_page_memcg()
 * to prevent double accounting when the page is concurrently being
 * moved to another memcg:
613
 *
614
 *   lock_page(page) or lock_page_memcg(page)
615
 *   if (TestClearPageState(page))
616
 *     mod_memcg_page_state(page, state, -1);
617
 *   unlock_page(page) or unlock_page_memcg(page)
618 619
 *
 * Kernel pages are an exception to this, since they'll never move.
M
Michal Hocko 已提交
620
 */
621
static inline void __mod_memcg_page_state(struct page *page,
622
					  int idx, int val)
623 624 625 626 627
{
	if (page->mem_cgroup)
		__mod_memcg_state(page->mem_cgroup, idx, val);
}

628
static inline void mod_memcg_page_state(struct page *page,
629
					int idx, int val)
M
Michal Hocko 已提交
630
{
J
Johannes Weiner 已提交
631
	if (page->mem_cgroup)
632
		mod_memcg_state(page->mem_cgroup, idx, val);
M
Michal Hocko 已提交
633 634
}

635 636
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
637
{
638
	struct mem_cgroup_per_node *pn;
639
	long x;
640 641 642 643 644

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
645 646 647 648 649 650
	x = atomic_long_read(&pn->lruvec_stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
651 652
}

653 654
static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
655
{
656
	struct mem_cgroup_per_node *pn;
657
	long x;
658

659
	/* Update node */
660
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
661

662 663
	if (mem_cgroup_disabled())
		return;
664

665
	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
666 667

	/* Update memcg */
668
	__mod_memcg_state(pn->memcg, idx, val);
669 670

	/* Update lruvec */
671 672 673 674 675 676
	x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
	if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &pn->lruvec_stat[idx]);
		x = 0;
	}
	__this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
677 678 679 680 681
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
682 683 684
	unsigned long flags;

	local_irq_save(flags);
685
	__mod_lruvec_state(lruvec, idx, val);
686
	local_irq_restore(flags);
687 688 689 690 691
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
692 693
	pg_data_t *pgdat = page_pgdat(page);
	struct lruvec *lruvec;
694

695 696 697
	/* Untracked pages have no memcg, no lruvec. Update only the node */
	if (!page->mem_cgroup) {
		__mod_node_page_state(pgdat, idx, val);
698
		return;
699 700 701 702
	}

	lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
	__mod_lruvec_state(lruvec, idx, val);
703 704 705 706 707
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
708 709 710
	unsigned long flags;

	local_irq_save(flags);
711
	__mod_lruvec_page_state(page, idx, val);
712
	local_irq_restore(flags);
713 714
}

715
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
716 717
						gfp_t gfp_mask,
						unsigned long *total_scanned);
D
David Rientjes 已提交
718

719
static inline void __count_memcg_events(struct mem_cgroup *memcg,
720 721
					enum vm_event_item idx,
					unsigned long count)
722
{
723 724 725 726 727 728 729 730 731 732 733
	unsigned long x;

	if (mem_cgroup_disabled())
		return;

	x = count + __this_cpu_read(memcg->stat_cpu->events[idx]);
	if (unlikely(x > MEMCG_CHARGE_BATCH)) {
		atomic_long_add(x, &memcg->events[idx]);
		x = 0;
	}
	__this_cpu_write(memcg->stat_cpu->events[idx], x);
734 735
}

736
static inline void count_memcg_events(struct mem_cgroup *memcg,
737 738
				      enum vm_event_item idx,
				      unsigned long count)
739
{
740 741 742
	unsigned long flags;

	local_irq_save(flags);
743
	__count_memcg_events(memcg, idx, count);
744
	local_irq_restore(flags);
745 746 747
}

static inline void count_memcg_page_event(struct page *page,
748
					  enum vm_event_item idx)
749 750 751 752 753 754 755
{
	if (page->mem_cgroup)
		count_memcg_events(page->mem_cgroup, idx, 1);
}

static inline void count_memcg_event_mm(struct mm_struct *mm,
					enum vm_event_item idx)
756
{
M
Michal Hocko 已提交
757 758
	struct mem_cgroup *memcg;

759 760
	if (mem_cgroup_disabled())
		return;
M
Michal Hocko 已提交
761 762 763

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
R
Roman Gushchin 已提交
764
	if (likely(memcg))
765
		count_memcg_events(memcg, idx, 1);
M
Michal Hocko 已提交
766
	rcu_read_unlock();
767
}
768

769 770
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
771
{
772
	atomic_long_inc(&memcg->memory_events[event]);
773 774 775
	cgroup_file_notify(&memcg->events_file);
}

R
Roman Gushchin 已提交
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return;

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (likely(memcg))
		memcg_memory_event(memcg, event);
	rcu_read_unlock();
}

791
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
792
void mem_cgroup_split_huge_fixup(struct page *head);
793 794
#endif

A
Andrew Morton 已提交
795
#else /* CONFIG_MEMCG */
796 797 798 799

#define MEM_CGROUP_ID_SHIFT	0
#define MEM_CGROUP_ID_MAX	0

800 801
struct mem_cgroup;

802 803 804 805 806
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return true;
}

807 808 809 810 811
static inline bool mem_cgroup_disabled(void)
{
	return true;
}

812 813
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
814 815 816
{
}

R
Roman Gushchin 已提交
817 818 819 820 821
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
}

R
Roman Gushchin 已提交
822 823
static inline enum mem_cgroup_protection mem_cgroup_protected(
	struct mem_cgroup *root, struct mem_cgroup *memcg)
824
{
R
Roman Gushchin 已提交
825
	return MEMCG_PROT_NONE;
826 827
}

828 829
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
830 831
					struct mem_cgroup **memcgp,
					bool compound)
832
{
833
	*memcgp = NULL;
834 835 836
	return 0;
}

837 838 839 840 841 842 843 844 845 846
static inline int mem_cgroup_try_charge_delay(struct page *page,
					      struct mm_struct *mm,
					      gfp_t gfp_mask,
					      struct mem_cgroup **memcgp,
					      bool compound)
{
	*memcgp = NULL;
	return 0;
}

847 848
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
849
					    bool lrucare, bool compound)
850 851 852
{
}

853
static inline void mem_cgroup_cancel_charge(struct page *page,
854 855
					    struct mem_cgroup *memcg,
					    bool compound)
856 857 858
{
}

859
static inline void mem_cgroup_uncharge(struct page *page)
860 861 862
{
}

863
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
864 865 866
{
}

867
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
868 869 870
{
}

871
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
872
				struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
873
{
874
	return node_lruvec(pgdat);
K
KAMEZAWA Hiroyuki 已提交
875 876
}

877
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
M
Mel Gorman 已提交
878
						    struct pglist_data *pgdat)
879
{
M
Mel Gorman 已提交
880
	return &pgdat->lruvec;
881 882
}

883
static inline bool mm_match_cgroup(struct mm_struct *mm,
884
		struct mem_cgroup *memcg)
885
{
886
	return true;
887 888
}

889 890
static inline bool task_in_mem_cgroup(struct task_struct *task,
				      const struct mem_cgroup *memcg)
891
{
892
	return true;
893 894
}

895 896 897 898 899
static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
{
	return NULL;
}

900 901 902 903 904
static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
	return NULL;
}

905 906 907 908
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}

909 910 911 912 913 914 915 916 917 918 919 920 921
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

922 923 924 925 926 927
static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
		int (*fn)(struct task_struct *, void *), void *arg)
{
	return 0;
}

928
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
929
{
930 931 932 933 934 935 936 937
	return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(id);
	/* XXX: This should always return root_mem_cgroup */
	return NULL;
938
}
939

940 941 942 943 944
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	return NULL;
}

945
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
946
{
947
	return true;
948 949
}

950
static inline unsigned long
951
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
952 953 954
{
	return 0;
}
955 956 957 958 959 960
static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	return 0;
}
961

962 963 964 965 966 967 968
static inline unsigned long
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
			     int nid, unsigned int lru_mask)
{
	return 0;
}

969
static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
970 971 972 973
{
	return 0;
}

974
static inline void
975 976 977 978 979 980
mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
{
}

static inline void
mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
981 982 983
{
}

984 985 986 987 988 989
static inline struct mem_cgroup *lock_page_memcg(struct page *page)
{
	return NULL;
}

static inline void __unlock_page_memcg(struct mem_cgroup *memcg)
990 991 992
{
}

J
Johannes Weiner 已提交
993
static inline void unlock_page_memcg(struct page *page)
994 995 996
{
}

997 998 999 1000
static inline void mem_cgroup_handle_over_high(void)
{
}

1001
static inline void mem_cgroup_enter_user_fault(void)
1002 1003 1004
{
}

1005
static inline void mem_cgroup_exit_user_fault(void)
1006 1007 1008
{
}

1009 1010 1011 1012 1013
static inline bool task_in_memcg_oom(struct task_struct *p)
{
	return false;
}

1014
static inline bool mem_cgroup_oom_synchronize(bool wait)
1015 1016 1017 1018
{
	return false;
}

1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
static inline struct mem_cgroup *mem_cgroup_get_oom_group(
	struct task_struct *victim, struct mem_cgroup *oom_domain)
{
	return NULL;
}

static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
{
}

1029
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
1030
					     int idx)
1031 1032 1033 1034
{
	return 0;
}

1035
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
1036
				     int idx,
1037
				     int nr)
1038 1039 1040
{
}

1041
static inline void mod_memcg_state(struct mem_cgroup *memcg,
1042
				   int idx,
1043
				   int nr)
1044 1045 1046
{
}

1047
static inline void __mod_memcg_page_state(struct page *page,
1048
					  int idx,
1049
					  int nr)
1050 1051 1052
{
}

1053
static inline void mod_memcg_page_state(struct page *page,
1054
					int idx,
1055
					int nr)
1056 1057 1058
{
}

1059 1060
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
1061
{
1062
	return node_page_state(lruvec_pgdat(lruvec), idx);
1063 1064
}

1065 1066
static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
1067
{
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
	mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
	__mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
	mod_node_page_state(page_pgdat(page), idx, val);
1087 1088
}

1089
static inline
1090
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1091 1092
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
1093
{
1094
	return 0;
1095 1096
}

1097
static inline void mem_cgroup_split_huge_fixup(struct page *head)
1098 1099 1100
{
}

1101 1102 1103 1104 1105 1106 1107
static inline void count_memcg_events(struct mem_cgroup *memcg,
				      enum vm_event_item idx,
				      unsigned long count)
{
}

static inline void count_memcg_page_event(struct page *page,
1108
					  int idx)
1109 1110 1111
{
}

1112
static inline
1113
void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
1114 1115
{
}
A
Andrew Morton 已提交
1116
#endif /* CONFIG_MEMCG */
1117

1118
/* idx can be of type enum memcg_stat_item or node_stat_item */
1119
static inline void __inc_memcg_state(struct mem_cgroup *memcg,
1120
				     int idx)
1121 1122 1123 1124
{
	__mod_memcg_state(memcg, idx, 1);
}

1125
/* idx can be of type enum memcg_stat_item or node_stat_item */
1126
static inline void __dec_memcg_state(struct mem_cgroup *memcg,
1127
				     int idx)
1128 1129 1130 1131
{
	__mod_memcg_state(memcg, idx, -1);
}

1132
/* idx can be of type enum memcg_stat_item or node_stat_item */
1133
static inline void __inc_memcg_page_state(struct page *page,
1134
					  int idx)
1135 1136 1137 1138
{
	__mod_memcg_page_state(page, idx, 1);
}

1139
/* idx can be of type enum memcg_stat_item or node_stat_item */
1140
static inline void __dec_memcg_page_state(struct page *page,
1141
					  int idx)
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
{
	__mod_memcg_page_state(page, idx, -1);
}

static inline void __inc_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx)
{
	__mod_lruvec_state(lruvec, idx, 1);
}

static inline void __dec_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx)
{
	__mod_lruvec_state(lruvec, idx, -1);
}

static inline void __inc_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, 1);
}

static inline void __dec_lruvec_page_state(struct page *page,
					   enum node_stat_item idx)
{
	__mod_lruvec_page_state(page, idx, -1);
}

1170
/* idx can be of type enum memcg_stat_item or node_stat_item */
1171
static inline void inc_memcg_state(struct mem_cgroup *memcg,
1172
				   int idx)
1173 1174 1175 1176
{
	mod_memcg_state(memcg, idx, 1);
}

1177
/* idx can be of type enum memcg_stat_item or node_stat_item */
1178
static inline void dec_memcg_state(struct mem_cgroup *memcg,
1179
				   int idx)
1180 1181 1182 1183
{
	mod_memcg_state(memcg, idx, -1);
}

1184
/* idx can be of type enum memcg_stat_item or node_stat_item */
1185
static inline void inc_memcg_page_state(struct page *page,
1186
					int idx)
1187 1188 1189 1190
{
	mod_memcg_page_state(page, idx, 1);
}

1191
/* idx can be of type enum memcg_stat_item or node_stat_item */
1192
static inline void dec_memcg_page_state(struct page *page,
1193
					int idx)
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
{
	mod_memcg_page_state(page, idx, -1);
}

static inline void inc_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx)
{
	mod_lruvec_state(lruvec, idx, 1);
}

static inline void dec_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx)
{
	mod_lruvec_state(lruvec, idx, -1);
}

static inline void inc_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, 1);
}

static inline void dec_lruvec_page_state(struct page *page,
					 enum node_stat_item idx)
{
	mod_lruvec_page_state(page, idx, -1);
}

1222
#ifdef CONFIG_CGROUP_WRITEBACK
T
Tejun Heo 已提交
1223 1224

struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
1225 1226 1227
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback);
T
Tejun Heo 已提交
1228 1229 1230 1231 1232 1233 1234 1235

#else	/* CONFIG_CGROUP_WRITEBACK */

static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	return NULL;
}

1236
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
1237 1238
				       unsigned long *pfilepages,
				       unsigned long *pheadroom,
1239 1240 1241 1242 1243
				       unsigned long *pdirty,
				       unsigned long *pwriteback)
{
}

T
Tejun Heo 已提交
1244
#endif	/* CONFIG_CGROUP_WRITEBACK */
1245

G
Glauber Costa 已提交
1246
struct sock;
1247 1248
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1249
#ifdef CONFIG_MEMCG
1250 1251
extern struct static_key_false memcg_sockets_enabled_key;
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
1252 1253
void mem_cgroup_sk_alloc(struct sock *sk);
void mem_cgroup_sk_free(struct sock *sk);
1254
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1255
{
1256
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
1257 1258 1259 1260 1261 1262
		return true;
	do {
		if (time_before(jiffies, memcg->socket_pressure))
			return true;
	} while ((memcg = parent_mem_cgroup(memcg)));
	return false;
1263 1264
}
#else
1265
#define mem_cgroup_sockets_enabled 0
1266 1267
static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
static inline void mem_cgroup_sk_free(struct sock *sk) { };
1268
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
1269 1270 1271 1272
{
	return false;
}
#endif
1273

1274 1275
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
void memcg_kmem_put_cache(struct kmem_cache *cachep);
1276 1277

#ifdef CONFIG_MEMCG_KMEM
1278 1279 1280 1281
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
void __memcg_kmem_uncharge(struct page *page, int order);
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			      struct mem_cgroup *memcg);
1282

1283
extern struct static_key_false memcg_kmem_enabled_key;
1284
extern struct workqueue_struct *memcg_kmem_cache_wq;
1285

1286
extern int memcg_nr_cache_ids;
1287 1288
void memcg_get_cache_ids(void);
void memcg_put_cache_ids(void);
1289 1290 1291 1292 1293 1294

/*
 * Helper macro to loop through all memcg-specific caches. Callers must still
 * check if the cache is valid (it is either valid or NULL).
 * the slab_mutex must be held when looping through those caches
 */
1295
#define for_each_memcg_cache_index(_idx)	\
1296
	for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
1297

1298 1299
static inline bool memcg_kmem_enabled(void)
{
1300
	return static_branch_unlikely(&memcg_kmem_enabled_key);
1301 1302
}

1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{
	if (memcg_kmem_enabled())
		return __memcg_kmem_charge(page, gfp, order);
	return 0;
}

static inline void memcg_kmem_uncharge(struct page *page, int order)
{
	if (memcg_kmem_enabled())
		__memcg_kmem_uncharge(page, order);
}

static inline int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp,
					  int order, struct mem_cgroup *memcg)
{
	if (memcg_kmem_enabled())
		return __memcg_kmem_charge_memcg(page, gfp, order, memcg);
	return 0;
}
M
Michal Hocko 已提交
1323
/*
J
Jesper Dangaard Brouer 已提交
1324
 * helper for accessing a memcg's index. It will be used as an index in the
M
Michal Hocko 已提交
1325 1326 1327 1328 1329 1330 1331
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}
1332

1333 1334
extern int memcg_expand_shrinker_maps(int new_id);

1335 1336
extern void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
				   int nid, int shrinker_id);
1337
#else
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347

static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{
	return 0;
}

static inline void memcg_kmem_uncharge(struct page *page, int order)
{
}

1348 1349 1350 1351 1352 1353 1354 1355 1356
static inline int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
{
	return 0;
}

static inline void __memcg_kmem_uncharge(struct page *page, int order)
{
}

1357 1358 1359
#define for_each_memcg_cache_index(_idx)	\
	for (; NULL; )

1360 1361 1362 1363 1364
static inline bool memcg_kmem_enabled(void)
{
	return false;
}

1365 1366 1367 1368 1369
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return -1;
}

1370 1371 1372 1373 1374 1375 1376 1377
static inline void memcg_get_cache_ids(void)
{
}

static inline void memcg_put_cache_ids(void)
{
}

1378 1379
static inline void memcg_set_shrinker_bit(struct mem_cgroup *memcg,
					  int nid, int shrinker_id) { }
1380
#endif /* CONFIG_MEMCG_KMEM */
1381

B
Balbir Singh 已提交
1382
#endif /* _LINUX_MEMCONTROL_H */