memcontrol.h 23.6 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
22
#include <linux/cgroup.h>
23
#include <linux/vm_event_item.h>
24
#include <linux/hardirq.h>
25
#include <linux/jump_label.h>
M
Michal Hocko 已提交
26 27 28 29 30
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
#include <linux/mmzone.h>
#include <linux/writeback.h>
31
#include <linux/page-flags.h>
32

33
struct mem_cgroup;
34 35
struct page;
struct mm_struct;
36
struct kmem_cache;
37

S
Sha Zhengju 已提交
38 39 40 41 42 43 44 45 46 47 48 49
/*
 * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c,
 * These two lists should keep in accord with each other.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE,		/* # of pages charged as cache */
	MEM_CGROUP_STAT_RSS,		/* # of pages charged as anon rss */
	MEM_CGROUP_STAT_RSS_HUGE,	/* # of pages charged as anon huge */
	MEM_CGROUP_STAT_FILE_MAPPED,	/* # of pages charged as file rss */
50
	MEM_CGROUP_STAT_DIRTY,          /* # of dirty pages in page cache */
51
	MEM_CGROUP_STAT_WRITEBACK,	/* # of pages under writeback */
S
Sha Zhengju 已提交
52 53
	MEM_CGROUP_STAT_SWAP,		/* # of pages, swapped out */
	MEM_CGROUP_STAT_NSTATS,
54
	/* default hierarchy stats */
55
	MEMCG_SOCK = MEM_CGROUP_STAT_NSTATS,
56 57
	MEMCG_SLAB_RECLAIMABLE,
	MEMCG_SLAB_UNRECLAIMABLE,
58
	MEMCG_NR_STAT,
59 60
};

61 62 63 64 65 66
struct mem_cgroup_reclaim_cookie {
	struct zone *zone;
	int priority;
	unsigned int generation;
};

67 68 69 70 71 72 73 74 75 76 77 78 79 80
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
	MEM_CGROUP_EVENTS_NSTATS,
	/* default hierarchy events */
	MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
	MEMCG_NR_EVENTS,
};

M
Michal Hocko 已提交
81 82 83 84 85 86 87 88 89 90 91 92 93
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_TARGET_NUMAINFO,
	MEM_CGROUP_NTARGETS,
};

A
Andrew Morton 已提交
94
#ifdef CONFIG_MEMCG
95 96 97 98

#define MEM_CGROUP_ID_SHIFT	16
#define MEM_CGROUP_ID_MAX	USHRT_MAX

M
Michal Hocko 已提交
99
struct mem_cgroup_stat_cpu {
100
	long count[MEMCG_NR_STAT];
M
Michal Hocko 已提交
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
	unsigned long events[MEMCG_NR_EVENTS];
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
	struct lruvec		lruvec;
	unsigned long		lru_size[NR_LRU_LISTS];

	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];

	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

159 160 161 162 163 164
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

M
Michal Hocko 已提交
165 166 167 168 169 170 171 172 173 174 175
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

	/* Accounted resources */
	struct page_counter memory;
176
	struct page_counter swap;
177 178

	/* Legacy consumer-oriented counters */
M
Michal Hocko 已提交
179 180
	struct page_counter memsw;
	struct page_counter kmem;
181
	struct page_counter tcpmem;
M
Michal Hocko 已提交
182 183 184 185 186

	/* Normal memory consumption range */
	unsigned long low;
	unsigned long high;

187 188 189
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

M
Michal Hocko 已提交
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

208 209 210
	/* handle for "memory.events" */
	struct cgroup_file events_file;

M
Michal Hocko 已提交
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	struct task_struct	*move_lock_task;
	unsigned long		move_lock_flags;
	/*
	 * percpu counter.
	 */
	struct mem_cgroup_stat_cpu __percpu *stat;

241 242 243
	unsigned long		socket_pressure;

	/* Legacy tcp memory accounting */
244 245
	bool			tcpmem_active;
	int			tcpmem_pressure;
246

247
#ifndef CONFIG_SLOB
M
Michal Hocko 已提交
248 249
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
250
	enum memcg_kmem_state kmem_state;
M
Michal Hocko 已提交
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
#endif

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
#endif

	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
272 273

extern struct mem_cgroup *root_mem_cgroup;
T
Tejun Heo 已提交
274

275 276 277 278 279
static inline bool mem_cgroup_disabled(void)
{
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

M
Michal Hocko 已提交
280 281 282 283 284 285 286
/**
 * mem_cgroup_events - count memory events against a cgroup
 * @memcg: the memory cgroup
 * @idx: the event index
 * @nr: the number of events to account for
 */
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
287
		       enum mem_cgroup_events_index idx,
M
Michal Hocko 已提交
288 289 290
		       unsigned int nr)
{
	this_cpu_add(memcg->stat->events[idx], nr);
291
	cgroup_file_notify(&memcg->events_file);
M
Michal Hocko 已提交
292
}
293 294 295

bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);

296
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
297 298
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
299
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
300 301 302
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
303
void mem_cgroup_uncharge(struct page *page);
304
void mem_cgroup_uncharge_list(struct list_head *page_list);
305

306
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
307

308 309
struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *);
310

311
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
312
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
313

M
Michal Hocko 已提交
314 315 316 317 318
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

319 320 321
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

M
Michal Hocko 已提交
322 323 324 325 326
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return 0;

	return memcg->css.id;
}

/**
 * mem_cgroup_from_id - look up a memcg from an id
 * @id: the id to look up
 *
 * Caller must hold rcu_read_lock() and use css_tryget() as necessary.
 */
static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	struct cgroup_subsys_state *css;

	css = css_from_id(id, &memory_cgrp_subsys);
	return mem_cgroup_from_css(css);
}

349 350 351 352 353 354 355 356 357 358 359 360 361 362
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

M
Michal Hocko 已提交
363 364 365 366 367 368 369 370 371
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
G
Glauber Costa 已提交
372

373 374
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
375
{
376
	struct mem_cgroup *task_memcg;
377
	bool match = false;
378

379
	rcu_read_lock();
380
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
381
	if (task_memcg)
382
		match = mem_cgroup_is_descendant(task_memcg, memcg);
383
	rcu_read_unlock();
384
	return match;
385
}
386

387
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
388
ino_t page_cgroup_ino(struct page *page);
389

390 391 392 393 394 395 396
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return true;
	return !!(memcg->css.flags & CSS_ONLINE);
}

397 398 399
/*
 * For memory reclaim.
 */
400
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
M
Michal Hocko 已提交
401 402 403 404 405 406 407 408 409 410 411 412 413

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
		int nr_pages);

static inline
unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	return mz->lru_size[lru];
}

414
static inline bool mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
M
Michal Hocko 已提交
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
{
	unsigned long inactive_ratio;
	unsigned long inactive;
	unsigned long active;
	unsigned long gb;

	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);

	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

	return inactive * inactive_ratio < active;
}

433 434
void mem_cgroup_handle_over_high(void);

435 436
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
				struct task_struct *p);
437

438
static inline void mem_cgroup_oom_enable(void)
439
{
T
Tejun Heo 已提交
440 441
	WARN_ON(current->memcg_may_oom);
	current->memcg_may_oom = 1;
442 443
}

444
static inline void mem_cgroup_oom_disable(void)
445
{
T
Tejun Heo 已提交
446 447
	WARN_ON(!current->memcg_may_oom);
	current->memcg_may_oom = 0;
448 449
}

450 451
static inline bool task_in_memcg_oom(struct task_struct *p)
{
T
Tejun Heo 已提交
452
	return p->memcg_in_oom;
453 454
}

455
bool mem_cgroup_oom_synchronize(bool wait);
456

A
Andrew Morton 已提交
457
#ifdef CONFIG_MEMCG_SWAP
458 459
extern int do_swap_account;
#endif
460

J
Johannes Weiner 已提交
461 462
void lock_page_memcg(struct page *page);
void unlock_page_memcg(struct page *page);
463

M
Michal Hocko 已提交
464 465
/**
 * mem_cgroup_update_page_stat - update page state statistics
J
Johannes Weiner 已提交
466
 * @page: the page
M
Michal Hocko 已提交
467 468 469
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
470 471 472
 * The @page must be locked or the caller must use lock_page_memcg()
 * to prevent double accounting when the page is concurrently being
 * moved to another memcg:
473
 *
474
 *   lock_page(page) or lock_page_memcg(page)
475
 *   if (TestClearPageState(page))
J
Johannes Weiner 已提交
476
 *     mem_cgroup_update_page_stat(page, state, -1);
477
 *   unlock_page(page) or unlock_page_memcg(page)
M
Michal Hocko 已提交
478
 */
J
Johannes Weiner 已提交
479
static inline void mem_cgroup_update_page_stat(struct page *page,
M
Michal Hocko 已提交
480 481
				 enum mem_cgroup_stat_index idx, int val)
{
482
	VM_BUG_ON(!(rcu_read_lock_held() || PageLocked(page)));
M
Michal Hocko 已提交
483

J
Johannes Weiner 已提交
484 485
	if (page->mem_cgroup)
		this_cpu_add(page->mem_cgroup->stat->count[idx], val);
M
Michal Hocko 已提交
486 487
}

J
Johannes Weiner 已提交
488
static inline void mem_cgroup_inc_page_stat(struct page *page,
S
Sha Zhengju 已提交
489
					    enum mem_cgroup_stat_index idx)
490
{
J
Johannes Weiner 已提交
491
	mem_cgroup_update_page_stat(page, idx, 1);
492 493
}

J
Johannes Weiner 已提交
494
static inline void mem_cgroup_dec_page_stat(struct page *page,
S
Sha Zhengju 已提交
495
					    enum mem_cgroup_stat_index idx)
496
{
J
Johannes Weiner 已提交
497
	mem_cgroup_update_page_stat(page, idx, -1);
498 499
}

500 501 502
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
						gfp_t gfp_mask,
						unsigned long *total_scanned);
D
David Rientjes 已提交
503

504 505 506
static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
					     enum vm_event_item idx)
{
M
Michal Hocko 已提交
507 508
	struct mem_cgroup *memcg;

509 510
	if (mem_cgroup_disabled())
		return;
M
Michal Hocko 已提交
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
		goto out;

	switch (idx) {
	case PGFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
		break;
	case PGMAJFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
529
}
530
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
531
void mem_cgroup_split_huge_fixup(struct page *head);
532 533
#endif

A
Andrew Morton 已提交
534
#else /* CONFIG_MEMCG */
535 536 537 538

#define MEM_CGROUP_ID_SHIFT	0
#define MEM_CGROUP_ID_MAX	0

539 540
struct mem_cgroup;

541 542 543 544 545
static inline bool mem_cgroup_disabled(void)
{
	return true;
}

546 547 548 549 550 551 552 553 554 555 556 557
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
				     enum mem_cgroup_events_index idx,
				     unsigned int nr)
{
}

static inline bool mem_cgroup_low(struct mem_cgroup *root,
				  struct mem_cgroup *memcg)
{
	return false;
}

558 559
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
560 561
					struct mem_cgroup **memcgp,
					bool compound)
562
{
563
	*memcgp = NULL;
564 565 566
	return 0;
}

567 568
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
569
					    bool lrucare, bool compound)
570 571 572
{
}

573
static inline void mem_cgroup_cancel_charge(struct page *page,
574 575
					    struct mem_cgroup *memcg,
					    bool compound)
576 577 578
{
}

579
static inline void mem_cgroup_uncharge(struct page *page)
580 581 582
{
}

583
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
584 585 586
{
}

587
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
588 589 590
{
}

591 592
static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
						    struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
593
{
594
	return &zone->lruvec;
K
KAMEZAWA Hiroyuki 已提交
595 596
}

597 598
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
						    struct zone *zone)
599
{
600
	return &zone->lruvec;
601 602
}

603
static inline bool mm_match_cgroup(struct mm_struct *mm,
604
		struct mem_cgroup *memcg)
605
{
606
	return true;
607 608
}

609 610
static inline bool task_in_mem_cgroup(struct task_struct *task,
				      const struct mem_cgroup *memcg)
611
{
612
	return true;
613 614
}

615 616 617 618 619 620 621 622 623 624 625 626 627
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

628
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
629
{
630 631 632 633 634 635 636 637
	return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(id);
	/* XXX: This should always return root_mem_cgroup */
	return NULL;
638
}
639

640
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
641
{
642
	return true;
643 644
}

645 646
static inline bool
mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
647 648 649 650
{
	return true;
}

651
static inline unsigned long
652
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
653 654 655 656
{
	return 0;
}

657 658 659
static inline void
mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
			      int increment)
K
KOSAKI Motohiro 已提交
660 661 662
{
}

663 664 665 666 667
static inline void
mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
}

J
Johannes Weiner 已提交
668
static inline void lock_page_memcg(struct page *page)
669 670 671
{
}

J
Johannes Weiner 已提交
672
static inline void unlock_page_memcg(struct page *page)
673 674 675
{
}

676 677 678 679
static inline void mem_cgroup_handle_over_high(void)
{
}

680
static inline void mem_cgroup_oom_enable(void)
681 682 683
{
}

684
static inline void mem_cgroup_oom_disable(void)
685 686 687
{
}

688 689 690 691 692
static inline bool task_in_memcg_oom(struct task_struct *p)
{
	return false;
}

693
static inline bool mem_cgroup_oom_synchronize(bool wait)
694 695 696 697
{
	return false;
}

J
Johannes Weiner 已提交
698
static inline void mem_cgroup_inc_page_stat(struct page *page,
S
Sha Zhengju 已提交
699
					    enum mem_cgroup_stat_index idx)
700 701 702
{
}

J
Johannes Weiner 已提交
703
static inline void mem_cgroup_dec_page_stat(struct page *page,
S
Sha Zhengju 已提交
704
					    enum mem_cgroup_stat_index idx)
705 706 707
{
}

708
static inline
709 710 711
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
712
{
713
	return 0;
714 715
}

716
static inline void mem_cgroup_split_huge_fixup(struct page *head)
717 718 719
{
}

720 721 722 723
static inline
void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
{
}
A
Andrew Morton 已提交
724
#endif /* CONFIG_MEMCG */
725

726
#ifdef CONFIG_CGROUP_WRITEBACK
T
Tejun Heo 已提交
727

728
struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
T
Tejun Heo 已提交
729
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
730 731 732
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback);
T
Tejun Heo 已提交
733 734 735 736 737 738 739 740

#else	/* CONFIG_CGROUP_WRITEBACK */

static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	return NULL;
}

741
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
742 743
				       unsigned long *pfilepages,
				       unsigned long *pheadroom,
744 745 746 747 748
				       unsigned long *pdirty,
				       unsigned long *pwriteback)
{
}

T
Tejun Heo 已提交
749
#endif	/* CONFIG_CGROUP_WRITEBACK */
750

G
Glauber Costa 已提交
751 752 753
struct sock;
void sock_update_memcg(struct sock *sk);
void sock_release_memcg(struct sock *sk);
754 755
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
756
#ifdef CONFIG_MEMCG
757 758
extern struct static_key_false memcg_sockets_enabled_key;
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
759
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
760
{
761
	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
762 763 764 765 766 767
		return true;
	do {
		if (time_before(jiffies, memcg->socket_pressure))
			return true;
	} while ((memcg = parent_mem_cgroup(memcg)));
	return false;
768 769
}
#else
770
#define mem_cgroup_sockets_enabled 0
771
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
772 773 774 775
{
	return false;
}
#endif
776

777
#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
778
extern struct static_key_false memcg_kmem_enabled_key;
779

780
extern int memcg_nr_cache_ids;
781 782
void memcg_get_cache_ids(void);
void memcg_put_cache_ids(void);
783 784 785 786 787 788

/*
 * Helper macro to loop through all memcg-specific caches. Callers must still
 * check if the cache is valid (it is either valid or NULL).
 * the slab_mutex must be held when looping through those caches
 */
789
#define for_each_memcg_cache_index(_idx)	\
790
	for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
791

792 793
static inline bool memcg_kmem_enabled(void)
{
794
	return static_branch_unlikely(&memcg_kmem_enabled_key);
795 796
}

797
static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
M
Michal Hocko 已提交
798
{
799
	return memcg->kmem_state == KMEM_ONLINE;
M
Michal Hocko 已提交
800
}
801

802 803 804 805 806 807 808 809 810 811 812
/*
 * In general, we'll do everything in our power to not incur in any overhead
 * for non-memcg users for the kmem functions. Not even a function call, if we
 * can avoid it.
 *
 * Therefore, we'll inline all those functions so that in the best case, we'll
 * see that kmemcg is off for everybody and proceed quickly.  If it is on,
 * we'll still do most of the flag checking inline. We check a lot of
 * conditions, but because they are pretty simple, they are expected to be
 * fast.
 */
813 814
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			      struct mem_cgroup *memcg);
815 816
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
void __memcg_kmem_uncharge(struct page *page, int order);
817

M
Michal Hocko 已提交
818
/*
J
Jesper Dangaard Brouer 已提交
819
 * helper for accessing a memcg's index. It will be used as an index in the
M
Michal Hocko 已提交
820 821 822 823 824 825 826
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}
827

V
Vladimir Davydov 已提交
828
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp);
829
void __memcg_kmem_put_cache(struct kmem_cache *cachep);
830

V
Vladimir Davydov 已提交
831
static inline bool __memcg_kmem_bypass(void)
832 833 834 835 836
{
	if (!memcg_kmem_enabled())
		return true;
	if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
		return true;
837
	return false;
838 839 840
}

/**
841 842 843 844
 * memcg_kmem_charge: charge a kmem page
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
845
 *
846
 * Returns 0 on success, an error code on failure.
847
 */
848 849
static __always_inline int memcg_kmem_charge(struct page *page,
					     gfp_t gfp, int order)
850
{
V
Vladimir Davydov 已提交
851 852 853
	if (__memcg_kmem_bypass())
		return 0;
	if (!(gfp & __GFP_ACCOUNT))
854 855
		return 0;
	return __memcg_kmem_charge(page, gfp, order);
856 857 858
}

/**
859 860 861
 * memcg_kmem_uncharge: uncharge a kmem page
 * @page: page to uncharge
 * @order: allocation order
862
 */
863
static __always_inline void memcg_kmem_uncharge(struct page *page, int order)
864 865
{
	if (memcg_kmem_enabled())
866
		__memcg_kmem_uncharge(page, order);
867 868
}

869 870 871 872
/**
 * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation
 * @cachep: the original global kmem cache
 *
873
 * All memory allocated from a per-memcg cache is charged to the owner memcg.
874 875 876 877
 */
static __always_inline struct kmem_cache *
memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
{
V
Vladimir Davydov 已提交
878
	if (__memcg_kmem_bypass())
879
		return cachep;
V
Vladimir Davydov 已提交
880
	return __memcg_kmem_get_cache(cachep, gfp);
881
}
882 883 884 885 886 887

static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
{
	if (memcg_kmem_enabled())
		__memcg_kmem_put_cache(cachep);
}
888 889 890 891 892 893 894 895 896 897 898 899 900 901

/**
 * memcg_kmem_update_page_stat - update kmem page state statistics
 * @page: the page
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 */
static inline void memcg_kmem_update_page_stat(struct page *page,
				enum mem_cgroup_stat_index idx, int val)
{
	if (memcg_kmem_enabled() && page->mem_cgroup)
		this_cpu_add(page->mem_cgroup->stat->count[idx], val);
}

902
#else
903 904 905
#define for_each_memcg_cache_index(_idx)	\
	for (; NULL; )

906 907 908 909 910
static inline bool memcg_kmem_enabled(void)
{
	return false;
}

911
static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
912 913 914 915
{
	return false;
}

916
static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
917
{
918
	return 0;
919 920
}

921
static inline void memcg_kmem_uncharge(struct page *page, int order)
922 923
{
}
924 925 926 927 928 929

static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return -1;
}

930 931 932 933 934 935 936 937
static inline void memcg_get_cache_ids(void)
{
}

static inline void memcg_put_cache_ids(void)
{
}

938 939 940 941 942
static inline struct kmem_cache *
memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
{
	return cachep;
}
943 944 945 946

static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
{
}
947 948 949 950 951

static inline void memcg_kmem_update_page_stat(struct page *page,
				enum mem_cgroup_stat_index idx, int val)
{
}
952 953
#endif /* CONFIG_MEMCG && !CONFIG_SLOB */

B
Balbir Singh 已提交
954
#endif /* _LINUX_MEMCONTROL_H */