memcontrol.h 22.2 KB
Newer Older
B
Balbir Singh 已提交
1 2 3 4 5
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6 7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
B
Balbir Singh 已提交
9 10 11 12 13 14 15 16 17 18 19 20 21
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
22
#include <linux/cgroup.h>
23
#include <linux/vm_event_item.h>
24
#include <linux/hardirq.h>
25
#include <linux/jump_label.h>
M
Michal Hocko 已提交
26 27 28 29 30
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
#include <linux/mmzone.h>
#include <linux/writeback.h>
31

32
struct mem_cgroup;
33 34
struct page;
struct mm_struct;
35
struct kmem_cache;
36

S
Sha Zhengju 已提交
37 38 39 40 41 42 43 44 45 46 47 48
/*
 * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c,
 * These two lists should keep in accord with each other.
 */
enum mem_cgroup_stat_index {
	/*
	 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
	 */
	MEM_CGROUP_STAT_CACHE,		/* # of pages charged as cache */
	MEM_CGROUP_STAT_RSS,		/* # of pages charged as anon rss */
	MEM_CGROUP_STAT_RSS_HUGE,	/* # of pages charged as anon huge */
	MEM_CGROUP_STAT_FILE_MAPPED,	/* # of pages charged as file rss */
49
	MEM_CGROUP_STAT_DIRTY,          /* # of dirty pages in page cache */
50
	MEM_CGROUP_STAT_WRITEBACK,	/* # of pages under writeback */
S
Sha Zhengju 已提交
51 52
	MEM_CGROUP_STAT_SWAP,		/* # of pages, swapped out */
	MEM_CGROUP_STAT_NSTATS,
53 54
};

55 56 57 58 59 60
struct mem_cgroup_reclaim_cookie {
	struct zone *zone;
	int priority;
	unsigned int generation;
};

61 62 63 64 65 66 67 68 69 70 71 72 73 74
enum mem_cgroup_events_index {
	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */
	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */
	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */
	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */
	MEM_CGROUP_EVENTS_NSTATS,
	/* default hierarchy events */
	MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
	MEMCG_NR_EVENTS,
};

M
Michal Hocko 已提交
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_TARGET_NUMAINFO,
	MEM_CGROUP_NTARGETS,
};

struct cg_proto {
	struct page_counter	memory_allocated;	/* Current allocated memory. */
	int			memory_pressure;
91
	bool			active;
M
Michal Hocko 已提交
92 93
};

A
Andrew Morton 已提交
94
#ifdef CONFIG_MEMCG
M
Michal Hocko 已提交
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
struct mem_cgroup_stat_cpu {
	long count[MEM_CGROUP_STAT_NSTATS];
	unsigned long events[MEMCG_NR_EVENTS];
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

/*
 * per-zone information in memory controller.
 */
struct mem_cgroup_per_zone {
	struct lruvec		lruvec;
	unsigned long		lru_size[NR_LRU_LISTS];

	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];

	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_per_node {
	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

155 156 157 158 159 160
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

M
Michal Hocko 已提交
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

	/* Accounted resources */
	struct page_counter memory;
	struct page_counter memsw;
	struct page_counter kmem;

	/* Normal memory consumption range */
	unsigned long low;
	unsigned long high;

179 180 181
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

M
Michal Hocko 已提交
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/* css_online() has been completed */
	int initialized;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

203 204 205
	/* handle for "memory.events" */
	struct cgroup_file events_file;

M
Michal Hocko 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	struct task_struct	*move_lock_task;
	unsigned long		move_lock_flags;
	/*
	 * percpu counter.
	 */
	struct mem_cgroup_stat_cpu __percpu *stat;

#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
	struct cg_proto tcp_mem;
#endif
239
#ifndef CONFIG_SLOB
M
Michal Hocko 已提交
240 241
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
242
	enum memcg_kmem_state kmem_state;
M
Michal Hocko 已提交
243 244 245 246 247 248 249 250 251 252 253 254 255 256
#endif

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
#endif

257 258 259 260
#ifdef CONFIG_INET
	unsigned long		socket_pressure;
#endif

M
Michal Hocko 已提交
261 262 263 264 265 266 267
	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
268 269

extern struct mem_cgroup *root_mem_cgroup;
T
Tejun Heo 已提交
270

M
Michal Hocko 已提交
271 272 273 274 275 276 277
/**
 * mem_cgroup_events - count memory events against a cgroup
 * @memcg: the memory cgroup
 * @idx: the event index
 * @nr: the number of events to account for
 */
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
278
		       enum mem_cgroup_events_index idx,
M
Michal Hocko 已提交
279 280 281
		       unsigned int nr)
{
	this_cpu_add(memcg->stat->events[idx], nr);
282
	cgroup_file_notify(&memcg->events_file);
M
Michal Hocko 已提交
283
}
284 285 286

bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);

287
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
288 289
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
290
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
291 292 293
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
294
void mem_cgroup_uncharge(struct page *page);
295
void mem_cgroup_uncharge_list(struct list_head *page_list);
296

297
void mem_cgroup_replace_page(struct page *oldpage, struct page *newpage);
298

299 300
struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *);
301

302
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
303
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
304

M
Michal Hocko 已提交
305 306 307 308 309
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

310 311 312
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

M
Michal Hocko 已提交
313 314 315 316 317
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);

318 319 320 321 322 323 324 325 326 327 328 329 330 331
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

M
Michal Hocko 已提交
332 333 334 335 336 337 338 339 340
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
G
Glauber Costa 已提交
341

342 343
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
344
{
345
	struct mem_cgroup *task_memcg;
346
	bool match = false;
347

348
	rcu_read_lock();
349
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
350
	if (task_memcg)
351
		match = mem_cgroup_is_descendant(task_memcg, memcg);
352
	rcu_read_unlock();
353
	return match;
354
}
355

356
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
357
ino_t page_cgroup_ino(struct page *page);
358

M
Michal Hocko 已提交
359 360
static inline bool mem_cgroup_disabled(void)
{
361
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
M
Michal Hocko 已提交
362
}
363

364 365 366
/*
 * For memory reclaim.
 */
367
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
M
Michal Hocko 已提交
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
		int nr_pages);

static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
{
	struct mem_cgroup_per_zone *mz;
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return true;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	memcg = mz->memcg;

	return !!(memcg->css.flags & CSS_ONLINE);
}

static inline
unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
{
	struct mem_cgroup_per_zone *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
	return mz->lru_size[lru];
}

395
static inline bool mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
M
Michal Hocko 已提交
396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
{
	unsigned long inactive_ratio;
	unsigned long inactive;
	unsigned long active;
	unsigned long gb;

	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);

	gb = (inactive + active) >> (30 - PAGE_SHIFT);
	if (gb)
		inactive_ratio = int_sqrt(10 * gb);
	else
		inactive_ratio = 1;

	return inactive * inactive_ratio < active;
}

414 415
void mem_cgroup_handle_over_high(void);

416 417
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
				struct task_struct *p);
418

419
static inline void mem_cgroup_oom_enable(void)
420
{
T
Tejun Heo 已提交
421 422
	WARN_ON(current->memcg_may_oom);
	current->memcg_may_oom = 1;
423 424
}

425
static inline void mem_cgroup_oom_disable(void)
426
{
T
Tejun Heo 已提交
427 428
	WARN_ON(!current->memcg_may_oom);
	current->memcg_may_oom = 0;
429 430
}

431 432
static inline bool task_in_memcg_oom(struct task_struct *p)
{
T
Tejun Heo 已提交
433
	return p->memcg_in_oom;
434 435
}

436
bool mem_cgroup_oom_synchronize(bool wait);
437

A
Andrew Morton 已提交
438
#ifdef CONFIG_MEMCG_SWAP
439 440
extern int do_swap_account;
#endif
441

442 443
struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page);
void mem_cgroup_end_page_stat(struct mem_cgroup *memcg);
444

M
Michal Hocko 已提交
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461
/**
 * mem_cgroup_update_page_stat - update page state statistics
 * @memcg: memcg to account against
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
 * See mem_cgroup_begin_page_stat() for locking requirements.
 */
static inline void mem_cgroup_update_page_stat(struct mem_cgroup *memcg,
				 enum mem_cgroup_stat_index idx, int val)
{
	VM_BUG_ON(!rcu_read_lock_held());

	if (memcg)
		this_cpu_add(memcg->stat->count[idx], val);
}

462
static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
S
Sha Zhengju 已提交
463
					    enum mem_cgroup_stat_index idx)
464
{
465
	mem_cgroup_update_page_stat(memcg, idx, 1);
466 467
}

468
static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
S
Sha Zhengju 已提交
469
					    enum mem_cgroup_stat_index idx)
470
{
471
	mem_cgroup_update_page_stat(memcg, idx, -1);
472 473
}

474 475 476
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
						gfp_t gfp_mask,
						unsigned long *total_scanned);
D
David Rientjes 已提交
477

478 479 480
static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
					     enum vm_event_item idx)
{
M
Michal Hocko 已提交
481 482
	struct mem_cgroup *memcg;

483 484
	if (mem_cgroup_disabled())
		return;
M
Michal Hocko 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (unlikely(!memcg))
		goto out;

	switch (idx) {
	case PGFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
		break;
	case PGMAJFAULT:
		this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
		break;
	default:
		BUG();
	}
out:
	rcu_read_unlock();
503
}
504
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
505
void mem_cgroup_split_huge_fixup(struct page *head);
506 507
#endif

A
Andrew Morton 已提交
508
#else /* CONFIG_MEMCG */
509 510
struct mem_cgroup;

511 512 513 514 515 516 517 518 519 520 521 522
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
				     enum mem_cgroup_events_index idx,
				     unsigned int nr)
{
}

static inline bool mem_cgroup_low(struct mem_cgroup *root,
				  struct mem_cgroup *memcg)
{
	return false;
}

523 524
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
525 526
					struct mem_cgroup **memcgp,
					bool compound)
527
{
528
	*memcgp = NULL;
529 530 531
	return 0;
}

532 533
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
534
					    bool lrucare, bool compound)
535 536 537
{
}

538
static inline void mem_cgroup_cancel_charge(struct page *page,
539 540
					    struct mem_cgroup *memcg,
					    bool compound)
541 542 543
{
}

544
static inline void mem_cgroup_uncharge(struct page *page)
545 546 547
{
}

548
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
549 550 551
{
}

552
static inline void mem_cgroup_replace_page(struct page *old, struct page *new)
553 554 555
{
}

556 557
static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
						    struct mem_cgroup *memcg)
K
KAMEZAWA Hiroyuki 已提交
558
{
559
	return &zone->lruvec;
K
KAMEZAWA Hiroyuki 已提交
560 561
}

562 563
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
						    struct zone *zone)
564
{
565
	return &zone->lruvec;
566 567
}

568
static inline bool mm_match_cgroup(struct mm_struct *mm,
569
		struct mem_cgroup *memcg)
570
{
571
	return true;
572 573
}

574 575
static inline bool task_in_mem_cgroup(struct task_struct *task,
				      const struct mem_cgroup *memcg)
576
{
577
	return true;
578 579
}

580 581 582 583 584 585 586 587 588 589 590 591 592
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

593 594 595 596
static inline bool mem_cgroup_disabled(void)
{
	return true;
}
597

598
static inline bool
599
mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
600
{
601
	return true;
602 603
}

604 605 606 607 608
static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec)
{
	return true;
}

609
static inline unsigned long
610
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
611 612 613 614
{
	return 0;
}

615 616 617
static inline void
mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
			      int increment)
K
KOSAKI Motohiro 已提交
618 619 620
{
}

621 622 623 624 625
static inline void
mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
{
}

626
static inline struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page)
627
{
628
	return NULL;
629 630
}

631
static inline void mem_cgroup_end_page_stat(struct mem_cgroup *memcg)
632 633 634
{
}

635 636 637 638
static inline void mem_cgroup_handle_over_high(void)
{
}

639
static inline void mem_cgroup_oom_enable(void)
640 641 642
{
}

643
static inline void mem_cgroup_oom_disable(void)
644 645 646
{
}

647 648 649 650 651
static inline bool task_in_memcg_oom(struct task_struct *p)
{
	return false;
}

652
static inline bool mem_cgroup_oom_synchronize(bool wait)
653 654 655 656
{
	return false;
}

657
static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg,
S
Sha Zhengju 已提交
658
					    enum mem_cgroup_stat_index idx)
659 660 661
{
}

662
static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg,
S
Sha Zhengju 已提交
663
					    enum mem_cgroup_stat_index idx)
664 665 666
{
}

667
static inline
668 669 670
unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
671
{
672
	return 0;
673 674
}

675
static inline void mem_cgroup_split_huge_fixup(struct page *head)
676 677 678
{
}

679 680 681 682
static inline
void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
{
}
A
Andrew Morton 已提交
683
#endif /* CONFIG_MEMCG */
684

685
#ifdef CONFIG_CGROUP_WRITEBACK
T
Tejun Heo 已提交
686

687
struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
T
Tejun Heo 已提交
688
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
689 690 691
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
			 unsigned long *pheadroom, unsigned long *pdirty,
			 unsigned long *pwriteback);
T
Tejun Heo 已提交
692 693 694 695 696 697 698 699

#else	/* CONFIG_CGROUP_WRITEBACK */

static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
{
	return NULL;
}

700
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
701 702
				       unsigned long *pfilepages,
				       unsigned long *pheadroom,
703 704 705 706 707
				       unsigned long *pdirty,
				       unsigned long *pwriteback)
{
}

T
Tejun Heo 已提交
708
#endif	/* CONFIG_CGROUP_WRITEBACK */
709

G
Glauber Costa 已提交
710 711 712
struct sock;
void sock_update_memcg(struct sock *sk);
void sock_release_memcg(struct sock *sk);
713 714
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
715
#if defined(CONFIG_MEMCG) && defined(CONFIG_INET)
716 717
extern struct static_key_false memcg_sockets_enabled_key;
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
718
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
719
{
720
#ifdef CONFIG_MEMCG_KMEM
721 722
	if (memcg->tcp_mem.memory_pressure)
		return true;
723
#endif
724 725 726 727 728
	do {
		if (time_before(jiffies, memcg->socket_pressure))
			return true;
	} while ((memcg = parent_mem_cgroup(memcg)));
	return false;
729 730
}
#else
731
#define mem_cgroup_sockets_enabled 0
732
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
733 734 735 736
{
	return false;
}
#endif
737

738
#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
739
extern struct static_key_false memcg_kmem_enabled_key;
740

741
extern int memcg_nr_cache_ids;
742 743
void memcg_get_cache_ids(void);
void memcg_put_cache_ids(void);
744 745 746 747 748 749

/*
 * Helper macro to loop through all memcg-specific caches. Callers must still
 * check if the cache is valid (it is either valid or NULL).
 * the slab_mutex must be held when looping through those caches
 */
750
#define for_each_memcg_cache_index(_idx)	\
751
	for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
752

753 754
static inline bool memcg_kmem_enabled(void)
{
755
	return static_branch_unlikely(&memcg_kmem_enabled_key);
756 757
}

758
static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
M
Michal Hocko 已提交
759
{
760
	return memcg->kmem_state == KMEM_ONLINE;
M
Michal Hocko 已提交
761
}
762

763 764 765 766 767 768 769 770 771 772 773
/*
 * In general, we'll do everything in our power to not incur in any overhead
 * for non-memcg users for the kmem functions. Not even a function call, if we
 * can avoid it.
 *
 * Therefore, we'll inline all those functions so that in the best case, we'll
 * see that kmemcg is off for everybody and proceed quickly.  If it is on,
 * we'll still do most of the flag checking inline. We check a lot of
 * conditions, but because they are pretty simple, they are expected to be
 * fast.
 */
774 775
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
			      struct mem_cgroup *memcg);
776 777
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
void __memcg_kmem_uncharge(struct page *page, int order);
778

M
Michal Hocko 已提交
779 780 781 782 783 784 785 786 787
/*
 * helper for acessing a memcg's index. It will be used as an index in the
 * child cache array in kmem_cache, and also to derive its name. This function
 * will return -1 when this is not a kmem-limited memcg.
 */
static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return memcg ? memcg->kmemcg_id : -1;
}
788

V
Vladimir Davydov 已提交
789
struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp);
790
void __memcg_kmem_put_cache(struct kmem_cache *cachep);
791

V
Vladimir Davydov 已提交
792
static inline bool __memcg_kmem_bypass(void)
793 794 795 796 797
{
	if (!memcg_kmem_enabled())
		return true;
	if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
		return true;
798
	return false;
799 800 801
}

/**
802 803 804 805
 * memcg_kmem_charge: charge a kmem page
 * @page: page to charge
 * @gfp: reclaim mode
 * @order: allocation order
806
 *
807
 * Returns 0 on success, an error code on failure.
808
 */
809 810
static __always_inline int memcg_kmem_charge(struct page *page,
					     gfp_t gfp, int order)
811
{
V
Vladimir Davydov 已提交
812 813 814
	if (__memcg_kmem_bypass())
		return 0;
	if (!(gfp & __GFP_ACCOUNT))
815 816
		return 0;
	return __memcg_kmem_charge(page, gfp, order);
817 818 819
}

/**
820 821 822
 * memcg_kmem_uncharge: uncharge a kmem page
 * @page: page to uncharge
 * @order: allocation order
823
 */
824
static __always_inline void memcg_kmem_uncharge(struct page *page, int order)
825 826
{
	if (memcg_kmem_enabled())
827
		__memcg_kmem_uncharge(page, order);
828 829
}

830 831 832 833
/**
 * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation
 * @cachep: the original global kmem cache
 *
834
 * All memory allocated from a per-memcg cache is charged to the owner memcg.
835 836 837 838
 */
static __always_inline struct kmem_cache *
memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
{
V
Vladimir Davydov 已提交
839
	if (__memcg_kmem_bypass())
840
		return cachep;
V
Vladimir Davydov 已提交
841
	return __memcg_kmem_get_cache(cachep, gfp);
842
}
843 844 845 846 847 848

static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
{
	if (memcg_kmem_enabled())
		__memcg_kmem_put_cache(cachep);
}
849
#else
850 851 852
#define for_each_memcg_cache_index(_idx)	\
	for (; NULL; )

853 854 855 856 857
static inline bool memcg_kmem_enabled(void)
{
	return false;
}

858
static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
859 860 861 862
{
	return false;
}

863
static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
864
{
865
	return 0;
866 867
}

868
static inline void memcg_kmem_uncharge(struct page *page, int order)
869 870
{
}
871 872 873 874 875 876

static inline int memcg_cache_id(struct mem_cgroup *memcg)
{
	return -1;
}

877 878 879 880 881 882 883 884
static inline void memcg_get_cache_ids(void)
{
}

static inline void memcg_put_cache_ids(void)
{
}

885 886 887 888 889
static inline struct kmem_cache *
memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp)
{
	return cachep;
}
890 891 892 893

static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
{
}
894 895
#endif /* CONFIG_MEMCG && !CONFIG_SLOB */

B
Balbir Singh 已提交
896
#endif /* _LINUX_MEMCONTROL_H */