slab.c 106.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * linux/mm/slab.c
 * Written by Mark Hemment, 1996/97.
 * (markhe@nextd.demon.co.uk)
 *
 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
 *
 * Major cleanup, different bufctl logic, per-cpu arrays
 *	(c) 2000 Manfred Spraul
 *
 * Cleanup, make the head arrays unconditional, preparation for NUMA
 * 	(c) 2002 Manfred Spraul
 *
 * An implementation of the Slab Allocator as described in outline in;
 *	UNIX Internals: The New Frontiers by Uresh Vahalia
 *	Pub: Prentice Hall	ISBN 0-13-101908-2
 * or with a little more detail in;
 *	The Slab Allocator: An Object-Caching Kernel Memory Allocator
 *	Jeff Bonwick (Sun Microsystems).
 *	Presented at: USENIX Summer 1994 Technical Conference
 *
 * The memory is organized in caches, one cache for each object type.
 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
 * Each cache consists out of many slabs (they are small (usually one
 * page long) and always contiguous), and each slab contains multiple
 * initialized objects.
 *
 * This means, that your constructor is used only for newly allocated
S
Simon Arlott 已提交
29
 * slabs and you must pass objects with the same initializations to
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
 * kmem_cache_free.
 *
 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
 * normal). If you need a special memory type, then must create a new
 * cache for that memory type.
 *
 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
 *   full slabs with 0 free objects
 *   partial slabs
 *   empty slabs with no allocated objects
 *
 * If partial slabs exist, then new allocations come from these slabs,
 * otherwise from empty slabs or new slabs are allocated.
 *
 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
 *
 * Each cache has a short per-cpu head array, most allocs
 * and frees go into that array, and if that array overflows, then 1/2
 * of the entries in the array are given back into the global cache.
 * The head array is strictly LIFO and should improve the cache hit rates.
 * On SMP, it additionally reduces the spinlock operations.
 *
A
Andrew Morton 已提交
53
 * The c_cpuarray may not be read with enabled local interrupts -
L
Linus Torvalds 已提交
54 55 56 57
 * it's changed with a smp_call_function().
 *
 * SMP synchronization:
 *  constructors and destructors are called without any locking.
58
 *  Several members in struct kmem_cache and struct slab never change, they
L
Linus Torvalds 已提交
59 60 61 62 63 64 65 66 67 68 69 70
 *	are accessed without any locking.
 *  The per-cpu arrays are never accessed from the wrong cpu, no locking,
 *  	and local interrupts are disabled so slab code is preempt-safe.
 *  The non-constant members are protected with a per-cache irq spinlock.
 *
 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
 * in 2000 - many ideas in the current implementation are derived from
 * his patch.
 *
 * Further notes from the original documentation:
 *
 * 11 April '97.  Started multi-threading - markhe
71
 *	The global cache-chain is protected by the mutex 'slab_mutex'.
L
Linus Torvalds 已提交
72 73 74 75 76 77
 *	The sem is only needed when accessing/extending the cache-chain, which
 *	can never happen inside an interrupt (kmem_cache_create(),
 *	kmem_cache_shrink() and kmem_cache_reap()).
 *
 *	At present, each engine can be growing a cache.  This should be blocked.
 *
78 79 80 81 82 83 84 85 86
 * 15 March 2005. NUMA slab allocator.
 *	Shai Fultheim <shai@scalex86.org>.
 *	Shobhit Dayal <shobhit@calsoftinc.com>
 *	Alok N Kataria <alokk@calsoftinc.com>
 *	Christoph Lameter <christoph@lameter.com>
 *
 *	Modified the slab allocator to be node aware on NUMA systems.
 *	Each node has its own list of partial, free and full slabs.
 *	All object allocations for a node occur from node specific slab lists.
L
Linus Torvalds 已提交
87 88 89 90
 */

#include	<linux/slab.h>
#include	<linux/mm.h>
91
#include	<linux/poison.h>
L
Linus Torvalds 已提交
92 93 94 95 96
#include	<linux/swap.h>
#include	<linux/cache.h>
#include	<linux/interrupt.h>
#include	<linux/init.h>
#include	<linux/compiler.h>
97
#include	<linux/cpuset.h>
98
#include	<linux/proc_fs.h>
L
Linus Torvalds 已提交
99 100 101 102 103 104 105
#include	<linux/seq_file.h>
#include	<linux/notifier.h>
#include	<linux/kallsyms.h>
#include	<linux/cpu.h>
#include	<linux/sysctl.h>
#include	<linux/module.h>
#include	<linux/rcupdate.h>
106
#include	<linux/string.h>
107
#include	<linux/uaccess.h>
108
#include	<linux/nodemask.h>
109
#include	<linux/kmemleak.h>
110
#include	<linux/mempolicy.h>
I
Ingo Molnar 已提交
111
#include	<linux/mutex.h>
112
#include	<linux/fault-inject.h>
I
Ingo Molnar 已提交
113
#include	<linux/rtmutex.h>
114
#include	<linux/reciprocal_div.h>
115
#include	<linux/debugobjects.h>
P
Pekka Enberg 已提交
116
#include	<linux/kmemcheck.h>
117
#include	<linux/memory.h>
118
#include	<linux/prefetch.h>
L
Linus Torvalds 已提交
119

120 121
#include	<net/sock.h>

L
Linus Torvalds 已提交
122 123 124 125
#include	<asm/cacheflush.h>
#include	<asm/tlbflush.h>
#include	<asm/page.h>

126 127
#include <trace/events/kmem.h>

128 129
#include	"internal.h"

130 131
#include	"slab.h"

L
Linus Torvalds 已提交
132
/*
133
 * DEBUG	- 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
L
Linus Torvalds 已提交
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
 *		  0 for faster, smaller code (especially in the critical paths).
 *
 * STATS	- 1 to collect stats for /proc/slabinfo.
 *		  0 for faster, smaller code (especially in the critical paths).
 *
 * FORCED_DEBUG	- 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
 */

#ifdef CONFIG_DEBUG_SLAB
#define	DEBUG		1
#define	STATS		1
#define	FORCED_DEBUG	1
#else
#define	DEBUG		0
#define	STATS		0
#define	FORCED_DEBUG	0
#endif

/* Shouldn't this be in a header file somewhere? */
#define	BYTES_PER_WORD		sizeof(void *)
D
David Woodhouse 已提交
154
#define	REDZONE_ALIGN		max(BYTES_PER_WORD, __alignof__(unsigned long long))
L
Linus Torvalds 已提交
155 156 157 158 159

#ifndef ARCH_KMALLOC_FLAGS
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
#endif

160 161 162 163 164 165 166 167 168
#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
				<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)

#if FREELIST_BYTE_INDEX
typedef unsigned char freelist_idx_t;
#else
typedef unsigned short freelist_idx_t;
#endif

169
#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
170

171 172 173 174 175 176
/*
 * true if a page was allocated from pfmemalloc reserves for network-based
 * swap
 */
static bool pfmemalloc_active __read_mostly;

L
Linus Torvalds 已提交
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
/*
 * struct array_cache
 *
 * Purpose:
 * - LIFO ordering, to hand out cache-warm objects from _alloc
 * - reduce the number of linked list operations
 * - reduce spinlock operations
 *
 * The limit is stored in the per-cpu structure to reduce the data cache
 * footprint.
 *
 */
struct array_cache {
	unsigned int avail;
	unsigned int limit;
	unsigned int batchcount;
	unsigned int touched;
194
	void *entry[];	/*
A
Andrew Morton 已提交
195 196 197
			 * Must have this definition in here for the proper
			 * alignment of array_cache. Also simplifies accessing
			 * the entries.
198 199 200 201
			 *
			 * Entries should not be directly dereferenced as
			 * entries belonging to slabs marked pfmemalloc will
			 * have the lower bits set SLAB_OBJ_PFMEMALLOC
A
Andrew Morton 已提交
202
			 */
L
Linus Torvalds 已提交
203 204
};

J
Joonsoo Kim 已提交
205 206 207 208 209
struct alien_cache {
	spinlock_t lock;
	struct array_cache ac;
};

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
#define SLAB_OBJ_PFMEMALLOC	1
static inline bool is_obj_pfmemalloc(void *objp)
{
	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
}

static inline void set_obj_pfmemalloc(void **objp)
{
	*objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
	return;
}

static inline void clear_obj_pfmemalloc(void **objp)
{
	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
}

A
Andrew Morton 已提交
227 228 229
/*
 * bootstrap: The caches do not work without cpuarrays anymore, but the
 * cpuarrays are allocated from the generic caches...
L
Linus Torvalds 已提交
230 231 232 233
 */
#define BOOT_CPUCACHE_ENTRIES	1
struct arraycache_init {
	struct array_cache cache;
P
Pekka Enberg 已提交
234
	void *entries[BOOT_CPUCACHE_ENTRIES];
L
Linus Torvalds 已提交
235 236
};

237 238 239
/*
 * Need this for bootstrapping a per node allocator.
 */
240
#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
241
static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
242
#define	CACHE_CACHE 0
243
#define	SIZE_NODE (MAX_NUMNODES)
244

245
static int drain_freelist(struct kmem_cache *cache,
246
			struct kmem_cache_node *n, int tofree);
247
static void free_block(struct kmem_cache *cachep, void **objpp, int len,
248 249
			int node, struct list_head *list);
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
250
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
251
static void cache_reap(struct work_struct *unused);
252

253 254
static int slab_early_init = 1;

255
#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
L
Linus Torvalds 已提交
256

257
static void kmem_cache_node_init(struct kmem_cache_node *parent)
258 259 260 261 262 263
{
	INIT_LIST_HEAD(&parent->slabs_full);
	INIT_LIST_HEAD(&parent->slabs_partial);
	INIT_LIST_HEAD(&parent->slabs_free);
	parent->shared = NULL;
	parent->alien = NULL;
264
	parent->colour_next = 0;
265 266 267 268 269
	spin_lock_init(&parent->list_lock);
	parent->free_objects = 0;
	parent->free_touched = 0;
}

A
Andrew Morton 已提交
270 271 272
#define MAKE_LIST(cachep, listp, slab, nodeid)				\
	do {								\
		INIT_LIST_HEAD(listp);					\
273
		list_splice(&get_node(cachep, nodeid)->slab, listp);	\
274 275
	} while (0)

A
Andrew Morton 已提交
276 277
#define	MAKE_ALL_LISTS(cachep, ptr, nodeid)				\
	do {								\
278 279 280 281
	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid);	\
	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);	\
	} while (0)
L
Linus Torvalds 已提交
282 283 284

#define CFLGS_OFF_SLAB		(0x80000000UL)
#define	OFF_SLAB(x)	((x)->flags & CFLGS_OFF_SLAB)
285
#define OFF_SLAB_MIN_SIZE (max_t(size_t, PAGE_SIZE >> 5, KMALLOC_MIN_SIZE + 1))
L
Linus Torvalds 已提交
286 287

#define BATCHREFILL_LIMIT	16
A
Andrew Morton 已提交
288 289 290
/*
 * Optimization question: fewer reaps means less probability for unnessary
 * cpucache drain/refill cycles.
L
Linus Torvalds 已提交
291
 *
A
Adrian Bunk 已提交
292
 * OTOH the cpuarrays can contain lots of objects,
L
Linus Torvalds 已提交
293 294
 * which could lock up otherwise freeable slabs.
 */
295 296
#define REAPTIMEOUT_AC		(2*HZ)
#define REAPTIMEOUT_NODE	(4*HZ)
L
Linus Torvalds 已提交
297 298 299 300 301 302

#if STATS
#define	STATS_INC_ACTIVE(x)	((x)->num_active++)
#define	STATS_DEC_ACTIVE(x)	((x)->num_active--)
#define	STATS_INC_ALLOCED(x)	((x)->num_allocations++)
#define	STATS_INC_GROWN(x)	((x)->grown++)
303
#define	STATS_ADD_REAPED(x,y)	((x)->reaped += (y))
A
Andrew Morton 已提交
304 305 306 307 308
#define	STATS_SET_HIGH(x)						\
	do {								\
		if ((x)->num_active > (x)->high_mark)			\
			(x)->high_mark = (x)->num_active;		\
	} while (0)
L
Linus Torvalds 已提交
309 310
#define	STATS_INC_ERR(x)	((x)->errors++)
#define	STATS_INC_NODEALLOCS(x)	((x)->node_allocs++)
311
#define	STATS_INC_NODEFREES(x)	((x)->node_frees++)
312
#define STATS_INC_ACOVERFLOW(x)   ((x)->node_overflow++)
A
Andrew Morton 已提交
313 314 315 316 317
#define	STATS_SET_FREEABLE(x, i)					\
	do {								\
		if ((x)->max_freeable < i)				\
			(x)->max_freeable = i;				\
	} while (0)
L
Linus Torvalds 已提交
318 319 320 321 322 323 324 325 326
#define STATS_INC_ALLOCHIT(x)	atomic_inc(&(x)->allochit)
#define STATS_INC_ALLOCMISS(x)	atomic_inc(&(x)->allocmiss)
#define STATS_INC_FREEHIT(x)	atomic_inc(&(x)->freehit)
#define STATS_INC_FREEMISS(x)	atomic_inc(&(x)->freemiss)
#else
#define	STATS_INC_ACTIVE(x)	do { } while (0)
#define	STATS_DEC_ACTIVE(x)	do { } while (0)
#define	STATS_INC_ALLOCED(x)	do { } while (0)
#define	STATS_INC_GROWN(x)	do { } while (0)
327
#define	STATS_ADD_REAPED(x,y)	do { (void)(y); } while (0)
L
Linus Torvalds 已提交
328 329 330
#define	STATS_SET_HIGH(x)	do { } while (0)
#define	STATS_INC_ERR(x)	do { } while (0)
#define	STATS_INC_NODEALLOCS(x)	do { } while (0)
331
#define	STATS_INC_NODEFREES(x)	do { } while (0)
332
#define STATS_INC_ACOVERFLOW(x)   do { } while (0)
A
Andrew Morton 已提交
333
#define	STATS_SET_FREEABLE(x, i) do { } while (0)
L
Linus Torvalds 已提交
334 335 336 337 338 339 340 341
#define STATS_INC_ALLOCHIT(x)	do { } while (0)
#define STATS_INC_ALLOCMISS(x)	do { } while (0)
#define STATS_INC_FREEHIT(x)	do { } while (0)
#define STATS_INC_FREEMISS(x)	do { } while (0)
#endif

#if DEBUG

A
Andrew Morton 已提交
342 343
/*
 * memory layout of objects:
L
Linus Torvalds 已提交
344
 * 0		: objp
345
 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
L
Linus Torvalds 已提交
346 347
 * 		the end of an object is aligned with the end of the real
 * 		allocation. Catches writes behind the end of the allocation.
348
 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
L
Linus Torvalds 已提交
349
 * 		redzone word.
350
 * cachep->obj_offset: The real object.
351 352
 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
 * cachep->size - 1* BYTES_PER_WORD: last caller address
A
Andrew Morton 已提交
353
 *					[BYTES_PER_WORD long]
L
Linus Torvalds 已提交
354
 */
355
static int obj_offset(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
356
{
357
	return cachep->obj_offset;
L
Linus Torvalds 已提交
358 359
}

360
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
361 362
{
	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
363 364
	return (unsigned long long*) (objp + obj_offset(cachep) -
				      sizeof(unsigned long long));
L
Linus Torvalds 已提交
365 366
}

367
static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
368 369 370
{
	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
	if (cachep->flags & SLAB_STORE_USER)
371
		return (unsigned long long *)(objp + cachep->size -
372
					      sizeof(unsigned long long) -
D
David Woodhouse 已提交
373
					      REDZONE_ALIGN);
374
	return (unsigned long long *) (objp + cachep->size -
375
				       sizeof(unsigned long long));
L
Linus Torvalds 已提交
376 377
}

378
static void **dbg_userword(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
379 380
{
	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
381
	return (void **)(objp + cachep->size - BYTES_PER_WORD);
L
Linus Torvalds 已提交
382 383 384 385
}

#else

386
#define obj_offset(x)			0
387 388
#define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
#define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
L
Linus Torvalds 已提交
389 390 391 392
#define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})

#endif

393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
#define OBJECT_FREE (0)
#define OBJECT_ACTIVE (1)

#ifdef CONFIG_DEBUG_SLAB_LEAK

static void set_obj_status(struct page *page, int idx, int val)
{
	int freelist_size;
	char *status;
	struct kmem_cache *cachep = page->slab_cache;

	freelist_size = cachep->num * sizeof(freelist_idx_t);
	status = (char *)page->freelist + freelist_size;
	status[idx] = val;
}

static inline unsigned int get_obj_status(struct page *page, int idx)
{
	int freelist_size;
	char *status;
	struct kmem_cache *cachep = page->slab_cache;

	freelist_size = cachep->num * sizeof(freelist_idx_t);
	status = (char *)page->freelist + freelist_size;

	return status[idx];
}

#else
static inline void set_obj_status(struct page *page, int idx, int val) {}

#endif

L
Linus Torvalds 已提交
426
/*
427 428
 * Do not go above this order unless 0 objects fit into the slab or
 * overridden on the command line.
L
Linus Torvalds 已提交
429
 */
430 431 432
#define	SLAB_MAX_ORDER_HI	1
#define	SLAB_MAX_ORDER_LO	0
static int slab_max_order = SLAB_MAX_ORDER_LO;
433
static bool slab_max_order_set __initdata;
L
Linus Torvalds 已提交
434

435 436
static inline struct kmem_cache *virt_to_cache(const void *obj)
{
437
	struct page *page = virt_to_head_page(obj);
C
Christoph Lameter 已提交
438
	return page->slab_cache;
439 440
}

441
static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
442 443
				 unsigned int idx)
{
444
	return page->s_mem + cache->size * idx;
445 446
}

447
/*
448 449 450
 * We want to avoid an expensive divide : (offset / cache->size)
 *   Using the fact that size is a constant for a particular cache,
 *   we can replace (offset / cache->size) by
451 452 453
 *   reciprocal_divide(offset, cache->reciprocal_buffer_size)
 */
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
454
					const struct page *page, void *obj)
455
{
456
	u32 offset = (obj - page->s_mem);
457
	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
458 459
}

L
Linus Torvalds 已提交
460
/* internal cache of cache description objs */
461
static struct kmem_cache kmem_cache_boot = {
P
Pekka Enberg 已提交
462 463 464
	.batchcount = 1,
	.limit = BOOT_CPUCACHE_ENTRIES,
	.shared = 1,
465
	.size = sizeof(struct kmem_cache),
P
Pekka Enberg 已提交
466
	.name = "kmem_cache",
L
Linus Torvalds 已提交
467 468
};

469 470
#define BAD_ALIEN_MAGIC 0x01020304ul

471
static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
L
Linus Torvalds 已提交
472

473
static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
474
{
475
	return this_cpu_ptr(cachep->cpu_cache);
L
Linus Torvalds 已提交
476 477
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491
static size_t calculate_freelist_size(int nr_objs, size_t align)
{
	size_t freelist_size;

	freelist_size = nr_objs * sizeof(freelist_idx_t);
	if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
		freelist_size += nr_objs * sizeof(char);

	if (align)
		freelist_size = ALIGN(freelist_size, align);

	return freelist_size;
}

492 493
static int calculate_nr_objs(size_t slab_size, size_t buffer_size,
				size_t idx_size, size_t align)
L
Linus Torvalds 已提交
494
{
495
	int nr_objs;
496
	size_t remained_size;
497
	size_t freelist_size;
498
	int extra_space = 0;
499

500 501
	if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
		extra_space = sizeof(char);
502 503 504 505 506 507 508 509
	/*
	 * Ignore padding for the initial guess. The padding
	 * is at most @align-1 bytes, and @buffer_size is at
	 * least @align. In the worst case, this result will
	 * be one greater than the number of objects that fit
	 * into the memory allocation when taking the padding
	 * into account.
	 */
510
	nr_objs = slab_size / (buffer_size + idx_size + extra_space);
511 512 513 514 515

	/*
	 * This calculated number will be either the right
	 * amount, or one greater than what we want.
	 */
516 517 518
	remained_size = slab_size - nr_objs * buffer_size;
	freelist_size = calculate_freelist_size(nr_objs, align);
	if (remained_size < freelist_size)
519 520 521
		nr_objs--;

	return nr_objs;
522
}
L
Linus Torvalds 已提交
523

A
Andrew Morton 已提交
524 525 526
/*
 * Calculate the number of objects and left-over bytes for a given buffer size.
 */
527 528 529 530 531 532 533
static void cache_estimate(unsigned long gfporder, size_t buffer_size,
			   size_t align, int flags, size_t *left_over,
			   unsigned int *num)
{
	int nr_objs;
	size_t mgmt_size;
	size_t slab_size = PAGE_SIZE << gfporder;
L
Linus Torvalds 已提交
534

535 536 537 538 539
	/*
	 * The slab management structure can be either off the slab or
	 * on it. For the latter case, the memory allocated for a
	 * slab is used for:
	 *
J
Joonsoo Kim 已提交
540
	 * - One unsigned int for each object
541 542 543 544 545 546 547 548 549 550 551 552 553
	 * - Padding to respect alignment of @align
	 * - @buffer_size bytes for each object
	 *
	 * If the slab management structure is off the slab, then the
	 * alignment will already be calculated into the size. Because
	 * the slabs are all pages aligned, the objects will be at the
	 * correct alignment when allocated.
	 */
	if (flags & CFLGS_OFF_SLAB) {
		mgmt_size = 0;
		nr_objs = slab_size / buffer_size;

	} else {
554
		nr_objs = calculate_nr_objs(slab_size, buffer_size,
555
					sizeof(freelist_idx_t), align);
556
		mgmt_size = calculate_freelist_size(nr_objs, align);
557 558 559
	}
	*num = nr_objs;
	*left_over = slab_size - nr_objs*buffer_size - mgmt_size;
L
Linus Torvalds 已提交
560 561
}

562
#if DEBUG
563
#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
L
Linus Torvalds 已提交
564

A
Andrew Morton 已提交
565 566
static void __slab_error(const char *function, struct kmem_cache *cachep,
			char *msg)
L
Linus Torvalds 已提交
567 568
{
	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
P
Pekka Enberg 已提交
569
	       function, cachep->name, msg);
L
Linus Torvalds 已提交
570
	dump_stack();
571
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
572
}
573
#endif
L
Linus Torvalds 已提交
574

575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
/*
 * By default on NUMA we use alien caches to stage the freeing of
 * objects allocated from other nodes. This causes massive memory
 * inefficiencies when using fake NUMA setup to split memory into a
 * large number of small nodes, so it can be disabled on the command
 * line
  */

static int use_alien_caches __read_mostly = 1;
static int __init noaliencache_setup(char *s)
{
	use_alien_caches = 0;
	return 1;
}
__setup("noaliencache", noaliencache_setup);

591 592 593 594 595 596 597 598 599 600 601
static int __init slab_max_order_setup(char *str)
{
	get_option(&str, &slab_max_order);
	slab_max_order = slab_max_order < 0 ? 0 :
				min(slab_max_order, MAX_ORDER - 1);
	slab_max_order_set = true;

	return 1;
}
__setup("slab_max_order=", slab_max_order_setup);

602 603 604 605 606 607 608
#ifdef CONFIG_NUMA
/*
 * Special reaping functions for NUMA systems called from cache_reap().
 * These take care of doing round robin flushing of alien caches (containing
 * objects freed on different nodes from which they were allocated) and the
 * flushing of remote pcps by calling drain_node_pages.
 */
609
static DEFINE_PER_CPU(unsigned long, slab_reap_node);
610 611 612 613 614

static void init_reap_node(int cpu)
{
	int node;

615
	node = next_node(cpu_to_mem(cpu), node_online_map);
616
	if (node == MAX_NUMNODES)
617
		node = first_node(node_online_map);
618

619
	per_cpu(slab_reap_node, cpu) = node;
620 621 622 623
}

static void next_reap_node(void)
{
624
	int node = __this_cpu_read(slab_reap_node);
625 626 627 628

	node = next_node(node, node_online_map);
	if (unlikely(node >= MAX_NUMNODES))
		node = first_node(node_online_map);
629
	__this_cpu_write(slab_reap_node, node);
630 631 632 633 634 635 636
}

#else
#define init_reap_node(cpu) do { } while (0)
#define next_reap_node(void) do { } while (0)
#endif

L
Linus Torvalds 已提交
637 638 639 640 641 642 643
/*
 * Initiate the reap timer running on the target CPU.  We run at around 1 to 2Hz
 * via the workqueue/eventd.
 * Add the CPU number into the expiration time to minimize the possibility of
 * the CPUs getting into lockstep and contending for the global cache chain
 * lock.
 */
644
static void start_cpu_timer(int cpu)
L
Linus Torvalds 已提交
645
{
646
	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
L
Linus Torvalds 已提交
647 648 649 650 651 652

	/*
	 * When this gets called from do_initcalls via cpucache_init(),
	 * init_workqueues() has already run, so keventd will be setup
	 * at that time.
	 */
653
	if (keventd_up() && reap_work->work.func == NULL) {
654
		init_reap_node(cpu);
655
		INIT_DEFERRABLE_WORK(reap_work, cache_reap);
656 657
		schedule_delayed_work_on(cpu, reap_work,
					__round_jiffies_relative(HZ, cpu));
L
Linus Torvalds 已提交
658 659 660
	}
}

661
static void init_arraycache(struct array_cache *ac, int limit, int batch)
L
Linus Torvalds 已提交
662
{
663 664
	/*
	 * The array_cache structures contain pointers to free object.
L
Lucas De Marchi 已提交
665
	 * However, when such objects are allocated or transferred to another
666 667 668 669
	 * cache the pointers are not cleared and they could be counted as
	 * valid references during a kmemleak scan. Therefore, kmemleak must
	 * not scan such objects.
	 */
670 671 672 673 674 675
	kmemleak_no_scan(ac);
	if (ac) {
		ac->avail = 0;
		ac->limit = limit;
		ac->batchcount = batch;
		ac->touched = 0;
L
Linus Torvalds 已提交
676
	}
677 678 679 680 681
}

static struct array_cache *alloc_arraycache(int node, int entries,
					    int batchcount, gfp_t gfp)
{
682
	size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
683 684 685 686 687
	struct array_cache *ac = NULL;

	ac = kmalloc_node(memsize, gfp, node);
	init_arraycache(ac, entries, batchcount);
	return ac;
L
Linus Torvalds 已提交
688 689
}

690
static inline bool is_slab_pfmemalloc(struct page *page)
691 692 693 694 695 696 697 698
{
	return PageSlabPfmemalloc(page);
}

/* Clears pfmemalloc_active if no slabs have pfmalloc set */
static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
						struct array_cache *ac)
{
699
	struct kmem_cache_node *n = get_node(cachep, numa_mem_id());
700
	struct page *page;
701 702 703 704 705
	unsigned long flags;

	if (!pfmemalloc_active)
		return;

706
	spin_lock_irqsave(&n->list_lock, flags);
707 708
	list_for_each_entry(page, &n->slabs_full, lru)
		if (is_slab_pfmemalloc(page))
709 710
			goto out;

711 712
	list_for_each_entry(page, &n->slabs_partial, lru)
		if (is_slab_pfmemalloc(page))
713 714
			goto out;

715 716
	list_for_each_entry(page, &n->slabs_free, lru)
		if (is_slab_pfmemalloc(page))
717 718 719 720
			goto out;

	pfmemalloc_active = false;
out:
721
	spin_unlock_irqrestore(&n->list_lock, flags);
722 723
}

724
static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
725 726 727 728 729 730 731
						gfp_t flags, bool force_refill)
{
	int i;
	void *objp = ac->entry[--ac->avail];

	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
	if (unlikely(is_obj_pfmemalloc(objp))) {
732
		struct kmem_cache_node *n;
733 734 735 736 737 738 739

		if (gfp_pfmemalloc_allowed(flags)) {
			clear_obj_pfmemalloc(&objp);
			return objp;
		}

		/* The caller cannot use PFMEMALLOC objects, find another one */
740
		for (i = 0; i < ac->avail; i++) {
741 742 743 744 745 746 747 748 749 750 751 752 753
			/* If a !PFMEMALLOC object is found, swap them */
			if (!is_obj_pfmemalloc(ac->entry[i])) {
				objp = ac->entry[i];
				ac->entry[i] = ac->entry[ac->avail];
				ac->entry[ac->avail] = objp;
				return objp;
			}
		}

		/*
		 * If there are empty slabs on the slabs_free list and we are
		 * being forced to refill the cache, mark this one !pfmemalloc.
		 */
754
		n = get_node(cachep, numa_mem_id());
755
		if (!list_empty(&n->slabs_free) && force_refill) {
756
			struct page *page = virt_to_head_page(objp);
757
			ClearPageSlabPfmemalloc(page);
758 759 760 761 762 763 764 765 766 767 768 769 770
			clear_obj_pfmemalloc(&objp);
			recheck_pfmemalloc_active(cachep, ac);
			return objp;
		}

		/* No !PFMEMALLOC objects available */
		ac->avail++;
		objp = NULL;
	}

	return objp;
}

771 772 773 774 775 776 777 778 779 780 781 782 783
static inline void *ac_get_obj(struct kmem_cache *cachep,
			struct array_cache *ac, gfp_t flags, bool force_refill)
{
	void *objp;

	if (unlikely(sk_memalloc_socks()))
		objp = __ac_get_obj(cachep, ac, flags, force_refill);
	else
		objp = ac->entry[--ac->avail];

	return objp;
}

J
Joonsoo Kim 已提交
784 785
static noinline void *__ac_put_obj(struct kmem_cache *cachep,
			struct array_cache *ac, void *objp)
786 787 788
{
	if (unlikely(pfmemalloc_active)) {
		/* Some pfmemalloc slabs exist, check if this is one */
789
		struct page *page = virt_to_head_page(objp);
790 791 792 793
		if (PageSlabPfmemalloc(page))
			set_obj_pfmemalloc(&objp);
	}

794 795 796 797 798 799 800 801 802
	return objp;
}

static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
								void *objp)
{
	if (unlikely(sk_memalloc_socks()))
		objp = __ac_put_obj(cachep, ac, objp);

803 804 805
	ac->entry[ac->avail++] = objp;
}

806 807 808 809 810 811 812 813 814 815
/*
 * Transfer objects in one arraycache to another.
 * Locking must be handled by the caller.
 *
 * Return the number of entries transferred.
 */
static int transfer_objects(struct array_cache *to,
		struct array_cache *from, unsigned int max)
{
	/* Figure out how many entries to transfer */
816
	int nr = min3(from->avail, max, to->limit - to->avail);
817 818 819 820 821 822 823 824 825 826 827 828

	if (!nr)
		return 0;

	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
			sizeof(void *) *nr);

	from->avail -= nr;
	to->avail += nr;
	return nr;
}

829 830 831
#ifndef CONFIG_NUMA

#define drain_alien_cache(cachep, alien) do { } while (0)
832
#define reap_alien(cachep, n) do { } while (0)
833

J
Joonsoo Kim 已提交
834 835
static inline struct alien_cache **alloc_alien_cache(int node,
						int limit, gfp_t gfp)
836
{
837
	return (struct alien_cache **)BAD_ALIEN_MAGIC;
838 839
}

J
Joonsoo Kim 已提交
840
static inline void free_alien_cache(struct alien_cache **ac_ptr)
841 842 843 844 845 846 847 848 849 850 851 852 853 854
{
}

static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
{
	return 0;
}

static inline void *alternate_node_alloc(struct kmem_cache *cachep,
		gfp_t flags)
{
	return NULL;
}

855
static inline void *____cache_alloc_node(struct kmem_cache *cachep,
856 857 858 859 860
		 gfp_t flags, int nodeid)
{
	return NULL;
}

D
David Rientjes 已提交
861 862 863 864 865
static inline gfp_t gfp_exact_node(gfp_t flags)
{
	return flags;
}

866 867
#else	/* CONFIG_NUMA */

868
static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
869
static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
870

J
Joonsoo Kim 已提交
871 872 873
static struct alien_cache *__alloc_alien_cache(int node, int entries,
						int batch, gfp_t gfp)
{
874
	size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
J
Joonsoo Kim 已提交
875 876 877 878
	struct alien_cache *alc = NULL;

	alc = kmalloc_node(memsize, gfp, node);
	init_arraycache(&alc->ac, entries, batch);
879
	spin_lock_init(&alc->lock);
J
Joonsoo Kim 已提交
880 881 882 883
	return alc;
}

static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
884
{
J
Joonsoo Kim 已提交
885
	struct alien_cache **alc_ptr;
886
	size_t memsize = sizeof(void *) * nr_node_ids;
887 888 889 890
	int i;

	if (limit > 1)
		limit = 12;
J
Joonsoo Kim 已提交
891 892 893 894 895 896 897 898 899 900 901 902 903
	alc_ptr = kzalloc_node(memsize, gfp, node);
	if (!alc_ptr)
		return NULL;

	for_each_node(i) {
		if (i == node || !node_online(i))
			continue;
		alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
		if (!alc_ptr[i]) {
			for (i--; i >= 0; i--)
				kfree(alc_ptr[i]);
			kfree(alc_ptr);
			return NULL;
904 905
		}
	}
J
Joonsoo Kim 已提交
906
	return alc_ptr;
907 908
}

J
Joonsoo Kim 已提交
909
static void free_alien_cache(struct alien_cache **alc_ptr)
910 911 912
{
	int i;

J
Joonsoo Kim 已提交
913
	if (!alc_ptr)
914 915
		return;
	for_each_node(i)
J
Joonsoo Kim 已提交
916 917
	    kfree(alc_ptr[i]);
	kfree(alc_ptr);
918 919
}

920
static void __drain_alien_cache(struct kmem_cache *cachep,
921 922
				struct array_cache *ac, int node,
				struct list_head *list)
923
{
924
	struct kmem_cache_node *n = get_node(cachep, node);
925 926

	if (ac->avail) {
927
		spin_lock(&n->list_lock);
928 929 930 931 932
		/*
		 * Stuff objects into the remote nodes shared array first.
		 * That way we could avoid the overhead of putting the objects
		 * into the free lists and getting them back later.
		 */
933 934
		if (n->shared)
			transfer_objects(n->shared, ac, ac->limit);
935

936
		free_block(cachep, ac->entry, ac->avail, node, list);
937
		ac->avail = 0;
938
		spin_unlock(&n->list_lock);
939 940 941
	}
}

942 943 944
/*
 * Called from cache_reap() to regularly drain alien caches round robin.
 */
945
static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
946
{
947
	int node = __this_cpu_read(slab_reap_node);
948

949
	if (n->alien) {
J
Joonsoo Kim 已提交
950 951 952 953 954
		struct alien_cache *alc = n->alien[node];
		struct array_cache *ac;

		if (alc) {
			ac = &alc->ac;
955
			if (ac->avail && spin_trylock_irq(&alc->lock)) {
956 957 958
				LIST_HEAD(list);

				__drain_alien_cache(cachep, ac, node, &list);
959
				spin_unlock_irq(&alc->lock);
960
				slabs_destroy(cachep, &list);
J
Joonsoo Kim 已提交
961
			}
962 963 964 965
		}
	}
}

A
Andrew Morton 已提交
966
static void drain_alien_cache(struct kmem_cache *cachep,
J
Joonsoo Kim 已提交
967
				struct alien_cache **alien)
968
{
P
Pekka Enberg 已提交
969
	int i = 0;
J
Joonsoo Kim 已提交
970
	struct alien_cache *alc;
971 972 973 974
	struct array_cache *ac;
	unsigned long flags;

	for_each_online_node(i) {
J
Joonsoo Kim 已提交
975 976
		alc = alien[i];
		if (alc) {
977 978
			LIST_HEAD(list);

J
Joonsoo Kim 已提交
979
			ac = &alc->ac;
980
			spin_lock_irqsave(&alc->lock, flags);
981
			__drain_alien_cache(cachep, ac, i, &list);
982
			spin_unlock_irqrestore(&alc->lock, flags);
983
			slabs_destroy(cachep, &list);
984 985 986
		}
	}
}
987

988 989
static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
				int node, int page_node)
990
{
991
	struct kmem_cache_node *n;
J
Joonsoo Kim 已提交
992 993
	struct alien_cache *alien = NULL;
	struct array_cache *ac;
994
	LIST_HEAD(list);
P
Pekka Enberg 已提交
995

996
	n = get_node(cachep, node);
997
	STATS_INC_NODEFREES(cachep);
998 999
	if (n->alien && n->alien[page_node]) {
		alien = n->alien[page_node];
J
Joonsoo Kim 已提交
1000
		ac = &alien->ac;
1001
		spin_lock(&alien->lock);
J
Joonsoo Kim 已提交
1002
		if (unlikely(ac->avail == ac->limit)) {
1003
			STATS_INC_ACOVERFLOW(cachep);
1004
			__drain_alien_cache(cachep, ac, page_node, &list);
1005
		}
J
Joonsoo Kim 已提交
1006
		ac_put_obj(cachep, ac, objp);
1007
		spin_unlock(&alien->lock);
1008
		slabs_destroy(cachep, &list);
1009
	} else {
1010
		n = get_node(cachep, page_node);
1011
		spin_lock(&n->list_lock);
1012
		free_block(cachep, &objp, 1, page_node, &list);
1013
		spin_unlock(&n->list_lock);
1014
		slabs_destroy(cachep, &list);
1015 1016 1017
	}
	return 1;
}
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031

static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
{
	int page_node = page_to_nid(virt_to_page(objp));
	int node = numa_mem_id();
	/*
	 * Make sure we are not freeing a object from another node to the array
	 * cache on this cpu.
	 */
	if (likely(node == page_node))
		return 0;

	return __cache_free_alien(cachep, objp, node, page_node);
}
D
David Rientjes 已提交
1032 1033

/*
1034 1035
 * Construct gfp mask to allocate from a specific node but do not direct reclaim
 * or warn about failures. kswapd may still wake to reclaim in the background.
D
David Rientjes 已提交
1036 1037 1038
 */
static inline gfp_t gfp_exact_node(gfp_t flags)
{
1039
	return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~__GFP_DIRECT_RECLAIM;
D
David Rientjes 已提交
1040
}
1041 1042
#endif

1043
/*
1044
 * Allocates and initializes node for a node on each slab cache, used for
1045
 * either memory or cpu hotplug.  If memory is being hot-added, the kmem_cache_node
1046
 * will be allocated off-node since memory is not yet online for the new node.
1047
 * When hotplugging memory or a cpu, existing node are not replaced if
1048 1049
 * already in use.
 *
1050
 * Must hold slab_mutex.
1051
 */
1052
static int init_cache_node_node(int node)
1053 1054
{
	struct kmem_cache *cachep;
1055
	struct kmem_cache_node *n;
1056
	const size_t memsize = sizeof(struct kmem_cache_node);
1057

1058
	list_for_each_entry(cachep, &slab_caches, list) {
1059
		/*
1060
		 * Set up the kmem_cache_node for cpu before we can
1061 1062 1063
		 * begin anything. Make sure some other cpu on this
		 * node has not already allocated this
		 */
1064 1065
		n = get_node(cachep, node);
		if (!n) {
1066 1067
			n = kmalloc_node(memsize, GFP_KERNEL, node);
			if (!n)
1068
				return -ENOMEM;
1069
			kmem_cache_node_init(n);
1070 1071
			n->next_reap = jiffies + REAPTIMEOUT_NODE +
			    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1072 1073

			/*
1074 1075
			 * The kmem_cache_nodes don't come and go as CPUs
			 * come and go.  slab_mutex is sufficient
1076 1077
			 * protection here.
			 */
1078
			cachep->node[node] = n;
1079 1080
		}

1081 1082
		spin_lock_irq(&n->list_lock);
		n->free_limit =
1083 1084
			(1 + nr_cpus_node(node)) *
			cachep->batchcount + cachep->num;
1085
		spin_unlock_irq(&n->list_lock);
1086 1087 1088 1089
	}
	return 0;
}

1090 1091 1092 1093 1094 1095
static inline int slabs_tofree(struct kmem_cache *cachep,
						struct kmem_cache_node *n)
{
	return (n->free_objects + cachep->num - 1) / cachep->num;
}

1096
static void cpuup_canceled(long cpu)
1097 1098
{
	struct kmem_cache *cachep;
1099
	struct kmem_cache_node *n = NULL;
1100
	int node = cpu_to_mem(cpu);
1101
	const struct cpumask *mask = cpumask_of_node(node);
1102

1103
	list_for_each_entry(cachep, &slab_caches, list) {
1104 1105
		struct array_cache *nc;
		struct array_cache *shared;
J
Joonsoo Kim 已提交
1106
		struct alien_cache **alien;
1107
		LIST_HEAD(list);
1108

1109
		n = get_node(cachep, node);
1110
		if (!n)
1111
			continue;
1112

1113
		spin_lock_irq(&n->list_lock);
1114

1115 1116
		/* Free limit for this kmem_cache_node */
		n->free_limit -= cachep->batchcount;
1117 1118 1119 1120

		/* cpu is dead; no one can alloc from it. */
		nc = per_cpu_ptr(cachep->cpu_cache, cpu);
		if (nc) {
1121
			free_block(cachep, nc->entry, nc->avail, node, &list);
1122 1123
			nc->avail = 0;
		}
1124

1125
		if (!cpumask_empty(mask)) {
1126
			spin_unlock_irq(&n->list_lock);
1127
			goto free_slab;
1128 1129
		}

1130
		shared = n->shared;
1131 1132
		if (shared) {
			free_block(cachep, shared->entry,
1133
				   shared->avail, node, &list);
1134
			n->shared = NULL;
1135 1136
		}

1137 1138
		alien = n->alien;
		n->alien = NULL;
1139

1140
		spin_unlock_irq(&n->list_lock);
1141 1142 1143 1144 1145 1146

		kfree(shared);
		if (alien) {
			drain_alien_cache(cachep, alien);
			free_alien_cache(alien);
		}
1147 1148

free_slab:
1149
		slabs_destroy(cachep, &list);
1150 1151 1152 1153 1154 1155
	}
	/*
	 * In the previous loop, all the objects were freed to
	 * the respective cache's slabs,  now we can go ahead and
	 * shrink each nodelist to its limit.
	 */
1156
	list_for_each_entry(cachep, &slab_caches, list) {
1157
		n = get_node(cachep, node);
1158
		if (!n)
1159
			continue;
1160
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1161 1162 1163
	}
}

1164
static int cpuup_prepare(long cpu)
L
Linus Torvalds 已提交
1165
{
1166
	struct kmem_cache *cachep;
1167
	struct kmem_cache_node *n = NULL;
1168
	int node = cpu_to_mem(cpu);
1169
	int err;
L
Linus Torvalds 已提交
1170

1171 1172 1173 1174
	/*
	 * We need to do this right in the beginning since
	 * alloc_arraycache's are going to use this list.
	 * kmalloc_node allows us to add the slab to the right
1175
	 * kmem_cache_node and not this cpu's kmem_cache_node
1176
	 */
1177
	err = init_cache_node_node(node);
1178 1179
	if (err < 0)
		goto bad;
1180 1181 1182 1183 1184

	/*
	 * Now we can go ahead with allocating the shared arrays and
	 * array caches
	 */
1185
	list_for_each_entry(cachep, &slab_caches, list) {
1186
		struct array_cache *shared = NULL;
J
Joonsoo Kim 已提交
1187
		struct alien_cache **alien = NULL;
1188 1189 1190 1191

		if (cachep->shared) {
			shared = alloc_arraycache(node,
				cachep->shared * cachep->batchcount,
1192
				0xbaadf00d, GFP_KERNEL);
1193
			if (!shared)
L
Linus Torvalds 已提交
1194
				goto bad;
1195 1196
		}
		if (use_alien_caches) {
1197
			alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
1198 1199
			if (!alien) {
				kfree(shared);
1200
				goto bad;
1201
			}
1202
		}
1203
		n = get_node(cachep, node);
1204
		BUG_ON(!n);
1205

1206 1207
		spin_lock_irq(&n->list_lock);
		if (!n->shared) {
1208 1209 1210 1211
			/*
			 * We are serialised from CPU_DEAD or
			 * CPU_UP_CANCELLED by the cpucontrol lock
			 */
1212
			n->shared = shared;
1213 1214
			shared = NULL;
		}
1215
#ifdef CONFIG_NUMA
1216 1217
		if (!n->alien) {
			n->alien = alien;
1218
			alien = NULL;
L
Linus Torvalds 已提交
1219
		}
1220
#endif
1221
		spin_unlock_irq(&n->list_lock);
1222 1223 1224
		kfree(shared);
		free_alien_cache(alien);
	}
1225

1226 1227
	return 0;
bad:
1228
	cpuup_canceled(cpu);
1229 1230 1231
	return -ENOMEM;
}

1232
static int cpuup_callback(struct notifier_block *nfb,
1233 1234 1235 1236 1237 1238 1239 1240
				    unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
	int err = 0;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
1241
		mutex_lock(&slab_mutex);
1242
		err = cpuup_prepare(cpu);
1243
		mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
1244 1245
		break;
	case CPU_ONLINE:
1246
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
1247 1248 1249
		start_cpu_timer(cpu);
		break;
#ifdef CONFIG_HOTPLUG_CPU
1250
  	case CPU_DOWN_PREPARE:
1251
  	case CPU_DOWN_PREPARE_FROZEN:
1252
		/*
1253
		 * Shutdown cache reaper. Note that the slab_mutex is
1254 1255 1256 1257
		 * held so that if cache_reap() is invoked it cannot do
		 * anything expensive but will only modify reap_work
		 * and reschedule the timer.
		*/
1258
		cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1259
		/* Now the cache_reaper is guaranteed to be not running. */
1260
		per_cpu(slab_reap_work, cpu).work.func = NULL;
1261 1262
  		break;
  	case CPU_DOWN_FAILED:
1263
  	case CPU_DOWN_FAILED_FROZEN:
1264 1265
		start_cpu_timer(cpu);
  		break;
L
Linus Torvalds 已提交
1266
	case CPU_DEAD:
1267
	case CPU_DEAD_FROZEN:
1268 1269
		/*
		 * Even if all the cpus of a node are down, we don't free the
1270
		 * kmem_cache_node of any cache. This to avoid a race between
1271
		 * cpu_down, and a kmalloc allocation from another cpu for
1272
		 * memory from the node of the cpu going down.  The node
1273 1274 1275
		 * structure is usually allocated from kmem_cache_create() and
		 * gets destroyed at kmem_cache_destroy().
		 */
S
Simon Arlott 已提交
1276
		/* fall through */
1277
#endif
L
Linus Torvalds 已提交
1278
	case CPU_UP_CANCELED:
1279
	case CPU_UP_CANCELED_FROZEN:
1280
		mutex_lock(&slab_mutex);
1281
		cpuup_canceled(cpu);
1282
		mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
1283 1284
		break;
	}
1285
	return notifier_from_errno(err);
L
Linus Torvalds 已提交
1286 1287
}

1288
static struct notifier_block cpucache_notifier = {
1289 1290
	&cpuup_callback, NULL, 0
};
L
Linus Torvalds 已提交
1291

1292 1293 1294 1295 1296 1297
#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
/*
 * Drains freelist for a node on each slab cache, used for memory hot-remove.
 * Returns -EBUSY if all objects cannot be drained so that the node is not
 * removed.
 *
1298
 * Must hold slab_mutex.
1299
 */
1300
static int __meminit drain_cache_node_node(int node)
1301 1302 1303 1304
{
	struct kmem_cache *cachep;
	int ret = 0;

1305
	list_for_each_entry(cachep, &slab_caches, list) {
1306
		struct kmem_cache_node *n;
1307

1308
		n = get_node(cachep, node);
1309
		if (!n)
1310 1311
			continue;

1312
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1313

1314 1315
		if (!list_empty(&n->slabs_full) ||
		    !list_empty(&n->slabs_partial)) {
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
			ret = -EBUSY;
			break;
		}
	}
	return ret;
}

static int __meminit slab_memory_callback(struct notifier_block *self,
					unsigned long action, void *arg)
{
	struct memory_notify *mnb = arg;
	int ret = 0;
	int nid;

	nid = mnb->status_change_nid;
	if (nid < 0)
		goto out;

	switch (action) {
	case MEM_GOING_ONLINE:
1336
		mutex_lock(&slab_mutex);
1337
		ret = init_cache_node_node(nid);
1338
		mutex_unlock(&slab_mutex);
1339 1340
		break;
	case MEM_GOING_OFFLINE:
1341
		mutex_lock(&slab_mutex);
1342
		ret = drain_cache_node_node(nid);
1343
		mutex_unlock(&slab_mutex);
1344 1345 1346 1347 1348 1349 1350 1351
		break;
	case MEM_ONLINE:
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
out:
1352
	return notifier_from_errno(ret);
1353 1354 1355
}
#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */

1356
/*
1357
 * swap the static kmem_cache_node with kmalloced memory
1358
 */
1359
static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
1360
				int nodeid)
1361
{
1362
	struct kmem_cache_node *ptr;
1363

1364
	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1365 1366
	BUG_ON(!ptr);

1367
	memcpy(ptr, list, sizeof(struct kmem_cache_node));
1368 1369 1370 1371 1372
	/*
	 * Do not assume that spinlocks can be initialized via memcpy:
	 */
	spin_lock_init(&ptr->list_lock);

1373
	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1374
	cachep->node[nodeid] = ptr;
1375 1376
}

1377
/*
1378 1379
 * For setting up all the kmem_cache_node for cache whose buffer_size is same as
 * size of kmem_cache_node.
1380
 */
1381
static void __init set_up_node(struct kmem_cache *cachep, int index)
1382 1383 1384 1385
{
	int node;

	for_each_online_node(node) {
1386
		cachep->node[node] = &init_kmem_cache_node[index + node];
1387
		cachep->node[node]->next_reap = jiffies +
1388 1389
		    REAPTIMEOUT_NODE +
		    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1390 1391 1392
	}
}

A
Andrew Morton 已提交
1393 1394 1395
/*
 * Initialisation.  Called after the page allocator have been initialised and
 * before smp_init().
L
Linus Torvalds 已提交
1396 1397 1398
 */
void __init kmem_cache_init(void)
{
1399 1400
	int i;

1401 1402
	BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
					sizeof(struct rcu_head));
1403 1404
	kmem_cache = &kmem_cache_boot;

1405
	if (num_possible_nodes() == 1)
1406 1407
		use_alien_caches = 0;

C
Christoph Lameter 已提交
1408
	for (i = 0; i < NUM_INIT_LISTS; i++)
1409
		kmem_cache_node_init(&init_kmem_cache_node[i]);
C
Christoph Lameter 已提交
1410

L
Linus Torvalds 已提交
1411 1412
	/*
	 * Fragmentation resistance on low memory - only use bigger
1413 1414
	 * page orders on machines with more than 32MB of memory if
	 * not overridden on the command line.
L
Linus Torvalds 已提交
1415
	 */
1416
	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1417
		slab_max_order = SLAB_MAX_ORDER_HI;
L
Linus Torvalds 已提交
1418 1419 1420

	/* Bootstrap is tricky, because several objects are allocated
	 * from caches that do not exist yet:
1421 1422 1423
	 * 1) initialize the kmem_cache cache: it contains the struct
	 *    kmem_cache structures of all caches, except kmem_cache itself:
	 *    kmem_cache is statically allocated.
1424
	 *    Initially an __init data area is used for the head array and the
1425
	 *    kmem_cache_node structures, it's replaced with a kmalloc allocated
1426
	 *    array at the end of the bootstrap.
L
Linus Torvalds 已提交
1427
	 * 2) Create the first kmalloc cache.
1428
	 *    The struct kmem_cache for the new cache is allocated normally.
1429 1430 1431
	 *    An __init data area is used for the head array.
	 * 3) Create the remaining kmalloc caches, with minimally sized
	 *    head arrays.
1432
	 * 4) Replace the __init data head arrays for kmem_cache and the first
L
Linus Torvalds 已提交
1433
	 *    kmalloc cache with kmalloc allocated arrays.
1434
	 * 5) Replace the __init data for kmem_cache_node for kmem_cache and
1435 1436
	 *    the other cache's with kmalloc allocated memory.
	 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
L
Linus Torvalds 已提交
1437 1438
	 */

1439
	/* 1) create the kmem_cache */
L
Linus Torvalds 已提交
1440

E
Eric Dumazet 已提交
1441
	/*
1442
	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
E
Eric Dumazet 已提交
1443
	 */
1444
	create_boot_cache(kmem_cache, "kmem_cache",
1445
		offsetof(struct kmem_cache, node) +
1446
				  nr_node_ids * sizeof(struct kmem_cache_node *),
1447 1448
				  SLAB_HWCACHE_ALIGN);
	list_add(&kmem_cache->list, &slab_caches);
1449
	slab_state = PARTIAL;
L
Linus Torvalds 已提交
1450

A
Andrew Morton 已提交
1451
	/*
1452 1453
	 * Initialize the caches that provide memory for the  kmem_cache_node
	 * structures first.  Without this, further allocations will bug.
1454
	 */
1455
	kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
1456
				kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
1457
	slab_state = PARTIAL_NODE;
1458
	setup_kmalloc_cache_index_table();
1459

1460 1461
	slab_early_init = 0;

1462
	/* 5) Replace the bootstrap kmem_cache_node */
1463
	{
P
Pekka Enberg 已提交
1464 1465
		int nid;

1466
		for_each_online_node(nid) {
1467
			init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1468

1469
			init_list(kmalloc_caches[INDEX_NODE],
1470
					  &init_kmem_cache_node[SIZE_NODE + nid], nid);
1471 1472
		}
	}
L
Linus Torvalds 已提交
1473

1474
	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1475 1476 1477 1478 1479 1480
}

void __init kmem_cache_init_late(void)
{
	struct kmem_cache *cachep;

1481
	slab_state = UP;
P
Peter Zijlstra 已提交
1482

1483
	/* 6) resize the head arrays to their final sizes */
1484 1485
	mutex_lock(&slab_mutex);
	list_for_each_entry(cachep, &slab_caches, list)
1486 1487
		if (enable_cpucache(cachep, GFP_NOWAIT))
			BUG();
1488
	mutex_unlock(&slab_mutex);
1489

1490 1491 1492
	/* Done! */
	slab_state = FULL;

A
Andrew Morton 已提交
1493 1494 1495
	/*
	 * Register a cpu startup notifier callback that initializes
	 * cpu_cache_get for all new cpus
L
Linus Torvalds 已提交
1496 1497 1498
	 */
	register_cpu_notifier(&cpucache_notifier);

1499 1500 1501
#ifdef CONFIG_NUMA
	/*
	 * Register a memory hotplug callback that initializes and frees
1502
	 * node.
1503 1504 1505 1506
	 */
	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
#endif

A
Andrew Morton 已提交
1507 1508 1509
	/*
	 * The reap timers are started later, with a module init call: That part
	 * of the kernel is not yet operational.
L
Linus Torvalds 已提交
1510 1511 1512 1513 1514 1515 1516
	 */
}

static int __init cpucache_init(void)
{
	int cpu;

A
Andrew Morton 已提交
1517 1518
	/*
	 * Register the timers that return unneeded pages to the page allocator
L
Linus Torvalds 已提交
1519
	 */
1520
	for_each_online_cpu(cpu)
A
Andrew Morton 已提交
1521
		start_cpu_timer(cpu);
1522 1523

	/* Done! */
1524
	slab_state = FULL;
L
Linus Torvalds 已提交
1525 1526 1527 1528
	return 0;
}
__initcall(cpucache_init);

1529 1530 1531
static noinline void
slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
{
1532
#if DEBUG
1533
	struct kmem_cache_node *n;
1534
	struct page *page;
1535 1536
	unsigned long flags;
	int node;
1537 1538 1539 1540 1541
	static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
		return;
1542 1543 1544 1545 1546

	printk(KERN_WARNING
		"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
		nodeid, gfpflags);
	printk(KERN_WARNING "  cache: %s, object size: %d, order: %d\n",
1547
		cachep->name, cachep->size, cachep->gfporder);
1548

1549
	for_each_kmem_cache_node(cachep, node, n) {
1550 1551 1552
		unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
		unsigned long active_slabs = 0, num_slabs = 0;

1553
		spin_lock_irqsave(&n->list_lock, flags);
1554
		list_for_each_entry(page, &n->slabs_full, lru) {
1555 1556 1557
			active_objs += cachep->num;
			active_slabs++;
		}
1558 1559
		list_for_each_entry(page, &n->slabs_partial, lru) {
			active_objs += page->active;
1560 1561
			active_slabs++;
		}
1562
		list_for_each_entry(page, &n->slabs_free, lru)
1563 1564
			num_slabs++;

1565 1566
		free_objects += n->free_objects;
		spin_unlock_irqrestore(&n->list_lock, flags);
1567 1568 1569 1570 1571 1572 1573 1574

		num_slabs += active_slabs;
		num_objs = num_slabs * cachep->num;
		printk(KERN_WARNING
			"  node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
			node, active_slabs, num_slabs, active_objs, num_objs,
			free_objects);
	}
1575
#endif
1576 1577
}

L
Linus Torvalds 已提交
1578
/*
W
Wang Sheng-Hui 已提交
1579 1580
 * Interface to system's page allocator. No need to hold the
 * kmem_cache_node ->list_lock.
L
Linus Torvalds 已提交
1581 1582 1583 1584 1585
 *
 * If we requested dmaable memory, we will get it. Even if we
 * did not request dmaable memory, we might get it, but that
 * would be relatively rare and ignorable.
 */
1586 1587
static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
								int nodeid)
L
Linus Torvalds 已提交
1588 1589
{
	struct page *page;
1590
	int nr_pages;
1591

1592
	flags |= cachep->allocflags;
1593 1594
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
		flags |= __GFP_RECLAIMABLE;
1595

1596
	page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
1597
	if (!page) {
1598
		slab_out_of_memory(cachep, flags, nodeid);
L
Linus Torvalds 已提交
1599
		return NULL;
1600
	}
L
Linus Torvalds 已提交
1601

1602 1603 1604 1605 1606
	if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
		__free_pages(page, cachep->gfporder);
		return NULL;
	}

1607
	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1608
	if (page_is_pfmemalloc(page))
1609 1610
		pfmemalloc_active = true;

1611
	nr_pages = (1 << cachep->gfporder);
L
Linus Torvalds 已提交
1612
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1613 1614 1615 1616 1617
		add_zone_page_state(page_zone(page),
			NR_SLAB_RECLAIMABLE, nr_pages);
	else
		add_zone_page_state(page_zone(page),
			NR_SLAB_UNRECLAIMABLE, nr_pages);
1618
	__SetPageSlab(page);
1619
	if (page_is_pfmemalloc(page))
1620
		SetPageSlabPfmemalloc(page);
1621

1622 1623 1624 1625 1626 1627 1628 1629
	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
		kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);

		if (cachep->ctor)
			kmemcheck_mark_uninitialized_pages(page, nr_pages);
		else
			kmemcheck_mark_unallocated_pages(page, nr_pages);
	}
P
Pekka Enberg 已提交
1630

1631
	return page;
L
Linus Torvalds 已提交
1632 1633 1634 1635 1636
}

/*
 * Interface to system's page release.
 */
1637
static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
L
Linus Torvalds 已提交
1638
{
1639
	const unsigned long nr_freed = (1 << cachep->gfporder);
L
Linus Torvalds 已提交
1640

1641
	kmemcheck_free_shadow(page, cachep->gfporder);
P
Pekka Enberg 已提交
1642

1643 1644 1645 1646 1647 1648
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
		sub_zone_page_state(page_zone(page),
				NR_SLAB_RECLAIMABLE, nr_freed);
	else
		sub_zone_page_state(page_zone(page),
				NR_SLAB_UNRECLAIMABLE, nr_freed);
J
Joonsoo Kim 已提交
1649

1650
	BUG_ON(!PageSlab(page));
J
Joonsoo Kim 已提交
1651
	__ClearPageSlabPfmemalloc(page);
1652
	__ClearPageSlab(page);
1653 1654
	page_mapcount_reset(page);
	page->mapping = NULL;
G
Glauber Costa 已提交
1655

L
Linus Torvalds 已提交
1656 1657
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += nr_freed;
1658
	__free_kmem_pages(page, cachep->gfporder);
L
Linus Torvalds 已提交
1659 1660 1661 1662
}

static void kmem_rcu_free(struct rcu_head *head)
{
1663 1664
	struct kmem_cache *cachep;
	struct page *page;
L
Linus Torvalds 已提交
1665

1666 1667 1668 1669
	page = container_of(head, struct page, rcu_head);
	cachep = page->slab_cache;

	kmem_freepages(cachep, page);
L
Linus Torvalds 已提交
1670 1671 1672 1673 1674
}

#if DEBUG

#ifdef CONFIG_DEBUG_PAGEALLOC
1675
static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
P
Pekka Enberg 已提交
1676
			    unsigned long caller)
L
Linus Torvalds 已提交
1677
{
1678
	int size = cachep->object_size;
L
Linus Torvalds 已提交
1679

1680
	addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
L
Linus Torvalds 已提交
1681

P
Pekka Enberg 已提交
1682
	if (size < 5 * sizeof(unsigned long))
L
Linus Torvalds 已提交
1683 1684
		return;

P
Pekka Enberg 已提交
1685 1686 1687 1688
	*addr++ = 0x12345678;
	*addr++ = caller;
	*addr++ = smp_processor_id();
	size -= 3 * sizeof(unsigned long);
L
Linus Torvalds 已提交
1689 1690 1691 1692 1693 1694 1695
	{
		unsigned long *sptr = &caller;
		unsigned long svalue;

		while (!kstack_end(sptr)) {
			svalue = *sptr++;
			if (kernel_text_address(svalue)) {
P
Pekka Enberg 已提交
1696
				*addr++ = svalue;
L
Linus Torvalds 已提交
1697 1698 1699 1700 1701 1702 1703
				size -= sizeof(unsigned long);
				if (size <= sizeof(unsigned long))
					break;
			}
		}

	}
P
Pekka Enberg 已提交
1704
	*addr++ = 0x87654321;
L
Linus Torvalds 已提交
1705 1706 1707
}
#endif

1708
static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
L
Linus Torvalds 已提交
1709
{
1710
	int size = cachep->object_size;
1711
	addr = &((char *)addr)[obj_offset(cachep)];
L
Linus Torvalds 已提交
1712 1713

	memset(addr, val, size);
P
Pekka Enberg 已提交
1714
	*(unsigned char *)(addr + size - 1) = POISON_END;
L
Linus Torvalds 已提交
1715 1716 1717 1718 1719
}

static void dump_line(char *data, int offset, int limit)
{
	int i;
D
Dave Jones 已提交
1720 1721 1722
	unsigned char error = 0;
	int bad_count = 0;

1723
	printk(KERN_ERR "%03x: ", offset);
D
Dave Jones 已提交
1724 1725 1726 1727 1728 1729
	for (i = 0; i < limit; i++) {
		if (data[offset + i] != POISON_FREE) {
			error = data[offset + i];
			bad_count++;
		}
	}
1730 1731
	print_hex_dump(KERN_CONT, "", 0, 16, 1,
			&data[offset], limit, 1);
D
Dave Jones 已提交
1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745

	if (bad_count == 1) {
		error ^= POISON_FREE;
		if (!(error & (error - 1))) {
			printk(KERN_ERR "Single bit error detected. Probably "
					"bad RAM.\n");
#ifdef CONFIG_X86
			printk(KERN_ERR "Run memtest86+ or a similar memory "
					"test tool.\n");
#else
			printk(KERN_ERR "Run a memory test tool.\n");
#endif
		}
	}
L
Linus Torvalds 已提交
1746 1747 1748 1749 1750
}
#endif

#if DEBUG

1751
static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
L
Linus Torvalds 已提交
1752 1753 1754 1755 1756
{
	int i, size;
	char *realobj;

	if (cachep->flags & SLAB_RED_ZONE) {
1757
		printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
A
Andrew Morton 已提交
1758 1759
			*dbg_redzone1(cachep, objp),
			*dbg_redzone2(cachep, objp));
L
Linus Torvalds 已提交
1760 1761 1762
	}

	if (cachep->flags & SLAB_STORE_USER) {
J
Joe Perches 已提交
1763 1764 1765
		printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
		       *dbg_userword(cachep, objp),
		       *dbg_userword(cachep, objp));
L
Linus Torvalds 已提交
1766
	}
1767
	realobj = (char *)objp + obj_offset(cachep);
1768
	size = cachep->object_size;
P
Pekka Enberg 已提交
1769
	for (i = 0; i < size && lines; i += 16, lines--) {
L
Linus Torvalds 已提交
1770 1771
		int limit;
		limit = 16;
P
Pekka Enberg 已提交
1772 1773
		if (i + limit > size)
			limit = size - i;
L
Linus Torvalds 已提交
1774 1775 1776 1777
		dump_line(realobj, i, limit);
	}
}

1778
static void check_poison_obj(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
1779 1780 1781 1782 1783
{
	char *realobj;
	int size, i;
	int lines = 0;

1784
	realobj = (char *)objp + obj_offset(cachep);
1785
	size = cachep->object_size;
L
Linus Torvalds 已提交
1786

P
Pekka Enberg 已提交
1787
	for (i = 0; i < size; i++) {
L
Linus Torvalds 已提交
1788
		char exp = POISON_FREE;
P
Pekka Enberg 已提交
1789
		if (i == size - 1)
L
Linus Torvalds 已提交
1790 1791 1792 1793 1794 1795
			exp = POISON_END;
		if (realobj[i] != exp) {
			int limit;
			/* Mismatch ! */
			/* Print header */
			if (lines == 0) {
P
Pekka Enberg 已提交
1796
				printk(KERN_ERR
1797 1798
					"Slab corruption (%s): %s start=%p, len=%d\n",
					print_tainted(), cachep->name, realobj, size);
L
Linus Torvalds 已提交
1799 1800 1801
				print_objinfo(cachep, objp, 0);
			}
			/* Hexdump the affected line */
P
Pekka Enberg 已提交
1802
			i = (i / 16) * 16;
L
Linus Torvalds 已提交
1803
			limit = 16;
P
Pekka Enberg 已提交
1804 1805
			if (i + limit > size)
				limit = size - i;
L
Linus Torvalds 已提交
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
			dump_line(realobj, i, limit);
			i += 16;
			lines++;
			/* Limit to 5 lines */
			if (lines > 5)
				break;
		}
	}
	if (lines != 0) {
		/* Print some data about the neighboring objects, if they
		 * exist:
		 */
1818
		struct page *page = virt_to_head_page(objp);
1819
		unsigned int objnr;
L
Linus Torvalds 已提交
1820

1821
		objnr = obj_to_index(cachep, page, objp);
L
Linus Torvalds 已提交
1822
		if (objnr) {
1823
			objp = index_to_obj(cachep, page, objnr - 1);
1824
			realobj = (char *)objp + obj_offset(cachep);
L
Linus Torvalds 已提交
1825
			printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
P
Pekka Enberg 已提交
1826
			       realobj, size);
L
Linus Torvalds 已提交
1827 1828
			print_objinfo(cachep, objp, 2);
		}
P
Pekka Enberg 已提交
1829
		if (objnr + 1 < cachep->num) {
1830
			objp = index_to_obj(cachep, page, objnr + 1);
1831
			realobj = (char *)objp + obj_offset(cachep);
L
Linus Torvalds 已提交
1832
			printk(KERN_ERR "Next obj: start=%p, len=%d\n",
P
Pekka Enberg 已提交
1833
			       realobj, size);
L
Linus Torvalds 已提交
1834 1835 1836 1837 1838 1839
			print_objinfo(cachep, objp, 2);
		}
	}
}
#endif

1840
#if DEBUG
1841 1842
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
						struct page *page)
L
Linus Torvalds 已提交
1843 1844 1845
{
	int i;
	for (i = 0; i < cachep->num; i++) {
1846
		void *objp = index_to_obj(cachep, page, i);
L
Linus Torvalds 已提交
1847 1848 1849

		if (cachep->flags & SLAB_POISON) {
#ifdef CONFIG_DEBUG_PAGEALLOC
1850
			if (cachep->size % PAGE_SIZE == 0 &&
A
Andrew Morton 已提交
1851
					OFF_SLAB(cachep))
P
Pekka Enberg 已提交
1852
				kernel_map_pages(virt_to_page(objp),
1853
					cachep->size / PAGE_SIZE, 1);
L
Linus Torvalds 已提交
1854 1855 1856 1857 1858 1859 1860 1861 1862
			else
				check_poison_obj(cachep, objp);
#else
			check_poison_obj(cachep, objp);
#endif
		}
		if (cachep->flags & SLAB_RED_ZONE) {
			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "start of a freed object "
P
Pekka Enberg 已提交
1863
					   "was overwritten");
L
Linus Torvalds 已提交
1864 1865
			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "end of a freed object "
P
Pekka Enberg 已提交
1866
					   "was overwritten");
L
Linus Torvalds 已提交
1867 1868
		}
	}
1869
}
L
Linus Torvalds 已提交
1870
#else
1871 1872
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
						struct page *page)
1873 1874
{
}
L
Linus Torvalds 已提交
1875 1876
#endif

1877 1878 1879
/**
 * slab_destroy - destroy and release all objects in a slab
 * @cachep: cache pointer being destroyed
1880
 * @page: page pointer being destroyed
1881
 *
W
Wang Sheng-Hui 已提交
1882 1883 1884
 * Destroy all the objs in a slab page, and release the mem back to the system.
 * Before calling the slab page must have been unlinked from the cache. The
 * kmem_cache_node ->list_lock is not held/needed.
1885
 */
1886
static void slab_destroy(struct kmem_cache *cachep, struct page *page)
1887
{
1888
	void *freelist;
1889

1890 1891
	freelist = page->freelist;
	slab_destroy_debugcheck(cachep, page);
1892 1893 1894
	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
		call_rcu(&page->rcu_head, kmem_rcu_free);
	else
1895
		kmem_freepages(cachep, page);
1896 1897

	/*
1898
	 * From now on, we don't use freelist
1899 1900 1901
	 * although actual page can be freed in rcu context
	 */
	if (OFF_SLAB(cachep))
1902
		kmem_cache_free(cachep->freelist_cache, freelist);
L
Linus Torvalds 已提交
1903 1904
}

1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
{
	struct page *page, *n;

	list_for_each_entry_safe(page, n, list, lru) {
		list_del(&page->lru);
		slab_destroy(cachep, page);
	}
}

1915
/**
1916 1917 1918 1919 1920 1921 1922
 * calculate_slab_order - calculate size (page order) of slabs
 * @cachep: pointer to the cache that is being created
 * @size: size of objects to be created in this cache.
 * @align: required alignment for the objects.
 * @flags: slab allocation flags
 *
 * Also calculates the number of objects per slab.
1923 1924 1925 1926 1927
 *
 * This could be made much more intelligent.  For now, try to avoid using
 * high order pages for slabs.  When the gfp() functions are more friendly
 * towards high-order requests, this should be changed.
 */
A
Andrew Morton 已提交
1928
static size_t calculate_slab_order(struct kmem_cache *cachep,
R
Randy Dunlap 已提交
1929
			size_t size, size_t align, unsigned long flags)
1930
{
1931
	unsigned long offslab_limit;
1932
	size_t left_over = 0;
1933
	int gfporder;
1934

1935
	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
1936 1937 1938
		unsigned int num;
		size_t remainder;

1939
		cache_estimate(gfporder, size, align, flags, &remainder, &num);
1940 1941
		if (!num)
			continue;
1942

1943 1944 1945 1946
		/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
		if (num > SLAB_OBJ_MAX_NUM)
			break;

1947
		if (flags & CFLGS_OFF_SLAB) {
1948
			size_t freelist_size_per_obj = sizeof(freelist_idx_t);
1949 1950 1951 1952 1953
			/*
			 * Max number of objs-per-slab for caches which
			 * use off-slab slabs. Needed to avoid a possible
			 * looping condition in cache_grow().
			 */
1954 1955
			if (IS_ENABLED(CONFIG_DEBUG_SLAB_LEAK))
				freelist_size_per_obj += sizeof(char);
1956
			offslab_limit = size;
1957
			offslab_limit /= freelist_size_per_obj;
1958 1959 1960 1961

 			if (num > offslab_limit)
				break;
		}
1962

1963
		/* Found something acceptable - save it away */
1964
		cachep->num = num;
1965
		cachep->gfporder = gfporder;
1966 1967
		left_over = remainder;

1968 1969 1970 1971 1972 1973 1974 1975
		/*
		 * A VFS-reclaimable slab tends to have most allocations
		 * as GFP_NOFS and we really don't want to have to be allocating
		 * higher-order pages when we are unable to shrink dcache.
		 */
		if (flags & SLAB_RECLAIM_ACCOUNT)
			break;

1976 1977 1978 1979
		/*
		 * Large number of objects is good, but very large slabs are
		 * currently bad for the gfp()s.
		 */
1980
		if (gfporder >= slab_max_order)
1981 1982
			break;

1983 1984 1985
		/*
		 * Acceptable internal fragmentation?
		 */
A
Andrew Morton 已提交
1986
		if (left_over * 8 <= (PAGE_SIZE << gfporder))
1987 1988 1989 1990 1991
			break;
	}
	return left_over;
}

1992 1993 1994 1995 1996 1997 1998 1999
static struct array_cache __percpu *alloc_kmem_cache_cpus(
		struct kmem_cache *cachep, int entries, int batchcount)
{
	int cpu;
	size_t size;
	struct array_cache __percpu *cpu_cache;

	size = sizeof(void *) * entries + sizeof(struct array_cache);
2000
	cpu_cache = __alloc_percpu(size, sizeof(void *));
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012

	if (!cpu_cache)
		return NULL;

	for_each_possible_cpu(cpu) {
		init_arraycache(per_cpu_ptr(cpu_cache, cpu),
				entries, batchcount);
	}

	return cpu_cache;
}

2013
static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2014
{
2015
	if (slab_state >= FULL)
2016
		return enable_cpucache(cachep, gfp);
2017

2018 2019 2020 2021
	cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
	if (!cachep->cpu_cache)
		return 1;

2022
	if (slab_state == DOWN) {
2023 2024
		/* Creation of first cache (kmem_cache). */
		set_up_node(kmem_cache, CACHE_CACHE);
2025
	} else if (slab_state == PARTIAL) {
2026 2027
		/* For kmem_cache_node */
		set_up_node(cachep, SIZE_NODE);
2028
	} else {
2029
		int node;
2030

2031 2032 2033 2034 2035
		for_each_online_node(node) {
			cachep->node[node] = kmalloc_node(
				sizeof(struct kmem_cache_node), gfp, node);
			BUG_ON(!cachep->node[node]);
			kmem_cache_node_init(cachep->node[node]);
2036 2037
		}
	}
2038

2039
	cachep->node[numa_mem_id()]->next_reap =
2040 2041
			jiffies + REAPTIMEOUT_NODE +
			((unsigned long)cachep) % REAPTIMEOUT_NODE;
2042 2043 2044 2045 2046 2047 2048

	cpu_cache_get(cachep)->avail = 0;
	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
	cpu_cache_get(cachep)->batchcount = 1;
	cpu_cache_get(cachep)->touched = 0;
	cachep->batchcount = 1;
	cachep->limit = BOOT_CPUCACHE_ENTRIES;
2049
	return 0;
2050 2051
}

J
Joonsoo Kim 已提交
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
unsigned long kmem_cache_flags(unsigned long object_size,
	unsigned long flags, const char *name,
	void (*ctor)(void *))
{
	return flags;
}

struct kmem_cache *
__kmem_cache_alias(const char *name, size_t size, size_t align,
		   unsigned long flags, void (*ctor)(void *))
{
	struct kmem_cache *cachep;

	cachep = find_mergeable(size, align, flags, name, ctor);
	if (cachep) {
		cachep->refcount++;

		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
		cachep->object_size = max_t(int, cachep->object_size, size);
	}
	return cachep;
}

L
Linus Torvalds 已提交
2078
/**
2079
 * __kmem_cache_create - Create a cache.
R
Randy Dunlap 已提交
2080
 * @cachep: cache management descriptor
L
Linus Torvalds 已提交
2081 2082 2083 2084
 * @flags: SLAB flags
 *
 * Returns a ptr to the cache on success, NULL on failure.
 * Cannot be called within a int, but can be interrupted.
2085
 * The @ctor is run when new pages are allocated by the cache.
L
Linus Torvalds 已提交
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
 *
 * The flags are
 *
 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
 * to catch references to uninitialised memory.
 *
 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
 * for buffer overruns.
 *
 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
 * cacheline.  This can be beneficial if you're counting cycles as closely
 * as davem.
 */
2099
int
2100
__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
L
Linus Torvalds 已提交
2101
{
2102 2103
	size_t left_over, freelist_size;
	size_t ralign = BYTES_PER_WORD;
2104
	gfp_t gfp;
2105
	int err;
2106
	size_t size = cachep->size;
L
Linus Torvalds 已提交
2107 2108 2109 2110 2111 2112 2113 2114 2115

#if DEBUG
#if FORCED_DEBUG
	/*
	 * Enable redzoning and last user accounting, except for caches with
	 * large objects, if the increased size would increase the object size
	 * above the next power of two: caches with object sizes just above a
	 * power of two have a significant amount of internal fragmentation.
	 */
D
David Woodhouse 已提交
2116 2117
	if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
						2 * sizeof(unsigned long long)))
P
Pekka Enberg 已提交
2118
		flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
L
Linus Torvalds 已提交
2119 2120 2121 2122 2123 2124 2125
	if (!(flags & SLAB_DESTROY_BY_RCU))
		flags |= SLAB_POISON;
#endif
	if (flags & SLAB_DESTROY_BY_RCU)
		BUG_ON(flags & SLAB_POISON);
#endif

A
Andrew Morton 已提交
2126 2127
	/*
	 * Check that size is in terms of words.  This is needed to avoid
L
Linus Torvalds 已提交
2128 2129 2130
	 * unaligned accesses for some archs when redzoning is used, and makes
	 * sure any on-slab bufctl's are also correctly aligned.
	 */
P
Pekka Enberg 已提交
2131 2132 2133
	if (size & (BYTES_PER_WORD - 1)) {
		size += (BYTES_PER_WORD - 1);
		size &= ~(BYTES_PER_WORD - 1);
L
Linus Torvalds 已提交
2134 2135
	}

D
David Woodhouse 已提交
2136 2137 2138 2139 2140 2141 2142
	if (flags & SLAB_RED_ZONE) {
		ralign = REDZONE_ALIGN;
		/* If redzoning, ensure that the second redzone is suitably
		 * aligned, by adjusting the object size accordingly. */
		size += REDZONE_ALIGN - 1;
		size &= ~(REDZONE_ALIGN - 1);
	}
2143

2144
	/* 3) caller mandated alignment */
2145 2146
	if (ralign < cachep->align) {
		ralign = cachep->align;
L
Linus Torvalds 已提交
2147
	}
2148 2149
	/* disable debug if necessary */
	if (ralign > __alignof__(unsigned long long))
2150
		flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
A
Andrew Morton 已提交
2151
	/*
2152
	 * 4) Store it.
L
Linus Torvalds 已提交
2153
	 */
2154
	cachep->align = ralign;
L
Linus Torvalds 已提交
2155

2156 2157 2158 2159 2160
	if (slab_is_available())
		gfp = GFP_KERNEL;
	else
		gfp = GFP_NOWAIT;

L
Linus Torvalds 已提交
2161 2162
#if DEBUG

2163 2164 2165 2166
	/*
	 * Both debugging options require word-alignment which is calculated
	 * into align above.
	 */
L
Linus Torvalds 已提交
2167 2168
	if (flags & SLAB_RED_ZONE) {
		/* add space for red zone words */
2169 2170
		cachep->obj_offset += sizeof(unsigned long long);
		size += 2 * sizeof(unsigned long long);
L
Linus Torvalds 已提交
2171 2172
	}
	if (flags & SLAB_STORE_USER) {
2173
		/* user store requires one word storage behind the end of
D
David Woodhouse 已提交
2174 2175
		 * the real object. But if the second red zone needs to be
		 * aligned to 64 bits, we must allow that much space.
L
Linus Torvalds 已提交
2176
		 */
D
David Woodhouse 已提交
2177 2178 2179 2180
		if (flags & SLAB_RED_ZONE)
			size += REDZONE_ALIGN;
		else
			size += BYTES_PER_WORD;
L
Linus Torvalds 已提交
2181 2182
	}
#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
	/*
	 * To activate debug pagealloc, off-slab management is necessary
	 * requirement. In early phase of initialization, small sized slab
	 * doesn't get initialized so it would not be possible. So, we need
	 * to check size >= 256. It guarantees that all necessary small
	 * sized slab is initialized in current slab initialization sequence.
	 */
	if (!slab_early_init && size >= kmalloc_size(INDEX_NODE) &&
		size >= 256 && cachep->object_size > cache_line_size() &&
		ALIGN(size, cachep->align) < PAGE_SIZE) {
2193
		cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
L
Linus Torvalds 已提交
2194 2195 2196 2197 2198
		size = PAGE_SIZE;
	}
#endif
#endif

2199 2200 2201
	/*
	 * Determine if the slab management is 'on' or 'off' slab.
	 * (bootstrapping cannot cope with offslab caches so don't do
2202 2203
	 * it too early on. Always use on-slab management when
	 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
2204
	 */
2205
	if (size >= OFF_SLAB_MIN_SIZE && !slab_early_init &&
2206
	    !(flags & SLAB_NOLEAKTRACE))
L
Linus Torvalds 已提交
2207 2208 2209 2210 2211 2212
		/*
		 * Size is large, assume best to place the slab management obj
		 * off-slab (should allow better packing of objs).
		 */
		flags |= CFLGS_OFF_SLAB;

2213
	size = ALIGN(size, cachep->align);
2214 2215 2216 2217 2218 2219
	/*
	 * We should restrict the number of objects in a slab to implement
	 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
	 */
	if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
		size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
L
Linus Torvalds 已提交
2220

2221
	left_over = calculate_slab_order(cachep, size, cachep->align, flags);
L
Linus Torvalds 已提交
2222

2223
	if (!cachep->num)
2224
		return -E2BIG;
L
Linus Torvalds 已提交
2225

2226
	freelist_size = calculate_freelist_size(cachep->num, cachep->align);
L
Linus Torvalds 已提交
2227 2228 2229 2230 2231

	/*
	 * If the slab has been placed off-slab, and we have enough space then
	 * move it on-slab. This is at the expense of any extra colouring.
	 */
2232
	if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
L
Linus Torvalds 已提交
2233
		flags &= ~CFLGS_OFF_SLAB;
2234
		left_over -= freelist_size;
L
Linus Torvalds 已提交
2235 2236 2237 2238
	}

	if (flags & CFLGS_OFF_SLAB) {
		/* really off slab. No need for manual alignment */
2239
		freelist_size = calculate_freelist_size(cachep->num, 0);
2240 2241 2242 2243 2244 2245 2246 2247 2248

#ifdef CONFIG_PAGE_POISONING
		/* If we're going to use the generic kernel_map_pages()
		 * poisoning, then it's going to smash the contents of
		 * the redzone and userword anyhow, so switch them off.
		 */
		if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
			flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
#endif
L
Linus Torvalds 已提交
2249 2250 2251 2252
	}

	cachep->colour_off = cache_line_size();
	/* Offset must be a multiple of the alignment. */
2253 2254
	if (cachep->colour_off < cachep->align)
		cachep->colour_off = cachep->align;
P
Pekka Enberg 已提交
2255
	cachep->colour = left_over / cachep->colour_off;
2256
	cachep->freelist_size = freelist_size;
L
Linus Torvalds 已提交
2257
	cachep->flags = flags;
2258
	cachep->allocflags = __GFP_COMP;
2259
	if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
2260
		cachep->allocflags |= GFP_DMA;
2261
	cachep->size = size;
2262
	cachep->reciprocal_buffer_size = reciprocal_value(size);
L
Linus Torvalds 已提交
2263

2264
	if (flags & CFLGS_OFF_SLAB) {
2265
		cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
2266
		/*
2267
		 * This is a possibility for one of the kmalloc_{dma,}_caches.
2268
		 * But since we go off slab only for object size greater than
2269
		 * OFF_SLAB_MIN_SIZE, and kmalloc_{dma,}_caches get created
2270
		 * in ascending order,this should not happen at all.
2271 2272
		 * But leave a BUG_ON for some lucky dude.
		 */
2273
		BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
2274
	}
L
Linus Torvalds 已提交
2275

2276 2277
	err = setup_cpu_cache(cachep, gfp);
	if (err) {
2278
		__kmem_cache_release(cachep);
2279
		return err;
2280
	}
L
Linus Torvalds 已提交
2281

2282
	return 0;
L
Linus Torvalds 已提交
2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
}

#if DEBUG
static void check_irq_off(void)
{
	BUG_ON(!irqs_disabled());
}

static void check_irq_on(void)
{
	BUG_ON(irqs_disabled());
}

2296
static void check_spinlock_acquired(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2297 2298 2299
{
#ifdef CONFIG_SMP
	check_irq_off();
2300
	assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
L
Linus Torvalds 已提交
2301 2302
#endif
}
2303

2304
static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2305 2306 2307
{
#ifdef CONFIG_SMP
	check_irq_off();
2308
	assert_spin_locked(&get_node(cachep, node)->list_lock);
2309 2310 2311
#endif
}

L
Linus Torvalds 已提交
2312 2313 2314 2315
#else
#define check_irq_off()	do { } while(0)
#define check_irq_on()	do { } while(0)
#define check_spinlock_acquired(x) do { } while(0)
2316
#define check_spinlock_acquired_node(x, y) do { } while(0)
L
Linus Torvalds 已提交
2317 2318
#endif

2319
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
2320 2321 2322
			struct array_cache *ac,
			int force, int node);

L
Linus Torvalds 已提交
2323 2324
static void do_drain(void *arg)
{
A
Andrew Morton 已提交
2325
	struct kmem_cache *cachep = arg;
L
Linus Torvalds 已提交
2326
	struct array_cache *ac;
2327
	int node = numa_mem_id();
2328
	struct kmem_cache_node *n;
2329
	LIST_HEAD(list);
L
Linus Torvalds 已提交
2330 2331

	check_irq_off();
2332
	ac = cpu_cache_get(cachep);
2333 2334
	n = get_node(cachep, node);
	spin_lock(&n->list_lock);
2335
	free_block(cachep, ac->entry, ac->avail, node, &list);
2336
	spin_unlock(&n->list_lock);
2337
	slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
2338 2339 2340
	ac->avail = 0;
}

2341
static void drain_cpu_caches(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2342
{
2343
	struct kmem_cache_node *n;
2344 2345
	int node;

2346
	on_each_cpu(do_drain, cachep, 1);
L
Linus Torvalds 已提交
2347
	check_irq_on();
2348 2349
	for_each_kmem_cache_node(cachep, node, n)
		if (n->alien)
2350
			drain_alien_cache(cachep, n->alien);
2351

2352 2353
	for_each_kmem_cache_node(cachep, node, n)
		drain_array(cachep, n, n->shared, 1, node);
L
Linus Torvalds 已提交
2354 2355
}

2356 2357 2358 2359 2360 2361 2362
/*
 * Remove slabs from the list of free slabs.
 * Specify the number of slabs to drain in tofree.
 *
 * Returns the actual number of slabs released.
 */
static int drain_freelist(struct kmem_cache *cache,
2363
			struct kmem_cache_node *n, int tofree)
L
Linus Torvalds 已提交
2364
{
2365 2366
	struct list_head *p;
	int nr_freed;
2367
	struct page *page;
L
Linus Torvalds 已提交
2368

2369
	nr_freed = 0;
2370
	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
L
Linus Torvalds 已提交
2371

2372 2373 2374 2375
		spin_lock_irq(&n->list_lock);
		p = n->slabs_free.prev;
		if (p == &n->slabs_free) {
			spin_unlock_irq(&n->list_lock);
2376 2377
			goto out;
		}
L
Linus Torvalds 已提交
2378

2379
		page = list_entry(p, struct page, lru);
L
Linus Torvalds 已提交
2380
#if DEBUG
2381
		BUG_ON(page->active);
L
Linus Torvalds 已提交
2382
#endif
2383
		list_del(&page->lru);
2384 2385 2386 2387
		/*
		 * Safe to drop the lock. The slab is no longer linked
		 * to the cache.
		 */
2388 2389
		n->free_objects -= cache->num;
		spin_unlock_irq(&n->list_lock);
2390
		slab_destroy(cache, page);
2391
		nr_freed++;
L
Linus Torvalds 已提交
2392
	}
2393 2394
out:
	return nr_freed;
L
Linus Torvalds 已提交
2395 2396
}

2397
int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
2398
{
2399 2400
	int ret = 0;
	int node;
2401
	struct kmem_cache_node *n;
2402 2403 2404 2405

	drain_cpu_caches(cachep);

	check_irq_on();
2406
	for_each_kmem_cache_node(cachep, node, n) {
2407
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
2408

2409 2410
		ret += !list_empty(&n->slabs_full) ||
			!list_empty(&n->slabs_partial);
2411 2412 2413 2414
	}
	return (ret ? 1 : 0);
}

2415
int __kmem_cache_shutdown(struct kmem_cache *cachep)
2416 2417 2418 2419 2420
{
	return __kmem_cache_shrink(cachep, false);
}

void __kmem_cache_release(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2421
{
2422
	int i;
2423
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2424

2425
	free_percpu(cachep->cpu_cache);
L
Linus Torvalds 已提交
2426

2427
	/* NUMA: free the node structures */
2428 2429 2430 2431 2432
	for_each_kmem_cache_node(cachep, i, n) {
		kfree(n->shared);
		free_alien_cache(n->alien);
		kfree(n);
		cachep->node[i] = NULL;
2433
	}
L
Linus Torvalds 已提交
2434 2435
}

2436 2437
/*
 * Get the memory for a slab management obj.
2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
 *
 * For a slab cache when the slab descriptor is off-slab, the
 * slab descriptor can't come from the same cache which is being created,
 * Because if it is the case, that means we defer the creation of
 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
 * And we eventually call down to __kmem_cache_create(), which
 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
 * This is a "chicken-and-egg" problem.
 *
 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
 * which are all initialized during kmem_cache_init().
2449
 */
2450
static void *alloc_slabmgmt(struct kmem_cache *cachep,
2451 2452
				   struct page *page, int colour_off,
				   gfp_t local_flags, int nodeid)
L
Linus Torvalds 已提交
2453
{
2454
	void *freelist;
2455
	void *addr = page_address(page);
P
Pekka Enberg 已提交
2456

L
Linus Torvalds 已提交
2457 2458
	if (OFF_SLAB(cachep)) {
		/* Slab management obj is off-slab. */
2459
		freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2460
					      local_flags, nodeid);
2461
		if (!freelist)
L
Linus Torvalds 已提交
2462 2463
			return NULL;
	} else {
2464 2465
		freelist = addr + colour_off;
		colour_off += cachep->freelist_size;
L
Linus Torvalds 已提交
2466
	}
2467 2468 2469
	page->active = 0;
	page->s_mem = addr + colour_off;
	return freelist;
L
Linus Torvalds 已提交
2470 2471
}

2472
static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
L
Linus Torvalds 已提交
2473
{
2474
	return ((freelist_idx_t *)page->freelist)[idx];
2475 2476 2477
}

static inline void set_free_obj(struct page *page,
2478
					unsigned int idx, freelist_idx_t val)
2479
{
2480
	((freelist_idx_t *)(page->freelist))[idx] = val;
L
Linus Torvalds 已提交
2481 2482
}

2483
static void cache_init_objs(struct kmem_cache *cachep,
2484
			    struct page *page)
L
Linus Torvalds 已提交
2485 2486 2487 2488
{
	int i;

	for (i = 0; i < cachep->num; i++) {
2489
		void *objp = index_to_obj(cachep, page, i);
L
Linus Torvalds 已提交
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
#if DEBUG
		/* need to poison the objs? */
		if (cachep->flags & SLAB_POISON)
			poison_obj(cachep, objp, POISON_FREE);
		if (cachep->flags & SLAB_STORE_USER)
			*dbg_userword(cachep, objp) = NULL;

		if (cachep->flags & SLAB_RED_ZONE) {
			*dbg_redzone1(cachep, objp) = RED_INACTIVE;
			*dbg_redzone2(cachep, objp) = RED_INACTIVE;
		}
		/*
A
Andrew Morton 已提交
2502 2503 2504
		 * Constructors are not allowed to allocate memory from the same
		 * cache which they are a constructor for.  Otherwise, deadlock.
		 * They must also be threaded.
L
Linus Torvalds 已提交
2505 2506
		 */
		if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2507
			cachep->ctor(objp + obj_offset(cachep));
L
Linus Torvalds 已提交
2508 2509 2510 2511

		if (cachep->flags & SLAB_RED_ZONE) {
			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "constructor overwrote the"
P
Pekka Enberg 已提交
2512
					   " end of an object");
L
Linus Torvalds 已提交
2513 2514
			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "constructor overwrote the"
P
Pekka Enberg 已提交
2515
					   " start of an object");
L
Linus Torvalds 已提交
2516
		}
2517
		if ((cachep->size % PAGE_SIZE) == 0 &&
A
Andrew Morton 已提交
2518
			    OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
P
Pekka Enberg 已提交
2519
			kernel_map_pages(virt_to_page(objp),
2520
					 cachep->size / PAGE_SIZE, 0);
L
Linus Torvalds 已提交
2521 2522
#else
		if (cachep->ctor)
2523
			cachep->ctor(objp);
L
Linus Torvalds 已提交
2524
#endif
2525
		set_obj_status(page, i, OBJECT_FREE);
2526
		set_free_obj(page, i, i);
L
Linus Torvalds 已提交
2527 2528 2529
	}
}

2530
static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
2531
{
2532 2533
	if (CONFIG_ZONE_DMA_FLAG) {
		if (flags & GFP_DMA)
2534
			BUG_ON(!(cachep->allocflags & GFP_DMA));
2535
		else
2536
			BUG_ON(cachep->allocflags & GFP_DMA);
2537
	}
L
Linus Torvalds 已提交
2538 2539
}

2540
static void *slab_get_obj(struct kmem_cache *cachep, struct page *page,
A
Andrew Morton 已提交
2541
				int nodeid)
2542
{
2543
	void *objp;
2544

2545
	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2546
	page->active++;
2547
#if DEBUG
J
Joonsoo Kim 已提交
2548
	WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
2549 2550 2551 2552 2553
#endif

	return objp;
}

2554
static void slab_put_obj(struct kmem_cache *cachep, struct page *page,
A
Andrew Morton 已提交
2555
				void *objp, int nodeid)
2556
{
2557
	unsigned int objnr = obj_to_index(cachep, page, objp);
2558
#if DEBUG
J
Joonsoo Kim 已提交
2559
	unsigned int i;
2560

2561
	/* Verify that the slab belongs to the intended node */
J
Joonsoo Kim 已提交
2562
	WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
2563

2564
	/* Verify double free bug */
2565
	for (i = page->active; i < cachep->num; i++) {
2566
		if (get_free_obj(page, i) == objnr) {
2567 2568 2569 2570
			printk(KERN_ERR "slab: double free detected in cache "
					"'%s', objp %p\n", cachep->name, objp);
			BUG();
		}
2571 2572
	}
#endif
2573
	page->active--;
2574
	set_free_obj(page, page->active, objnr);
2575 2576
}

2577 2578 2579
/*
 * Map pages beginning at addr to the given cache and slab. This is required
 * for the slab allocator to be able to lookup the cache and slab of a
2580
 * virtual address for kfree, ksize, and slab debugging.
2581
 */
2582
static void slab_map_pages(struct kmem_cache *cache, struct page *page,
2583
			   void *freelist)
L
Linus Torvalds 已提交
2584
{
2585
	page->slab_cache = cache;
2586
	page->freelist = freelist;
L
Linus Torvalds 已提交
2587 2588 2589 2590 2591 2592
}

/*
 * Grow (by 1) the number of slabs within a cache.  This is called by
 * kmem_cache_alloc() when there are no active objs left in a cache.
 */
2593
static int cache_grow(struct kmem_cache *cachep,
2594
		gfp_t flags, int nodeid, struct page *page)
L
Linus Torvalds 已提交
2595
{
2596
	void *freelist;
P
Pekka Enberg 已提交
2597 2598
	size_t offset;
	gfp_t local_flags;
2599
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2600

A
Andrew Morton 已提交
2601 2602 2603
	/*
	 * Be lazy and only check for valid flags here,  keeping it out of the
	 * critical path in kmem_cache_alloc().
L
Linus Torvalds 已提交
2604
	 */
2605 2606 2607 2608
	if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
		pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
		BUG();
	}
C
Christoph Lameter 已提交
2609
	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
L
Linus Torvalds 已提交
2610

2611
	/* Take the node list lock to change the colour_next on this node */
L
Linus Torvalds 已提交
2612
	check_irq_off();
2613
	n = get_node(cachep, nodeid);
2614
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2615 2616

	/* Get colour for the slab, and cal the next value. */
2617 2618 2619 2620 2621
	offset = n->colour_next;
	n->colour_next++;
	if (n->colour_next >= cachep->colour)
		n->colour_next = 0;
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2622

2623
	offset *= cachep->colour_off;
L
Linus Torvalds 已提交
2624

2625
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2626 2627 2628 2629 2630 2631 2632 2633 2634 2635
		local_irq_enable();

	/*
	 * The test for missing atomic flag is performed here, rather than
	 * the more obvious place, simply to reduce the critical path length
	 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
	 * will eventually be caught here (where it matters).
	 */
	kmem_flagcheck(cachep, flags);

A
Andrew Morton 已提交
2636 2637 2638
	/*
	 * Get mem for the objs.  Attempt to allocate a physical page from
	 * 'nodeid'.
2639
	 */
2640 2641 2642
	if (!page)
		page = kmem_getpages(cachep, local_flags, nodeid);
	if (!page)
L
Linus Torvalds 已提交
2643 2644 2645
		goto failed;

	/* Get slab management. */
2646
	freelist = alloc_slabmgmt(cachep, page, offset,
C
Christoph Lameter 已提交
2647
			local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
2648
	if (!freelist)
L
Linus Torvalds 已提交
2649 2650
		goto opps1;

2651
	slab_map_pages(cachep, page, freelist);
L
Linus Torvalds 已提交
2652

2653
	cache_init_objs(cachep, page);
L
Linus Torvalds 已提交
2654

2655
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2656 2657
		local_irq_disable();
	check_irq_off();
2658
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2659 2660

	/* Make slab active. */
2661
	list_add_tail(&page->lru, &(n->slabs_free));
L
Linus Torvalds 已提交
2662
	STATS_INC_GROWN(cachep);
2663 2664
	n->free_objects += cachep->num;
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2665
	return 1;
A
Andrew Morton 已提交
2666
opps1:
2667
	kmem_freepages(cachep, page);
A
Andrew Morton 已提交
2668
failed:
2669
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
		local_irq_disable();
	return 0;
}

#if DEBUG

/*
 * Perform extra freeing checks:
 * - detect bad pointers.
 * - POISON/RED_ZONE checking
 */
static void kfree_debugcheck(const void *objp)
{
	if (!virt_addr_valid(objp)) {
		printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
P
Pekka Enberg 已提交
2685 2686
		       (unsigned long)objp);
		BUG();
L
Linus Torvalds 已提交
2687 2688 2689
	}
}

2690 2691
static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
{
2692
	unsigned long long redzone1, redzone2;
2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707

	redzone1 = *dbg_redzone1(cache, obj);
	redzone2 = *dbg_redzone2(cache, obj);

	/*
	 * Redzone is ok.
	 */
	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
		return;

	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
		slab_error(cache, "double free detected");
	else
		slab_error(cache, "memory outside object was overwritten");

2708
	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
2709 2710 2711
			obj, redzone1, redzone2);
}

2712
static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
2713
				   unsigned long caller)
L
Linus Torvalds 已提交
2714 2715
{
	unsigned int objnr;
2716
	struct page *page;
L
Linus Torvalds 已提交
2717

2718 2719
	BUG_ON(virt_to_cache(objp) != cachep);

2720
	objp -= obj_offset(cachep);
L
Linus Torvalds 已提交
2721
	kfree_debugcheck(objp);
2722
	page = virt_to_head_page(objp);
L
Linus Torvalds 已提交
2723 2724

	if (cachep->flags & SLAB_RED_ZONE) {
2725
		verify_redzone_free(cachep, objp);
L
Linus Torvalds 已提交
2726 2727 2728 2729
		*dbg_redzone1(cachep, objp) = RED_INACTIVE;
		*dbg_redzone2(cachep, objp) = RED_INACTIVE;
	}
	if (cachep->flags & SLAB_STORE_USER)
2730
		*dbg_userword(cachep, objp) = (void *)caller;
L
Linus Torvalds 已提交
2731

2732
	objnr = obj_to_index(cachep, page, objp);
L
Linus Torvalds 已提交
2733 2734

	BUG_ON(objnr >= cachep->num);
2735
	BUG_ON(objp != index_to_obj(cachep, page, objnr));
L
Linus Torvalds 已提交
2736

2737
	set_obj_status(page, objnr, OBJECT_FREE);
L
Linus Torvalds 已提交
2738 2739
	if (cachep->flags & SLAB_POISON) {
#ifdef CONFIG_DEBUG_PAGEALLOC
2740
		if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
2741
			store_stackinfo(cachep, objp, caller);
P
Pekka Enberg 已提交
2742
			kernel_map_pages(virt_to_page(objp),
2743
					 cachep->size / PAGE_SIZE, 0);
L
Linus Torvalds 已提交
2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758
		} else {
			poison_obj(cachep, objp, POISON_FREE);
		}
#else
		poison_obj(cachep, objp, POISON_FREE);
#endif
	}
	return objp;
}

#else
#define kfree_debugcheck(x) do { } while(0)
#define cache_free_debugcheck(x,objp,z) (objp)
#endif

2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773
static struct page *get_first_slab(struct kmem_cache_node *n)
{
	struct page *page;

	page = list_first_entry_or_null(&n->slabs_partial,
			struct page, lru);
	if (!page) {
		n->free_touched = 1;
		page = list_first_entry_or_null(&n->slabs_free,
				struct page, lru);
	}

	return page;
}

2774 2775
static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
							bool force_refill)
L
Linus Torvalds 已提交
2776 2777
{
	int batchcount;
2778
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2779
	struct array_cache *ac;
P
Pekka Enberg 已提交
2780 2781
	int node;

L
Linus Torvalds 已提交
2782
	check_irq_off();
2783
	node = numa_mem_id();
2784 2785 2786
	if (unlikely(force_refill))
		goto force_grow;
retry:
2787
	ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
2788 2789
	batchcount = ac->batchcount;
	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
A
Andrew Morton 已提交
2790 2791 2792 2793
		/*
		 * If there was little recent activity on this cache, then
		 * perform only a partial refill.  Otherwise we could generate
		 * refill bouncing.
L
Linus Torvalds 已提交
2794 2795 2796
		 */
		batchcount = BATCHREFILL_LIMIT;
	}
2797
	n = get_node(cachep, node);
2798

2799 2800
	BUG_ON(ac->avail > 0 || !n);
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2801

2802
	/* See if we can refill from the shared array */
2803 2804
	if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
		n->shared->touched = 1;
2805
		goto alloc_done;
2806
	}
2807

L
Linus Torvalds 已提交
2808
	while (batchcount > 0) {
2809
		struct page *page;
L
Linus Torvalds 已提交
2810
		/* Get slab alloc is to come from. */
2811 2812 2813
		page = get_first_slab(n);
		if (!page)
			goto must_grow;
L
Linus Torvalds 已提交
2814 2815

		check_spinlock_acquired(cachep);
2816 2817 2818 2819 2820 2821

		/*
		 * The slab was either on partial or free list so
		 * there must be at least one object available for
		 * allocation.
		 */
2822
		BUG_ON(page->active >= cachep->num);
2823

2824
		while (page->active < cachep->num && batchcount--) {
L
Linus Torvalds 已提交
2825 2826 2827 2828
			STATS_INC_ALLOCED(cachep);
			STATS_INC_ACTIVE(cachep);
			STATS_SET_HIGH(cachep);

2829
			ac_put_obj(cachep, ac, slab_get_obj(cachep, page,
2830
									node));
L
Linus Torvalds 已提交
2831 2832 2833
		}

		/* move slabp to correct slabp list: */
2834 2835
		list_del(&page->lru);
		if (page->active == cachep->num)
2836
			list_add(&page->lru, &n->slabs_full);
L
Linus Torvalds 已提交
2837
		else
2838
			list_add(&page->lru, &n->slabs_partial);
L
Linus Torvalds 已提交
2839 2840
	}

A
Andrew Morton 已提交
2841
must_grow:
2842
	n->free_objects -= ac->avail;
A
Andrew Morton 已提交
2843
alloc_done:
2844
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2845 2846 2847

	if (unlikely(!ac->avail)) {
		int x;
2848
force_grow:
D
David Rientjes 已提交
2849
		x = cache_grow(cachep, gfp_exact_node(flags), node, NULL);
2850

A
Andrew Morton 已提交
2851
		/* cache_grow can reenable interrupts, then ac could change. */
2852
		ac = cpu_cache_get(cachep);
2853
		node = numa_mem_id();
2854 2855 2856

		/* no objects in sight? abort */
		if (!x && (ac->avail == 0 || force_refill))
L
Linus Torvalds 已提交
2857 2858
			return NULL;

A
Andrew Morton 已提交
2859
		if (!ac->avail)		/* objects refilled by interrupt? */
L
Linus Torvalds 已提交
2860 2861 2862
			goto retry;
	}
	ac->touched = 1;
2863 2864

	return ac_get_obj(cachep, ac, flags, force_refill);
L
Linus Torvalds 已提交
2865 2866
}

A
Andrew Morton 已提交
2867 2868
static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
						gfp_t flags)
L
Linus Torvalds 已提交
2869
{
2870
	might_sleep_if(gfpflags_allow_blocking(flags));
L
Linus Torvalds 已提交
2871 2872 2873 2874 2875 2876
#if DEBUG
	kmem_flagcheck(cachep, flags);
#endif
}

#if DEBUG
A
Andrew Morton 已提交
2877
static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
2878
				gfp_t flags, void *objp, unsigned long caller)
L
Linus Torvalds 已提交
2879
{
2880 2881
	struct page *page;

P
Pekka Enberg 已提交
2882
	if (!objp)
L
Linus Torvalds 已提交
2883
		return objp;
P
Pekka Enberg 已提交
2884
	if (cachep->flags & SLAB_POISON) {
L
Linus Torvalds 已提交
2885
#ifdef CONFIG_DEBUG_PAGEALLOC
2886
		if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
P
Pekka Enberg 已提交
2887
			kernel_map_pages(virt_to_page(objp),
2888
					 cachep->size / PAGE_SIZE, 1);
L
Linus Torvalds 已提交
2889 2890 2891 2892 2893 2894 2895 2896
		else
			check_poison_obj(cachep, objp);
#else
		check_poison_obj(cachep, objp);
#endif
		poison_obj(cachep, objp, POISON_INUSE);
	}
	if (cachep->flags & SLAB_STORE_USER)
2897
		*dbg_userword(cachep, objp) = (void *)caller;
L
Linus Torvalds 已提交
2898 2899

	if (cachep->flags & SLAB_RED_ZONE) {
A
Andrew Morton 已提交
2900 2901 2902 2903
		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
			slab_error(cachep, "double free, or memory outside"
						" object was overwritten");
P
Pekka Enberg 已提交
2904
			printk(KERN_ERR
2905
				"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
A
Andrew Morton 已提交
2906 2907
				objp, *dbg_redzone1(cachep, objp),
				*dbg_redzone2(cachep, objp));
L
Linus Torvalds 已提交
2908 2909 2910 2911
		}
		*dbg_redzone1(cachep, objp) = RED_ACTIVE;
		*dbg_redzone2(cachep, objp) = RED_ACTIVE;
	}
2912 2913 2914

	page = virt_to_head_page(objp);
	set_obj_status(page, obj_to_index(cachep, page, objp), OBJECT_ACTIVE);
2915
	objp += obj_offset(cachep);
2916
	if (cachep->ctor && cachep->flags & SLAB_POISON)
2917
		cachep->ctor(objp);
T
Tetsuo Handa 已提交
2918 2919
	if (ARCH_SLAB_MINALIGN &&
	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
2920
		printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
H
Hugh Dickins 已提交
2921
		       objp, (int)ARCH_SLAB_MINALIGN);
2922
	}
L
Linus Torvalds 已提交
2923 2924 2925 2926 2927 2928
	return objp;
}
#else
#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
#endif

2929
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
2930
{
P
Pekka Enberg 已提交
2931
	void *objp;
L
Linus Torvalds 已提交
2932
	struct array_cache *ac;
2933
	bool force_refill = false;
L
Linus Torvalds 已提交
2934

2935
	check_irq_off();
2936

2937
	ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
2938 2939
	if (likely(ac->avail)) {
		ac->touched = 1;
2940 2941
		objp = ac_get_obj(cachep, ac, flags, false);

2942
		/*
2943 2944
		 * Allow for the possibility all avail objects are not allowed
		 * by the current flags
2945
		 */
2946 2947 2948 2949 2950
		if (objp) {
			STATS_INC_ALLOCHIT(cachep);
			goto out;
		}
		force_refill = true;
L
Linus Torvalds 已提交
2951
	}
2952 2953 2954 2955 2956 2957 2958 2959 2960 2961

	STATS_INC_ALLOCMISS(cachep);
	objp = cache_alloc_refill(cachep, flags, force_refill);
	/*
	 * the 'ac' may be updated by cache_alloc_refill(),
	 * and kmemleak_erase() requires its correct value.
	 */
	ac = cpu_cache_get(cachep);

out:
2962 2963 2964 2965 2966
	/*
	 * To avoid a false negative, if an object that is in one of the
	 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
	 * treat the array pointers as a reference to the object.
	 */
2967 2968
	if (objp)
		kmemleak_erase(&ac->entry[ac->avail]);
2969 2970 2971
	return objp;
}

2972
#ifdef CONFIG_NUMA
2973
/*
2974
 * Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
2975 2976 2977 2978 2979 2980 2981 2982
 *
 * If we are in_interrupt, then process context, including cpusets and
 * mempolicy, may not apply and should not be used for allocation policy.
 */
static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
{
	int nid_alloc, nid_here;

2983
	if (in_interrupt() || (flags & __GFP_THISNODE))
2984
		return NULL;
2985
	nid_alloc = nid_here = numa_mem_id();
2986
	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
2987
		nid_alloc = cpuset_slab_spread_node();
2988
	else if (current->mempolicy)
2989
		nid_alloc = mempolicy_slab_node();
2990
	if (nid_alloc != nid_here)
2991
		return ____cache_alloc_node(cachep, flags, nid_alloc);
2992 2993 2994
	return NULL;
}

2995 2996
/*
 * Fallback function if there was no memory available and no objects on a
2997
 * certain node and fall back is permitted. First we scan all the
2998
 * available node for available objects. If that fails then we
2999 3000 3001
 * perform an allocation without specifying a node. This allows the page
 * allocator to do its reclaim / fallback magic. We then insert the
 * slab into the proper nodelist and then allocate from it.
3002
 */
3003
static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
3004
{
3005 3006
	struct zonelist *zonelist;
	gfp_t local_flags;
3007
	struct zoneref *z;
3008 3009
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
3010
	void *obj = NULL;
3011
	int nid;
3012
	unsigned int cpuset_mems_cookie;
3013 3014 3015 3016

	if (flags & __GFP_THISNODE)
		return NULL;

C
Christoph Lameter 已提交
3017
	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
3018

3019
retry_cpuset:
3020
	cpuset_mems_cookie = read_mems_allowed_begin();
3021
	zonelist = node_zonelist(mempolicy_slab_node(), flags);
3022

3023 3024 3025 3026 3027
retry:
	/*
	 * Look through allowed nodes for objects available
	 * from existing per node queues.
	 */
3028 3029
	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
		nid = zone_to_nid(zone);
3030

3031
		if (cpuset_zone_allowed(zone, flags) &&
3032 3033
			get_node(cache, nid) &&
			get_node(cache, nid)->free_objects) {
3034
				obj = ____cache_alloc_node(cache,
D
David Rientjes 已提交
3035
					gfp_exact_node(flags), nid);
3036 3037 3038
				if (obj)
					break;
		}
3039 3040
	}

3041
	if (!obj) {
3042 3043 3044 3045 3046 3047
		/*
		 * This allocation will be performed within the constraints
		 * of the current cpuset / memory policy requirements.
		 * We may trigger various forms of reclaim on the allowed
		 * set and go into memory reserves if necessary.
		 */
3048 3049
		struct page *page;

3050
		if (gfpflags_allow_blocking(local_flags))
3051 3052
			local_irq_enable();
		kmem_flagcheck(cache, flags);
3053
		page = kmem_getpages(cache, local_flags, numa_mem_id());
3054
		if (gfpflags_allow_blocking(local_flags))
3055
			local_irq_disable();
3056
		if (page) {
3057 3058 3059
			/*
			 * Insert into the appropriate per node queues
			 */
3060 3061
			nid = page_to_nid(page);
			if (cache_grow(cache, flags, nid, page)) {
3062
				obj = ____cache_alloc_node(cache,
D
David Rientjes 已提交
3063
					gfp_exact_node(flags), nid);
3064 3065 3066 3067 3068 3069 3070 3071
				if (!obj)
					/*
					 * Another processor may allocate the
					 * objects in the slab since we are
					 * not holding any locks.
					 */
					goto retry;
			} else {
3072
				/* cache_grow already freed obj */
3073 3074 3075
				obj = NULL;
			}
		}
3076
	}
3077

3078
	if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
3079
		goto retry_cpuset;
3080 3081 3082
	return obj;
}

3083 3084
/*
 * A interface to enable slab creation on nodeid
L
Linus Torvalds 已提交
3085
 */
3086
static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
A
Andrew Morton 已提交
3087
				int nodeid)
3088
{
3089
	struct page *page;
3090
	struct kmem_cache_node *n;
P
Pekka Enberg 已提交
3091 3092 3093
	void *obj;
	int x;

3094
	VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
3095
	n = get_node(cachep, nodeid);
3096
	BUG_ON(!n);
P
Pekka Enberg 已提交
3097

A
Andrew Morton 已提交
3098
retry:
3099
	check_irq_off();
3100
	spin_lock(&n->list_lock);
3101 3102 3103
	page = get_first_slab(n);
	if (!page)
		goto must_grow;
P
Pekka Enberg 已提交
3104 3105 3106 3107 3108 3109 3110

	check_spinlock_acquired_node(cachep, nodeid);

	STATS_INC_NODEALLOCS(cachep);
	STATS_INC_ACTIVE(cachep);
	STATS_SET_HIGH(cachep);

3111
	BUG_ON(page->active == cachep->num);
P
Pekka Enberg 已提交
3112

3113
	obj = slab_get_obj(cachep, page, nodeid);
3114
	n->free_objects--;
P
Pekka Enberg 已提交
3115
	/* move slabp to correct slabp list: */
3116
	list_del(&page->lru);
P
Pekka Enberg 已提交
3117

3118 3119
	if (page->active == cachep->num)
		list_add(&page->lru, &n->slabs_full);
A
Andrew Morton 已提交
3120
	else
3121
		list_add(&page->lru, &n->slabs_partial);
3122

3123
	spin_unlock(&n->list_lock);
P
Pekka Enberg 已提交
3124
	goto done;
3125

A
Andrew Morton 已提交
3126
must_grow:
3127
	spin_unlock(&n->list_lock);
D
David Rientjes 已提交
3128
	x = cache_grow(cachep, gfp_exact_node(flags), nodeid, NULL);
3129 3130
	if (x)
		goto retry;
L
Linus Torvalds 已提交
3131

3132
	return fallback_alloc(cachep, flags);
3133

A
Andrew Morton 已提交
3134
done:
P
Pekka Enberg 已提交
3135
	return obj;
3136
}
3137 3138

static __always_inline void *
3139
slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3140
		   unsigned long caller)
3141 3142 3143
{
	unsigned long save_flags;
	void *ptr;
3144
	int slab_node = numa_mem_id();
3145

3146
	flags &= gfp_allowed_mask;
3147 3148
	cachep = slab_pre_alloc_hook(cachep, flags);
	if (unlikely(!cachep))
3149 3150
		return NULL;

3151 3152 3153
	cache_alloc_debugcheck_before(cachep, flags);
	local_irq_save(save_flags);

A
Andrew Morton 已提交
3154
	if (nodeid == NUMA_NO_NODE)
3155
		nodeid = slab_node;
3156

3157
	if (unlikely(!get_node(cachep, nodeid))) {
3158 3159 3160 3161 3162
		/* Node not bootstrapped yet */
		ptr = fallback_alloc(cachep, flags);
		goto out;
	}

3163
	if (nodeid == slab_node) {
3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
		/*
		 * Use the locally cached objects if possible.
		 * However ____cache_alloc does not allow fallback
		 * to other nodes. It may fail while we still have
		 * objects on other nodes available.
		 */
		ptr = ____cache_alloc(cachep, flags);
		if (ptr)
			goto out;
	}
	/* ___cache_alloc_node can fall back to other nodes */
	ptr = ____cache_alloc_node(cachep, flags, nodeid);
  out:
	local_irq_restore(save_flags);
	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);

3180 3181
	if (unlikely(flags & __GFP_ZERO) && ptr)
		memset(ptr, 0, cachep->object_size);
3182

3183
	slab_post_alloc_hook(cachep, flags, 1, &ptr);
3184 3185 3186 3187 3188 3189 3190 3191
	return ptr;
}

static __always_inline void *
__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
{
	void *objp;

3192
	if (current->mempolicy || cpuset_do_slab_mem_spread()) {
3193 3194 3195 3196 3197 3198 3199 3200 3201 3202
		objp = alternate_node_alloc(cache, flags);
		if (objp)
			goto out;
	}
	objp = ____cache_alloc(cache, flags);

	/*
	 * We may just have run out of memory on the local node.
	 * ____cache_alloc_node() knows how to locate memory on other nodes
	 */
3203 3204
	if (!objp)
		objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219

  out:
	return objp;
}
#else

static __always_inline void *
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
{
	return ____cache_alloc(cachep, flags);
}

#endif /* CONFIG_NUMA */

static __always_inline void *
3220
slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3221 3222 3223 3224
{
	unsigned long save_flags;
	void *objp;

3225
	flags &= gfp_allowed_mask;
3226 3227
	cachep = slab_pre_alloc_hook(cachep, flags);
	if (unlikely(!cachep))
3228 3229
		return NULL;

3230 3231 3232 3233 3234 3235 3236
	cache_alloc_debugcheck_before(cachep, flags);
	local_irq_save(save_flags);
	objp = __do_cache_alloc(cachep, flags);
	local_irq_restore(save_flags);
	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
	prefetchw(objp);

3237 3238
	if (unlikely(flags & __GFP_ZERO) && objp)
		memset(objp, 0, cachep->object_size);
3239

3240
	slab_post_alloc_hook(cachep, flags, 1, &objp);
3241 3242
	return objp;
}
3243 3244

/*
3245
 * Caller needs to acquire correct kmem_cache_node's list_lock
3246
 * @list: List of detached free slabs should be freed by caller
3247
 */
3248 3249
static void free_block(struct kmem_cache *cachep, void **objpp,
			int nr_objects, int node, struct list_head *list)
L
Linus Torvalds 已提交
3250 3251
{
	int i;
3252
	struct kmem_cache_node *n = get_node(cachep, node);
L
Linus Torvalds 已提交
3253 3254

	for (i = 0; i < nr_objects; i++) {
3255
		void *objp;
3256
		struct page *page;
L
Linus Torvalds 已提交
3257

3258 3259 3260
		clear_obj_pfmemalloc(&objpp[i]);
		objp = objpp[i];

3261 3262
		page = virt_to_head_page(objp);
		list_del(&page->lru);
3263
		check_spinlock_acquired_node(cachep, node);
3264
		slab_put_obj(cachep, page, objp, node);
L
Linus Torvalds 已提交
3265
		STATS_DEC_ACTIVE(cachep);
3266
		n->free_objects++;
L
Linus Torvalds 已提交
3267 3268

		/* fixup slab chains */
3269
		if (page->active == 0) {
3270 3271
			if (n->free_objects > n->free_limit) {
				n->free_objects -= cachep->num;
3272
				list_add_tail(&page->lru, list);
L
Linus Torvalds 已提交
3273
			} else {
3274
				list_add(&page->lru, &n->slabs_free);
L
Linus Torvalds 已提交
3275 3276 3277 3278 3279 3280
			}
		} else {
			/* Unconditionally move a slab to the end of the
			 * partial list on free - maximum time for the
			 * other objects to be freed, too.
			 */
3281
			list_add_tail(&page->lru, &n->slabs_partial);
L
Linus Torvalds 已提交
3282 3283 3284 3285
		}
	}
}

3286
static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
L
Linus Torvalds 已提交
3287 3288
{
	int batchcount;
3289
	struct kmem_cache_node *n;
3290
	int node = numa_mem_id();
3291
	LIST_HEAD(list);
L
Linus Torvalds 已提交
3292 3293 3294 3295 3296 3297

	batchcount = ac->batchcount;
#if DEBUG
	BUG_ON(!batchcount || batchcount > ac->avail);
#endif
	check_irq_off();
3298
	n = get_node(cachep, node);
3299 3300 3301
	spin_lock(&n->list_lock);
	if (n->shared) {
		struct array_cache *shared_array = n->shared;
P
Pekka Enberg 已提交
3302
		int max = shared_array->limit - shared_array->avail;
L
Linus Torvalds 已提交
3303 3304 3305
		if (max) {
			if (batchcount > max)
				batchcount = max;
3306
			memcpy(&(shared_array->entry[shared_array->avail]),
P
Pekka Enberg 已提交
3307
			       ac->entry, sizeof(void *) * batchcount);
L
Linus Torvalds 已提交
3308 3309 3310 3311 3312
			shared_array->avail += batchcount;
			goto free_done;
		}
	}

3313
	free_block(cachep, ac->entry, batchcount, node, &list);
A
Andrew Morton 已提交
3314
free_done:
L
Linus Torvalds 已提交
3315 3316 3317
#if STATS
	{
		int i = 0;
3318
		struct page *page;
L
Linus Torvalds 已提交
3319

3320
		list_for_each_entry(page, &n->slabs_free, lru) {
3321
			BUG_ON(page->active);
L
Linus Torvalds 已提交
3322 3323 3324 3325 3326 3327

			i++;
		}
		STATS_SET_FREEABLE(cachep, i);
	}
#endif
3328
	spin_unlock(&n->list_lock);
3329
	slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3330
	ac->avail -= batchcount;
A
Andrew Morton 已提交
3331
	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
L
Linus Torvalds 已提交
3332 3333 3334
}

/*
A
Andrew Morton 已提交
3335 3336
 * Release an obj back to its cache. If the obj has a constructed state, it must
 * be in this state _before_ it is released.  Called with disabled ints.
L
Linus Torvalds 已提交
3337
 */
3338
static inline void __cache_free(struct kmem_cache *cachep, void *objp,
3339
				unsigned long caller)
L
Linus Torvalds 已提交
3340
{
3341
	struct array_cache *ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
3342 3343

	check_irq_off();
3344
	kmemleak_free_recursive(objp, cachep->flags);
3345
	objp = cache_free_debugcheck(cachep, objp, caller);
L
Linus Torvalds 已提交
3346

3347
	kmemcheck_slab_free(cachep, objp, cachep->object_size);
P
Pekka Enberg 已提交
3348

3349 3350 3351 3352 3353 3354 3355
	/*
	 * Skip calling cache_free_alien() when the platform is not numa.
	 * This will avoid cache misses that happen while accessing slabp (which
	 * is per page memory  reference) to get nodeid. Instead use a global
	 * variable to skip the call, which is mostly likely to be present in
	 * the cache.
	 */
3356
	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3357 3358
		return;

3359
	if (ac->avail < ac->limit) {
L
Linus Torvalds 已提交
3360 3361 3362 3363 3364
		STATS_INC_FREEHIT(cachep);
	} else {
		STATS_INC_FREEMISS(cachep);
		cache_flusharray(cachep, ac);
	}
Z
Zhao Jin 已提交
3365

3366
	ac_put_obj(cachep, ac, objp);
L
Linus Torvalds 已提交
3367 3368 3369 3370 3371 3372 3373 3374 3375 3376
}

/**
 * kmem_cache_alloc - Allocate an object
 * @cachep: The cache to allocate from.
 * @flags: See kmalloc().
 *
 * Allocate an object from this cache.  The flags are only relevant
 * if the cache has no available objects.
 */
3377
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
3378
{
3379
	void *ret = slab_alloc(cachep, flags, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3380

3381
	trace_kmem_cache_alloc(_RET_IP_, ret,
3382
			       cachep->object_size, cachep->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3383 3384

	return ret;
L
Linus Torvalds 已提交
3385 3386 3387
}
EXPORT_SYMBOL(kmem_cache_alloc);

3388 3389 3390 3391 3392 3393
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
{
	__kmem_cache_free_bulk(s, size, p);
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

3394
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3395 3396 3397 3398 3399 3400
								void **p)
{
	return __kmem_cache_alloc_bulk(s, flags, size, p);
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk);

3401
#ifdef CONFIG_TRACING
3402
void *
3403
kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
E
Eduard - Gabriel Munteanu 已提交
3404
{
3405 3406
	void *ret;

3407
	ret = slab_alloc(cachep, flags, _RET_IP_);
3408 3409

	trace_kmalloc(_RET_IP_, ret,
3410
		      size, cachep->size, flags);
3411
	return ret;
E
Eduard - Gabriel Munteanu 已提交
3412
}
3413
EXPORT_SYMBOL(kmem_cache_alloc_trace);
E
Eduard - Gabriel Munteanu 已提交
3414 3415
#endif

L
Linus Torvalds 已提交
3416
#ifdef CONFIG_NUMA
3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427
/**
 * kmem_cache_alloc_node - Allocate an object on the specified node
 * @cachep: The cache to allocate from.
 * @flags: See kmalloc().
 * @nodeid: node number of the target node.
 *
 * Identical to kmem_cache_alloc but it will allocate memory on the given
 * node, which can improve the performance for cpu bound structures.
 *
 * Fallback to other node is possible if __GFP_THISNODE is not set.
 */
3428 3429
void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
{
3430
	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3431

3432
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3433
				    cachep->object_size, cachep->size,
3434
				    flags, nodeid);
E
Eduard - Gabriel Munteanu 已提交
3435 3436

	return ret;
3437
}
L
Linus Torvalds 已提交
3438 3439
EXPORT_SYMBOL(kmem_cache_alloc_node);

3440
#ifdef CONFIG_TRACING
3441
void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
3442
				  gfp_t flags,
3443 3444
				  int nodeid,
				  size_t size)
E
Eduard - Gabriel Munteanu 已提交
3445
{
3446 3447
	void *ret;

3448
	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3449

3450
	trace_kmalloc_node(_RET_IP_, ret,
3451
			   size, cachep->size,
3452 3453
			   flags, nodeid);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
3454
}
3455
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
E
Eduard - Gabriel Munteanu 已提交
3456 3457
#endif

3458
static __always_inline void *
3459
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3460
{
3461
	struct kmem_cache *cachep;
3462

3463
	cachep = kmalloc_slab(size, flags);
3464 3465
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
		return cachep;
3466
	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
3467
}
3468 3469 3470

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
3471
	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3472
}
3473
EXPORT_SYMBOL(__kmalloc_node);
3474 3475

void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3476
		int node, unsigned long caller)
3477
{
3478
	return __do_kmalloc_node(size, flags, node, caller);
3479 3480 3481
}
EXPORT_SYMBOL(__kmalloc_node_track_caller);
#endif /* CONFIG_NUMA */
L
Linus Torvalds 已提交
3482 3483

/**
3484
 * __do_kmalloc - allocate memory
L
Linus Torvalds 已提交
3485
 * @size: how many bytes of memory are required.
3486
 * @flags: the type of memory to allocate (see kmalloc).
3487
 * @caller: function caller for debug tracking of the caller
L
Linus Torvalds 已提交
3488
 */
3489
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3490
					  unsigned long caller)
L
Linus Torvalds 已提交
3491
{
3492
	struct kmem_cache *cachep;
E
Eduard - Gabriel Munteanu 已提交
3493
	void *ret;
L
Linus Torvalds 已提交
3494

3495
	cachep = kmalloc_slab(size, flags);
3496 3497
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
		return cachep;
3498
	ret = slab_alloc(cachep, flags, caller);
E
Eduard - Gabriel Munteanu 已提交
3499

3500
	trace_kmalloc(caller, ret,
3501
		      size, cachep->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3502 3503

	return ret;
3504 3505 3506 3507
}

void *__kmalloc(size_t size, gfp_t flags)
{
3508
	return __do_kmalloc(size, flags, _RET_IP_);
L
Linus Torvalds 已提交
3509 3510 3511
}
EXPORT_SYMBOL(__kmalloc);

3512
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3513
{
3514
	return __do_kmalloc(size, flags, caller);
3515 3516
}
EXPORT_SYMBOL(__kmalloc_track_caller);
3517

L
Linus Torvalds 已提交
3518 3519 3520 3521 3522 3523 3524 3525
/**
 * kmem_cache_free - Deallocate an object
 * @cachep: The cache the allocation was from.
 * @objp: The previously allocated object.
 *
 * Free an object which was previously allocated from this
 * cache.
 */
3526
void kmem_cache_free(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
3527 3528
{
	unsigned long flags;
3529 3530 3531
	cachep = cache_from_obj(cachep, objp);
	if (!cachep)
		return;
L
Linus Torvalds 已提交
3532 3533

	local_irq_save(flags);
3534
	debug_check_no_locks_freed(objp, cachep->object_size);
3535
	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3536
		debug_check_no_obj_freed(objp, cachep->object_size);
3537
	__cache_free(cachep, objp, _RET_IP_);
L
Linus Torvalds 已提交
3538
	local_irq_restore(flags);
E
Eduard - Gabriel Munteanu 已提交
3539

3540
	trace_kmem_cache_free(_RET_IP_, objp);
L
Linus Torvalds 已提交
3541 3542 3543 3544 3545 3546 3547
}
EXPORT_SYMBOL(kmem_cache_free);

/**
 * kfree - free previously allocated memory
 * @objp: pointer returned by kmalloc.
 *
3548 3549
 * If @objp is NULL, no operation is performed.
 *
L
Linus Torvalds 已提交
3550 3551 3552 3553 3554
 * Don't free memory not originally allocated by kmalloc()
 * or you will run into trouble.
 */
void kfree(const void *objp)
{
3555
	struct kmem_cache *c;
L
Linus Torvalds 已提交
3556 3557
	unsigned long flags;

3558 3559
	trace_kfree(_RET_IP_, objp);

3560
	if (unlikely(ZERO_OR_NULL_PTR(objp)))
L
Linus Torvalds 已提交
3561 3562 3563
		return;
	local_irq_save(flags);
	kfree_debugcheck(objp);
3564
	c = virt_to_cache(objp);
3565 3566 3567
	debug_check_no_locks_freed(objp, c->object_size);

	debug_check_no_obj_freed(objp, c->object_size);
3568
	__cache_free(c, (void *)objp, _RET_IP_);
L
Linus Torvalds 已提交
3569 3570 3571 3572
	local_irq_restore(flags);
}
EXPORT_SYMBOL(kfree);

3573
/*
3574
 * This initializes kmem_cache_node or resizes various caches for all nodes.
3575
 */
3576
static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp)
3577 3578
{
	int node;
3579
	struct kmem_cache_node *n;
3580
	struct array_cache *new_shared;
J
Joonsoo Kim 已提交
3581
	struct alien_cache **new_alien = NULL;
3582

3583
	for_each_online_node(node) {
3584

3585 3586 3587 3588 3589
		if (use_alien_caches) {
			new_alien = alloc_alien_cache(node, cachep->limit, gfp);
			if (!new_alien)
				goto fail;
		}
3590

3591 3592 3593
		new_shared = NULL;
		if (cachep->shared) {
			new_shared = alloc_arraycache(node,
3594
				cachep->shared*cachep->batchcount,
3595
					0xbaadf00d, gfp);
3596 3597 3598 3599
			if (!new_shared) {
				free_alien_cache(new_alien);
				goto fail;
			}
3600
		}
3601

3602
		n = get_node(cachep, node);
3603 3604
		if (n) {
			struct array_cache *shared = n->shared;
3605
			LIST_HEAD(list);
3606

3607
			spin_lock_irq(&n->list_lock);
3608

3609
			if (shared)
3610
				free_block(cachep, shared->entry,
3611
						shared->avail, node, &list);
3612

3613 3614 3615
			n->shared = new_shared;
			if (!n->alien) {
				n->alien = new_alien;
3616 3617
				new_alien = NULL;
			}
3618
			n->free_limit = (1 + nr_cpus_node(node)) *
A
Andrew Morton 已提交
3619
					cachep->batchcount + cachep->num;
3620
			spin_unlock_irq(&n->list_lock);
3621
			slabs_destroy(cachep, &list);
3622
			kfree(shared);
3623 3624 3625
			free_alien_cache(new_alien);
			continue;
		}
3626 3627
		n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
		if (!n) {
3628 3629
			free_alien_cache(new_alien);
			kfree(new_shared);
3630
			goto fail;
3631
		}
3632

3633
		kmem_cache_node_init(n);
3634 3635
		n->next_reap = jiffies + REAPTIMEOUT_NODE +
				((unsigned long)cachep) % REAPTIMEOUT_NODE;
3636 3637 3638
		n->shared = new_shared;
		n->alien = new_alien;
		n->free_limit = (1 + nr_cpus_node(node)) *
A
Andrew Morton 已提交
3639
					cachep->batchcount + cachep->num;
3640
		cachep->node[node] = n;
3641
	}
3642
	return 0;
3643

A
Andrew Morton 已提交
3644
fail:
3645
	if (!cachep->list.next) {
3646 3647 3648
		/* Cache is not active yet. Roll back what we did */
		node--;
		while (node >= 0) {
3649 3650
			n = get_node(cachep, node);
			if (n) {
3651 3652 3653
				kfree(n->shared);
				free_alien_cache(n->alien);
				kfree(n);
3654
				cachep->node[node] = NULL;
3655 3656 3657 3658
			}
			node--;
		}
	}
3659
	return -ENOMEM;
3660 3661
}

3662
/* Always called with the slab_mutex held */
G
Glauber Costa 已提交
3663
static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3664
				int batchcount, int shared, gfp_t gfp)
L
Linus Torvalds 已提交
3665
{
3666 3667
	struct array_cache __percpu *cpu_cache, *prev;
	int cpu;
L
Linus Torvalds 已提交
3668

3669 3670
	cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
	if (!cpu_cache)
3671 3672
		return -ENOMEM;

3673 3674 3675
	prev = cachep->cpu_cache;
	cachep->cpu_cache = cpu_cache;
	kick_all_cpus_sync();
3676

L
Linus Torvalds 已提交
3677 3678 3679
	check_irq_on();
	cachep->batchcount = batchcount;
	cachep->limit = limit;
3680
	cachep->shared = shared;
L
Linus Torvalds 已提交
3681

3682 3683 3684 3685
	if (!prev)
		goto alloc_node;

	for_each_online_cpu(cpu) {
3686
		LIST_HEAD(list);
3687 3688
		int node;
		struct kmem_cache_node *n;
3689
		struct array_cache *ac = per_cpu_ptr(prev, cpu);
3690

3691
		node = cpu_to_mem(cpu);
3692 3693
		n = get_node(cachep, node);
		spin_lock_irq(&n->list_lock);
3694
		free_block(cachep, ac->entry, ac->avail, node, &list);
3695
		spin_unlock_irq(&n->list_lock);
3696
		slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3697
	}
3698 3699 3700
	free_percpu(prev);

alloc_node:
3701
	return alloc_kmem_cache_node(cachep, gfp);
L
Linus Torvalds 已提交
3702 3703
}

G
Glauber Costa 已提交
3704 3705 3706 3707
static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
				int batchcount, int shared, gfp_t gfp)
{
	int ret;
3708
	struct kmem_cache *c;
G
Glauber Costa 已提交
3709 3710 3711 3712 3713 3714 3715 3716 3717

	ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);

	if (slab_state < FULL)
		return ret;

	if ((ret < 0) || !is_root_cache(cachep))
		return ret;

3718 3719 3720 3721
	lockdep_assert_held(&slab_mutex);
	for_each_memcg_cache(c, cachep) {
		/* return value determined by the root cache only */
		__do_tune_cpucache(c, limit, batchcount, shared, gfp);
G
Glauber Costa 已提交
3722 3723 3724 3725 3726
	}

	return ret;
}

3727
/* Called with slab_mutex held always */
3728
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
L
Linus Torvalds 已提交
3729 3730
{
	int err;
G
Glauber Costa 已提交
3731 3732 3733 3734 3735 3736 3737 3738 3739 3740
	int limit = 0;
	int shared = 0;
	int batchcount = 0;

	if (!is_root_cache(cachep)) {
		struct kmem_cache *root = memcg_root_cache(cachep);
		limit = root->limit;
		shared = root->shared;
		batchcount = root->batchcount;
	}
L
Linus Torvalds 已提交
3741

G
Glauber Costa 已提交
3742 3743
	if (limit && shared && batchcount)
		goto skip_setup;
A
Andrew Morton 已提交
3744 3745
	/*
	 * The head array serves three purposes:
L
Linus Torvalds 已提交
3746 3747
	 * - create a LIFO ordering, i.e. return objects that are cache-warm
	 * - reduce the number of spinlock operations.
A
Andrew Morton 已提交
3748
	 * - reduce the number of linked list operations on the slab and
L
Linus Torvalds 已提交
3749 3750 3751 3752
	 *   bufctl chains: array operations are cheaper.
	 * The numbers are guessed, we should auto-tune as described by
	 * Bonwick.
	 */
3753
	if (cachep->size > 131072)
L
Linus Torvalds 已提交
3754
		limit = 1;
3755
	else if (cachep->size > PAGE_SIZE)
L
Linus Torvalds 已提交
3756
		limit = 8;
3757
	else if (cachep->size > 1024)
L
Linus Torvalds 已提交
3758
		limit = 24;
3759
	else if (cachep->size > 256)
L
Linus Torvalds 已提交
3760 3761 3762 3763
		limit = 54;
	else
		limit = 120;

A
Andrew Morton 已提交
3764 3765
	/*
	 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
L
Linus Torvalds 已提交
3766 3767 3768 3769 3770 3771 3772 3773
	 * allocation behaviour: Most allocs on one cpu, most free operations
	 * on another cpu. For these cases, an efficient object passing between
	 * cpus is necessary. This is provided by a shared array. The array
	 * replaces Bonwick's magazine layer.
	 * On uniprocessor, it's functionally equivalent (but less efficient)
	 * to a larger limit. Thus disabled by default.
	 */
	shared = 0;
3774
	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
L
Linus Torvalds 已提交
3775 3776 3777
		shared = 8;

#if DEBUG
A
Andrew Morton 已提交
3778 3779 3780
	/*
	 * With debugging enabled, large batchcount lead to excessively long
	 * periods with disabled local interrupts. Limit the batchcount
L
Linus Torvalds 已提交
3781 3782 3783 3784
	 */
	if (limit > 32)
		limit = 32;
#endif
G
Glauber Costa 已提交
3785 3786 3787
	batchcount = (limit + 1) / 2;
skip_setup:
	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
L
Linus Torvalds 已提交
3788 3789
	if (err)
		printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
P
Pekka Enberg 已提交
3790
		       cachep->name, -err);
3791
	return err;
L
Linus Torvalds 已提交
3792 3793
}

3794
/*
3795 3796
 * Drain an array if it contains any elements taking the node lock only if
 * necessary. Note that the node listlock also protects the array_cache
3797
 * if drain_array() is used on the shared array.
3798
 */
3799
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
3800
			 struct array_cache *ac, int force, int node)
L
Linus Torvalds 已提交
3801
{
3802
	LIST_HEAD(list);
L
Linus Torvalds 已提交
3803 3804
	int tofree;

3805 3806
	if (!ac || !ac->avail)
		return;
L
Linus Torvalds 已提交
3807 3808
	if (ac->touched && !force) {
		ac->touched = 0;
3809
	} else {
3810
		spin_lock_irq(&n->list_lock);
3811 3812 3813 3814
		if (ac->avail) {
			tofree = force ? ac->avail : (ac->limit + 4) / 5;
			if (tofree > ac->avail)
				tofree = (ac->avail + 1) / 2;
3815
			free_block(cachep, ac->entry, tofree, node, &list);
3816 3817 3818 3819
			ac->avail -= tofree;
			memmove(ac->entry, &(ac->entry[tofree]),
				sizeof(void *) * ac->avail);
		}
3820
		spin_unlock_irq(&n->list_lock);
3821
		slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3822 3823 3824 3825 3826
	}
}

/**
 * cache_reap - Reclaim memory from caches.
3827
 * @w: work descriptor
L
Linus Torvalds 已提交
3828 3829 3830 3831 3832 3833
 *
 * Called from workqueue/eventd every few seconds.
 * Purpose:
 * - clear the per-cpu caches for this CPU.
 * - return freeable pages to the main free memory pool.
 *
A
Andrew Morton 已提交
3834 3835
 * If we cannot acquire the cache chain mutex then just give up - we'll try
 * again on the next iteration.
L
Linus Torvalds 已提交
3836
 */
3837
static void cache_reap(struct work_struct *w)
L
Linus Torvalds 已提交
3838
{
3839
	struct kmem_cache *searchp;
3840
	struct kmem_cache_node *n;
3841
	int node = numa_mem_id();
3842
	struct delayed_work *work = to_delayed_work(w);
L
Linus Torvalds 已提交
3843

3844
	if (!mutex_trylock(&slab_mutex))
L
Linus Torvalds 已提交
3845
		/* Give up. Setup the next iteration. */
3846
		goto out;
L
Linus Torvalds 已提交
3847

3848
	list_for_each_entry(searchp, &slab_caches, list) {
L
Linus Torvalds 已提交
3849 3850
		check_irq_on();

3851
		/*
3852
		 * We only take the node lock if absolutely necessary and we
3853 3854 3855
		 * have established with reasonable certainty that
		 * we can do some work if the lock was obtained.
		 */
3856
		n = get_node(searchp, node);
3857

3858
		reap_alien(searchp, n);
L
Linus Torvalds 已提交
3859

3860
		drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
L
Linus Torvalds 已提交
3861

3862 3863 3864 3865
		/*
		 * These are racy checks but it does not matter
		 * if we skip one check or scan twice.
		 */
3866
		if (time_after(n->next_reap, jiffies))
3867
			goto next;
L
Linus Torvalds 已提交
3868

3869
		n->next_reap = jiffies + REAPTIMEOUT_NODE;
L
Linus Torvalds 已提交
3870

3871
		drain_array(searchp, n, n->shared, 0, node);
L
Linus Torvalds 已提交
3872

3873 3874
		if (n->free_touched)
			n->free_touched = 0;
3875 3876
		else {
			int freed;
L
Linus Torvalds 已提交
3877

3878
			freed = drain_freelist(searchp, n, (n->free_limit +
3879 3880 3881
				5 * searchp->num - 1) / (5 * searchp->num));
			STATS_ADD_REAPED(searchp, freed);
		}
3882
next:
L
Linus Torvalds 已提交
3883 3884 3885
		cond_resched();
	}
	check_irq_on();
3886
	mutex_unlock(&slab_mutex);
3887
	next_reap_node();
3888
out:
A
Andrew Morton 已提交
3889
	/* Set up the next iteration */
3890
	schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
L
Linus Torvalds 已提交
3891 3892
}

3893
#ifdef CONFIG_SLABINFO
3894
void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
L
Linus Torvalds 已提交
3895
{
3896
	struct page *page;
P
Pekka Enberg 已提交
3897 3898 3899 3900
	unsigned long active_objs;
	unsigned long num_objs;
	unsigned long active_slabs = 0;
	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
3901
	const char *name;
L
Linus Torvalds 已提交
3902
	char *error = NULL;
3903
	int node;
3904
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
3905 3906 3907

	active_objs = 0;
	num_slabs = 0;
3908
	for_each_kmem_cache_node(cachep, node, n) {
3909

3910
		check_irq_on();
3911
		spin_lock_irq(&n->list_lock);
3912

3913 3914
		list_for_each_entry(page, &n->slabs_full, lru) {
			if (page->active != cachep->num && !error)
3915 3916 3917 3918
				error = "slabs_full accounting error";
			active_objs += cachep->num;
			active_slabs++;
		}
3919 3920
		list_for_each_entry(page, &n->slabs_partial, lru) {
			if (page->active == cachep->num && !error)
3921
				error = "slabs_partial accounting error";
3922
			if (!page->active && !error)
3923
				error = "slabs_partial accounting error";
3924
			active_objs += page->active;
3925 3926
			active_slabs++;
		}
3927 3928
		list_for_each_entry(page, &n->slabs_free, lru) {
			if (page->active && !error)
3929
				error = "slabs_free accounting error";
3930 3931
			num_slabs++;
		}
3932 3933 3934
		free_objects += n->free_objects;
		if (n->shared)
			shared_avail += n->shared->avail;
3935

3936
		spin_unlock_irq(&n->list_lock);
L
Linus Torvalds 已提交
3937
	}
P
Pekka Enberg 已提交
3938 3939
	num_slabs += active_slabs;
	num_objs = num_slabs * cachep->num;
3940
	if (num_objs - active_objs != free_objects && !error)
L
Linus Torvalds 已提交
3941 3942
		error = "free_objects accounting error";

P
Pekka Enberg 已提交
3943
	name = cachep->name;
L
Linus Torvalds 已提交
3944 3945 3946
	if (error)
		printk(KERN_ERR "slab: cache %s error: %s\n", name, error);

3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960
	sinfo->active_objs = active_objs;
	sinfo->num_objs = num_objs;
	sinfo->active_slabs = active_slabs;
	sinfo->num_slabs = num_slabs;
	sinfo->shared_avail = shared_avail;
	sinfo->limit = cachep->limit;
	sinfo->batchcount = cachep->batchcount;
	sinfo->shared = cachep->shared;
	sinfo->objects_per_slab = cachep->num;
	sinfo->cache_order = cachep->gfporder;
}

void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
{
L
Linus Torvalds 已提交
3961
#if STATS
3962
	{			/* node stats */
L
Linus Torvalds 已提交
3963 3964 3965 3966 3967 3968 3969
		unsigned long high = cachep->high_mark;
		unsigned long allocs = cachep->num_allocations;
		unsigned long grown = cachep->grown;
		unsigned long reaped = cachep->reaped;
		unsigned long errors = cachep->errors;
		unsigned long max_freeable = cachep->max_freeable;
		unsigned long node_allocs = cachep->node_allocs;
3970
		unsigned long node_frees = cachep->node_frees;
3971
		unsigned long overflows = cachep->node_overflow;
L
Linus Torvalds 已提交
3972

J
Joe Perches 已提交
3973 3974 3975 3976 3977
		seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
			   "%4lu %4lu %4lu %4lu %4lu",
			   allocs, high, grown,
			   reaped, errors, max_freeable, node_allocs,
			   node_frees, overflows);
L
Linus Torvalds 已提交
3978 3979 3980 3981 3982 3983 3984 3985 3986
	}
	/* cpu stats */
	{
		unsigned long allochit = atomic_read(&cachep->allochit);
		unsigned long allocmiss = atomic_read(&cachep->allocmiss);
		unsigned long freehit = atomic_read(&cachep->freehit);
		unsigned long freemiss = atomic_read(&cachep->freemiss);

		seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
P
Pekka Enberg 已提交
3987
			   allochit, allocmiss, freehit, freemiss);
L
Linus Torvalds 已提交
3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999
	}
#endif
}

#define MAX_SLABINFO_WRITE 128
/**
 * slabinfo_write - Tuning for the slab allocator
 * @file: unused
 * @buffer: user buffer
 * @count: data length
 * @ppos: unused
 */
4000
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
P
Pekka Enberg 已提交
4001
		       size_t count, loff_t *ppos)
L
Linus Torvalds 已提交
4002
{
P
Pekka Enberg 已提交
4003
	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
L
Linus Torvalds 已提交
4004
	int limit, batchcount, shared, res;
4005
	struct kmem_cache *cachep;
P
Pekka Enberg 已提交
4006

L
Linus Torvalds 已提交
4007 4008 4009 4010
	if (count > MAX_SLABINFO_WRITE)
		return -EINVAL;
	if (copy_from_user(&kbuf, buffer, count))
		return -EFAULT;
P
Pekka Enberg 已提交
4011
	kbuf[MAX_SLABINFO_WRITE] = '\0';
L
Linus Torvalds 已提交
4012 4013 4014 4015 4016 4017 4018 4019 4020 4021

	tmp = strchr(kbuf, ' ');
	if (!tmp)
		return -EINVAL;
	*tmp = '\0';
	tmp++;
	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
		return -EINVAL;

	/* Find the cache in the chain of caches. */
4022
	mutex_lock(&slab_mutex);
L
Linus Torvalds 已提交
4023
	res = -EINVAL;
4024
	list_for_each_entry(cachep, &slab_caches, list) {
L
Linus Torvalds 已提交
4025
		if (!strcmp(cachep->name, kbuf)) {
A
Andrew Morton 已提交
4026 4027
			if (limit < 1 || batchcount < 1 ||
					batchcount > limit || shared < 0) {
4028
				res = 0;
L
Linus Torvalds 已提交
4029
			} else {
4030
				res = do_tune_cpucache(cachep, limit,
4031 4032
						       batchcount, shared,
						       GFP_KERNEL);
L
Linus Torvalds 已提交
4033 4034 4035 4036
			}
			break;
		}
	}
4037
	mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
4038 4039 4040 4041
	if (res >= 0)
		res = count;
	return res;
}
4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074

#ifdef CONFIG_DEBUG_SLAB_LEAK

static inline int add_caller(unsigned long *n, unsigned long v)
{
	unsigned long *p;
	int l;
	if (!v)
		return 1;
	l = n[1];
	p = n + 2;
	while (l) {
		int i = l/2;
		unsigned long *q = p + 2 * i;
		if (*q == v) {
			q[1]++;
			return 1;
		}
		if (*q > v) {
			l = i;
		} else {
			p = q + 2;
			l -= i + 1;
		}
	}
	if (++n[1] == n[0])
		return 0;
	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
	p[0] = v;
	p[1] = 1;
	return 1;
}

4075 4076
static void handle_slab(unsigned long *n, struct kmem_cache *c,
						struct page *page)
4077 4078
{
	void *p;
4079
	int i;
4080

4081 4082
	if (n[0] == n[1])
		return;
4083
	for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4084
		if (get_obj_status(page, i) != OBJECT_ACTIVE)
4085
			continue;
4086

4087 4088 4089 4090 4091 4092 4093 4094 4095
		if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
			return;
	}
}

static void show_symbol(struct seq_file *m, unsigned long address)
{
#ifdef CONFIG_KALLSYMS
	unsigned long offset, size;
4096
	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4097

4098
	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4099
		seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4100
		if (modname[0])
4101 4102 4103 4104 4105 4106 4107 4108 4109
			seq_printf(m, " [%s]", modname);
		return;
	}
#endif
	seq_printf(m, "%p", (void *)address);
}

static int leaks_show(struct seq_file *m, void *p)
{
4110
	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4111
	struct page *page;
4112
	struct kmem_cache_node *n;
4113
	const char *name;
4114
	unsigned long *x = m->private;
4115 4116 4117 4118 4119 4120 4121 4122 4123 4124
	int node;
	int i;

	if (!(cachep->flags & SLAB_STORE_USER))
		return 0;
	if (!(cachep->flags & SLAB_RED_ZONE))
		return 0;

	/* OK, we can do it */

4125
	x[1] = 0;
4126

4127
	for_each_kmem_cache_node(cachep, node, n) {
4128 4129

		check_irq_on();
4130
		spin_lock_irq(&n->list_lock);
4131

4132 4133 4134 4135
		list_for_each_entry(page, &n->slabs_full, lru)
			handle_slab(x, cachep, page);
		list_for_each_entry(page, &n->slabs_partial, lru)
			handle_slab(x, cachep, page);
4136
		spin_unlock_irq(&n->list_lock);
4137 4138
	}
	name = cachep->name;
4139
	if (x[0] == x[1]) {
4140
		/* Increase the buffer size */
4141
		mutex_unlock(&slab_mutex);
4142
		m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4143 4144
		if (!m->private) {
			/* Too bad, we are really out */
4145
			m->private = x;
4146
			mutex_lock(&slab_mutex);
4147 4148
			return -ENOMEM;
		}
4149 4150
		*(unsigned long *)m->private = x[0] * 2;
		kfree(x);
4151
		mutex_lock(&slab_mutex);
4152 4153 4154 4155
		/* Now make sure this entry will be retried */
		m->count = m->size;
		return 0;
	}
4156 4157 4158
	for (i = 0; i < x[1]; i++) {
		seq_printf(m, "%s: %lu ", name, x[2*i+3]);
		show_symbol(m, x[2*i+2]);
4159 4160
		seq_putc(m, '\n');
	}
4161

4162 4163 4164
	return 0;
}

4165
static const struct seq_operations slabstats_op = {
4166
	.start = slab_start,
4167 4168
	.next = slab_next,
	.stop = slab_stop,
4169 4170
	.show = leaks_show,
};
4171 4172 4173

static int slabstats_open(struct inode *inode, struct file *file)
{
4174 4175 4176 4177 4178 4179 4180 4181 4182
	unsigned long *n;

	n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
	if (!n)
		return -ENOMEM;

	*n = PAGE_SIZE / (2 * sizeof(unsigned long));

	return 0;
4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196
}

static const struct file_operations proc_slabstats_operations = {
	.open		= slabstats_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};
#endif

static int __init slab_proc_init(void)
{
#ifdef CONFIG_DEBUG_SLAB_LEAK
	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4197
#endif
4198 4199 4200
	return 0;
}
module_init(slab_proc_init);
L
Linus Torvalds 已提交
4201 4202
#endif

4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214
/**
 * ksize - get the actual amount of memory allocated for a given object
 * @objp: Pointer to the object
 *
 * kmalloc may internally round up allocations and return more memory
 * than requested. ksize() can be used to determine the actual amount of
 * memory allocated. The caller may use this additional memory, even though
 * a smaller amount of memory was initially specified with the kmalloc call.
 * The caller must guarantee that objp points to a valid object previously
 * allocated with either kmalloc() or kmem_cache_alloc(). The object
 * must not be freed during the duration of the call.
 */
P
Pekka Enberg 已提交
4215
size_t ksize(const void *objp)
L
Linus Torvalds 已提交
4216
{
4217 4218
	BUG_ON(!objp);
	if (unlikely(objp == ZERO_SIZE_PTR))
4219
		return 0;
L
Linus Torvalds 已提交
4220

4221
	return virt_to_cache(objp)->object_size;
L
Linus Torvalds 已提交
4222
}
K
Kirill A. Shutemov 已提交
4223
EXPORT_SYMBOL(ksize);