slab.c 106.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * linux/mm/slab.c
 * Written by Mark Hemment, 1996/97.
 * (markhe@nextd.demon.co.uk)
 *
 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
 *
 * Major cleanup, different bufctl logic, per-cpu arrays
 *	(c) 2000 Manfred Spraul
 *
 * Cleanup, make the head arrays unconditional, preparation for NUMA
 * 	(c) 2002 Manfred Spraul
 *
 * An implementation of the Slab Allocator as described in outline in;
 *	UNIX Internals: The New Frontiers by Uresh Vahalia
 *	Pub: Prentice Hall	ISBN 0-13-101908-2
 * or with a little more detail in;
 *	The Slab Allocator: An Object-Caching Kernel Memory Allocator
 *	Jeff Bonwick (Sun Microsystems).
 *	Presented at: USENIX Summer 1994 Technical Conference
 *
 * The memory is organized in caches, one cache for each object type.
 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
 * Each cache consists out of many slabs (they are small (usually one
 * page long) and always contiguous), and each slab contains multiple
 * initialized objects.
 *
 * This means, that your constructor is used only for newly allocated
S
Simon Arlott 已提交
29
 * slabs and you must pass objects with the same initializations to
L
Linus Torvalds 已提交
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
 * kmem_cache_free.
 *
 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
 * normal). If you need a special memory type, then must create a new
 * cache for that memory type.
 *
 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
 *   full slabs with 0 free objects
 *   partial slabs
 *   empty slabs with no allocated objects
 *
 * If partial slabs exist, then new allocations come from these slabs,
 * otherwise from empty slabs or new slabs are allocated.
 *
 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
 *
 * Each cache has a short per-cpu head array, most allocs
 * and frees go into that array, and if that array overflows, then 1/2
 * of the entries in the array are given back into the global cache.
 * The head array is strictly LIFO and should improve the cache hit rates.
 * On SMP, it additionally reduces the spinlock operations.
 *
A
Andrew Morton 已提交
53
 * The c_cpuarray may not be read with enabled local interrupts -
L
Linus Torvalds 已提交
54 55 56 57
 * it's changed with a smp_call_function().
 *
 * SMP synchronization:
 *  constructors and destructors are called without any locking.
58
 *  Several members in struct kmem_cache and struct slab never change, they
L
Linus Torvalds 已提交
59 60 61 62 63 64 65 66 67 68 69 70
 *	are accessed without any locking.
 *  The per-cpu arrays are never accessed from the wrong cpu, no locking,
 *  	and local interrupts are disabled so slab code is preempt-safe.
 *  The non-constant members are protected with a per-cache irq spinlock.
 *
 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
 * in 2000 - many ideas in the current implementation are derived from
 * his patch.
 *
 * Further notes from the original documentation:
 *
 * 11 April '97.  Started multi-threading - markhe
71
 *	The global cache-chain is protected by the mutex 'slab_mutex'.
L
Linus Torvalds 已提交
72 73 74 75 76 77
 *	The sem is only needed when accessing/extending the cache-chain, which
 *	can never happen inside an interrupt (kmem_cache_create(),
 *	kmem_cache_shrink() and kmem_cache_reap()).
 *
 *	At present, each engine can be growing a cache.  This should be blocked.
 *
78 79 80 81 82 83 84 85 86
 * 15 March 2005. NUMA slab allocator.
 *	Shai Fultheim <shai@scalex86.org>.
 *	Shobhit Dayal <shobhit@calsoftinc.com>
 *	Alok N Kataria <alokk@calsoftinc.com>
 *	Christoph Lameter <christoph@lameter.com>
 *
 *	Modified the slab allocator to be node aware on NUMA systems.
 *	Each node has its own list of partial, free and full slabs.
 *	All object allocations for a node occur from node specific slab lists.
L
Linus Torvalds 已提交
87 88 89 90
 */

#include	<linux/slab.h>
#include	<linux/mm.h>
91
#include	<linux/poison.h>
L
Linus Torvalds 已提交
92 93 94 95 96
#include	<linux/swap.h>
#include	<linux/cache.h>
#include	<linux/interrupt.h>
#include	<linux/init.h>
#include	<linux/compiler.h>
97
#include	<linux/cpuset.h>
98
#include	<linux/proc_fs.h>
L
Linus Torvalds 已提交
99 100 101 102 103 104 105
#include	<linux/seq_file.h>
#include	<linux/notifier.h>
#include	<linux/kallsyms.h>
#include	<linux/cpu.h>
#include	<linux/sysctl.h>
#include	<linux/module.h>
#include	<linux/rcupdate.h>
106
#include	<linux/string.h>
107
#include	<linux/uaccess.h>
108
#include	<linux/nodemask.h>
109
#include	<linux/kmemleak.h>
110
#include	<linux/mempolicy.h>
I
Ingo Molnar 已提交
111
#include	<linux/mutex.h>
112
#include	<linux/fault-inject.h>
I
Ingo Molnar 已提交
113
#include	<linux/rtmutex.h>
114
#include	<linux/reciprocal_div.h>
115
#include	<linux/debugobjects.h>
P
Pekka Enberg 已提交
116
#include	<linux/kmemcheck.h>
117
#include	<linux/memory.h>
118
#include	<linux/prefetch.h>
L
Linus Torvalds 已提交
119

120 121
#include	<net/sock.h>

L
Linus Torvalds 已提交
122 123 124 125
#include	<asm/cacheflush.h>
#include	<asm/tlbflush.h>
#include	<asm/page.h>

126 127
#include <trace/events/kmem.h>

128 129
#include	"internal.h"

130 131
#include	"slab.h"

L
Linus Torvalds 已提交
132
/*
133
 * DEBUG	- 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
L
Linus Torvalds 已提交
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
 *		  0 for faster, smaller code (especially in the critical paths).
 *
 * STATS	- 1 to collect stats for /proc/slabinfo.
 *		  0 for faster, smaller code (especially in the critical paths).
 *
 * FORCED_DEBUG	- 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
 */

#ifdef CONFIG_DEBUG_SLAB
#define	DEBUG		1
#define	STATS		1
#define	FORCED_DEBUG	1
#else
#define	DEBUG		0
#define	STATS		0
#define	FORCED_DEBUG	0
#endif

/* Shouldn't this be in a header file somewhere? */
#define	BYTES_PER_WORD		sizeof(void *)
D
David Woodhouse 已提交
154
#define	REDZONE_ALIGN		max(BYTES_PER_WORD, __alignof__(unsigned long long))
L
Linus Torvalds 已提交
155 156 157 158 159

#ifndef ARCH_KMALLOC_FLAGS
#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
#endif

160 161 162 163 164 165 166 167 168
#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
				<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)

#if FREELIST_BYTE_INDEX
typedef unsigned char freelist_idx_t;
#else
typedef unsigned short freelist_idx_t;
#endif

169
#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
170

171 172 173 174 175 176
/*
 * true if a page was allocated from pfmemalloc reserves for network-based
 * swap
 */
static bool pfmemalloc_active __read_mostly;

L
Linus Torvalds 已提交
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
/*
 * struct array_cache
 *
 * Purpose:
 * - LIFO ordering, to hand out cache-warm objects from _alloc
 * - reduce the number of linked list operations
 * - reduce spinlock operations
 *
 * The limit is stored in the per-cpu structure to reduce the data cache
 * footprint.
 *
 */
struct array_cache {
	unsigned int avail;
	unsigned int limit;
	unsigned int batchcount;
	unsigned int touched;
194
	void *entry[];	/*
A
Andrew Morton 已提交
195 196 197
			 * Must have this definition in here for the proper
			 * alignment of array_cache. Also simplifies accessing
			 * the entries.
198 199 200 201
			 *
			 * Entries should not be directly dereferenced as
			 * entries belonging to slabs marked pfmemalloc will
			 * have the lower bits set SLAB_OBJ_PFMEMALLOC
A
Andrew Morton 已提交
202
			 */
L
Linus Torvalds 已提交
203 204
};

J
Joonsoo Kim 已提交
205 206 207 208 209
struct alien_cache {
	spinlock_t lock;
	struct array_cache ac;
};

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
#define SLAB_OBJ_PFMEMALLOC	1
static inline bool is_obj_pfmemalloc(void *objp)
{
	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
}

static inline void set_obj_pfmemalloc(void **objp)
{
	*objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
	return;
}

static inline void clear_obj_pfmemalloc(void **objp)
{
	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
}

227 228 229
/*
 * Need this for bootstrapping a per node allocator.
 */
230
#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
231
static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
232
#define	CACHE_CACHE 0
233
#define	SIZE_NODE (MAX_NUMNODES)
234

235
static int drain_freelist(struct kmem_cache *cache,
236
			struct kmem_cache_node *n, int tofree);
237
static void free_block(struct kmem_cache *cachep, void **objpp, int len,
238 239
			int node, struct list_head *list);
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
240
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
241
static void cache_reap(struct work_struct *unused);
242

243 244
static int slab_early_init = 1;

245
#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
L
Linus Torvalds 已提交
246

247
static void kmem_cache_node_init(struct kmem_cache_node *parent)
248 249 250 251 252 253
{
	INIT_LIST_HEAD(&parent->slabs_full);
	INIT_LIST_HEAD(&parent->slabs_partial);
	INIT_LIST_HEAD(&parent->slabs_free);
	parent->shared = NULL;
	parent->alien = NULL;
254
	parent->colour_next = 0;
255 256 257 258 259
	spin_lock_init(&parent->list_lock);
	parent->free_objects = 0;
	parent->free_touched = 0;
}

A
Andrew Morton 已提交
260 261 262
#define MAKE_LIST(cachep, listp, slab, nodeid)				\
	do {								\
		INIT_LIST_HEAD(listp);					\
263
		list_splice(&get_node(cachep, nodeid)->slab, listp);	\
264 265
	} while (0)

A
Andrew Morton 已提交
266 267
#define	MAKE_ALL_LISTS(cachep, ptr, nodeid)				\
	do {								\
268 269 270 271
	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid);	\
	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);	\
	} while (0)
L
Linus Torvalds 已提交
272 273 274

#define CFLGS_OFF_SLAB		(0x80000000UL)
#define	OFF_SLAB(x)	((x)->flags & CFLGS_OFF_SLAB)
275
#define OFF_SLAB_MIN_SIZE (max_t(size_t, PAGE_SIZE >> 5, KMALLOC_MIN_SIZE + 1))
L
Linus Torvalds 已提交
276 277

#define BATCHREFILL_LIMIT	16
A
Andrew Morton 已提交
278 279 280
/*
 * Optimization question: fewer reaps means less probability for unnessary
 * cpucache drain/refill cycles.
L
Linus Torvalds 已提交
281
 *
A
Adrian Bunk 已提交
282
 * OTOH the cpuarrays can contain lots of objects,
L
Linus Torvalds 已提交
283 284
 * which could lock up otherwise freeable slabs.
 */
285 286
#define REAPTIMEOUT_AC		(2*HZ)
#define REAPTIMEOUT_NODE	(4*HZ)
L
Linus Torvalds 已提交
287 288 289 290 291 292

#if STATS
#define	STATS_INC_ACTIVE(x)	((x)->num_active++)
#define	STATS_DEC_ACTIVE(x)	((x)->num_active--)
#define	STATS_INC_ALLOCED(x)	((x)->num_allocations++)
#define	STATS_INC_GROWN(x)	((x)->grown++)
293
#define	STATS_ADD_REAPED(x,y)	((x)->reaped += (y))
A
Andrew Morton 已提交
294 295 296 297 298
#define	STATS_SET_HIGH(x)						\
	do {								\
		if ((x)->num_active > (x)->high_mark)			\
			(x)->high_mark = (x)->num_active;		\
	} while (0)
L
Linus Torvalds 已提交
299 300
#define	STATS_INC_ERR(x)	((x)->errors++)
#define	STATS_INC_NODEALLOCS(x)	((x)->node_allocs++)
301
#define	STATS_INC_NODEFREES(x)	((x)->node_frees++)
302
#define STATS_INC_ACOVERFLOW(x)   ((x)->node_overflow++)
A
Andrew Morton 已提交
303 304 305 306 307
#define	STATS_SET_FREEABLE(x, i)					\
	do {								\
		if ((x)->max_freeable < i)				\
			(x)->max_freeable = i;				\
	} while (0)
L
Linus Torvalds 已提交
308 309 310 311 312 313 314 315 316
#define STATS_INC_ALLOCHIT(x)	atomic_inc(&(x)->allochit)
#define STATS_INC_ALLOCMISS(x)	atomic_inc(&(x)->allocmiss)
#define STATS_INC_FREEHIT(x)	atomic_inc(&(x)->freehit)
#define STATS_INC_FREEMISS(x)	atomic_inc(&(x)->freemiss)
#else
#define	STATS_INC_ACTIVE(x)	do { } while (0)
#define	STATS_DEC_ACTIVE(x)	do { } while (0)
#define	STATS_INC_ALLOCED(x)	do { } while (0)
#define	STATS_INC_GROWN(x)	do { } while (0)
317
#define	STATS_ADD_REAPED(x,y)	do { (void)(y); } while (0)
L
Linus Torvalds 已提交
318 319 320
#define	STATS_SET_HIGH(x)	do { } while (0)
#define	STATS_INC_ERR(x)	do { } while (0)
#define	STATS_INC_NODEALLOCS(x)	do { } while (0)
321
#define	STATS_INC_NODEFREES(x)	do { } while (0)
322
#define STATS_INC_ACOVERFLOW(x)   do { } while (0)
A
Andrew Morton 已提交
323
#define	STATS_SET_FREEABLE(x, i) do { } while (0)
L
Linus Torvalds 已提交
324 325 326 327 328 329 330 331
#define STATS_INC_ALLOCHIT(x)	do { } while (0)
#define STATS_INC_ALLOCMISS(x)	do { } while (0)
#define STATS_INC_FREEHIT(x)	do { } while (0)
#define STATS_INC_FREEMISS(x)	do { } while (0)
#endif

#if DEBUG

A
Andrew Morton 已提交
332 333
/*
 * memory layout of objects:
L
Linus Torvalds 已提交
334
 * 0		: objp
335
 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
L
Linus Torvalds 已提交
336 337
 * 		the end of an object is aligned with the end of the real
 * 		allocation. Catches writes behind the end of the allocation.
338
 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
L
Linus Torvalds 已提交
339
 * 		redzone word.
340
 * cachep->obj_offset: The real object.
341 342
 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
 * cachep->size - 1* BYTES_PER_WORD: last caller address
A
Andrew Morton 已提交
343
 *					[BYTES_PER_WORD long]
L
Linus Torvalds 已提交
344
 */
345
static int obj_offset(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
346
{
347
	return cachep->obj_offset;
L
Linus Torvalds 已提交
348 349
}

350
static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
351 352
{
	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
353 354
	return (unsigned long long*) (objp + obj_offset(cachep) -
				      sizeof(unsigned long long));
L
Linus Torvalds 已提交
355 356
}

357
static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
358 359 360
{
	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
	if (cachep->flags & SLAB_STORE_USER)
361
		return (unsigned long long *)(objp + cachep->size -
362
					      sizeof(unsigned long long) -
D
David Woodhouse 已提交
363
					      REDZONE_ALIGN);
364
	return (unsigned long long *) (objp + cachep->size -
365
				       sizeof(unsigned long long));
L
Linus Torvalds 已提交
366 367
}

368
static void **dbg_userword(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
369 370
{
	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
371
	return (void **)(objp + cachep->size - BYTES_PER_WORD);
L
Linus Torvalds 已提交
372 373 374 375
}

#else

376
#define obj_offset(x)			0
377 378
#define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
#define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
L
Linus Torvalds 已提交
379 380 381 382
#define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})

#endif

383 384
#ifdef CONFIG_DEBUG_SLAB_LEAK

385
static inline bool is_store_user_clean(struct kmem_cache *cachep)
386
{
387 388
	return atomic_read(&cachep->store_user_clean) == 1;
}
389

390 391 392 393
static inline void set_store_user_clean(struct kmem_cache *cachep)
{
	atomic_set(&cachep->store_user_clean, 1);
}
394

395 396 397 398
static inline void set_store_user_dirty(struct kmem_cache *cachep)
{
	if (is_store_user_clean(cachep))
		atomic_set(&cachep->store_user_clean, 0);
399 400 401
}

#else
402
static inline void set_store_user_dirty(struct kmem_cache *cachep) {}
403 404 405

#endif

L
Linus Torvalds 已提交
406
/*
407 408
 * Do not go above this order unless 0 objects fit into the slab or
 * overridden on the command line.
L
Linus Torvalds 已提交
409
 */
410 411 412
#define	SLAB_MAX_ORDER_HI	1
#define	SLAB_MAX_ORDER_LO	0
static int slab_max_order = SLAB_MAX_ORDER_LO;
413
static bool slab_max_order_set __initdata;
L
Linus Torvalds 已提交
414

415 416
static inline struct kmem_cache *virt_to_cache(const void *obj)
{
417
	struct page *page = virt_to_head_page(obj);
C
Christoph Lameter 已提交
418
	return page->slab_cache;
419 420
}

421
static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
422 423
				 unsigned int idx)
{
424
	return page->s_mem + cache->size * idx;
425 426
}

427
/*
428 429 430
 * We want to avoid an expensive divide : (offset / cache->size)
 *   Using the fact that size is a constant for a particular cache,
 *   we can replace (offset / cache->size) by
431 432 433
 *   reciprocal_divide(offset, cache->reciprocal_buffer_size)
 */
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
434
					const struct page *page, void *obj)
435
{
436
	u32 offset = (obj - page->s_mem);
437
	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
438 439
}

440
#define BOOT_CPUCACHE_ENTRIES	1
L
Linus Torvalds 已提交
441
/* internal cache of cache description objs */
442
static struct kmem_cache kmem_cache_boot = {
P
Pekka Enberg 已提交
443 444 445
	.batchcount = 1,
	.limit = BOOT_CPUCACHE_ENTRIES,
	.shared = 1,
446
	.size = sizeof(struct kmem_cache),
P
Pekka Enberg 已提交
447
	.name = "kmem_cache",
L
Linus Torvalds 已提交
448 449
};

450 451
#define BAD_ALIEN_MAGIC 0x01020304ul

452
static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
L
Linus Torvalds 已提交
453

454
static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
455
{
456
	return this_cpu_ptr(cachep->cpu_cache);
L
Linus Torvalds 已提交
457 458
}

A
Andrew Morton 已提交
459 460 461
/*
 * Calculate the number of objects and left-over bytes for a given buffer size.
 */
462
static void cache_estimate(unsigned long gfporder, size_t buffer_size,
463
		unsigned long flags, size_t *left_over, unsigned int *num)
464 465
{
	size_t slab_size = PAGE_SIZE << gfporder;
L
Linus Torvalds 已提交
466

467 468 469 470 471 472
	/*
	 * The slab management structure can be either off the slab or
	 * on it. For the latter case, the memory allocated for a
	 * slab is used for:
	 *
	 * - @buffer_size bytes for each object
473 474 475 476 477
	 * - One freelist_idx_t for each object
	 *
	 * We don't need to consider alignment of freelist because
	 * freelist will be at the end of slab page. The objects will be
	 * at the correct alignment.
478 479 480 481 482 483 484
	 *
	 * If the slab management structure is off the slab, then the
	 * alignment will already be calculated into the size. Because
	 * the slabs are all pages aligned, the objects will be at the
	 * correct alignment when allocated.
	 */
	if (flags & CFLGS_OFF_SLAB) {
485 486
		*num = slab_size / buffer_size;
		*left_over = slab_size % buffer_size;
487
	} else {
488 489 490
		*num = slab_size / (buffer_size + sizeof(freelist_idx_t));
		*left_over = slab_size %
			(buffer_size + sizeof(freelist_idx_t));
491
	}
L
Linus Torvalds 已提交
492 493
}

494
#if DEBUG
495
#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
L
Linus Torvalds 已提交
496

A
Andrew Morton 已提交
497 498
static void __slab_error(const char *function, struct kmem_cache *cachep,
			char *msg)
L
Linus Torvalds 已提交
499 500
{
	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
P
Pekka Enberg 已提交
501
	       function, cachep->name, msg);
L
Linus Torvalds 已提交
502
	dump_stack();
503
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
504
}
505
#endif
L
Linus Torvalds 已提交
506

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
/*
 * By default on NUMA we use alien caches to stage the freeing of
 * objects allocated from other nodes. This causes massive memory
 * inefficiencies when using fake NUMA setup to split memory into a
 * large number of small nodes, so it can be disabled on the command
 * line
  */

static int use_alien_caches __read_mostly = 1;
static int __init noaliencache_setup(char *s)
{
	use_alien_caches = 0;
	return 1;
}
__setup("noaliencache", noaliencache_setup);

523 524 525 526 527 528 529 530 531 532 533
static int __init slab_max_order_setup(char *str)
{
	get_option(&str, &slab_max_order);
	slab_max_order = slab_max_order < 0 ? 0 :
				min(slab_max_order, MAX_ORDER - 1);
	slab_max_order_set = true;

	return 1;
}
__setup("slab_max_order=", slab_max_order_setup);

534 535 536 537 538 539 540
#ifdef CONFIG_NUMA
/*
 * Special reaping functions for NUMA systems called from cache_reap().
 * These take care of doing round robin flushing of alien caches (containing
 * objects freed on different nodes from which they were allocated) and the
 * flushing of remote pcps by calling drain_node_pages.
 */
541
static DEFINE_PER_CPU(unsigned long, slab_reap_node);
542 543 544 545 546

static void init_reap_node(int cpu)
{
	int node;

547
	node = next_node(cpu_to_mem(cpu), node_online_map);
548
	if (node == MAX_NUMNODES)
549
		node = first_node(node_online_map);
550

551
	per_cpu(slab_reap_node, cpu) = node;
552 553 554 555
}

static void next_reap_node(void)
{
556
	int node = __this_cpu_read(slab_reap_node);
557 558 559 560

	node = next_node(node, node_online_map);
	if (unlikely(node >= MAX_NUMNODES))
		node = first_node(node_online_map);
561
	__this_cpu_write(slab_reap_node, node);
562 563 564 565 566 567 568
}

#else
#define init_reap_node(cpu) do { } while (0)
#define next_reap_node(void) do { } while (0)
#endif

L
Linus Torvalds 已提交
569 570 571 572 573 574 575
/*
 * Initiate the reap timer running on the target CPU.  We run at around 1 to 2Hz
 * via the workqueue/eventd.
 * Add the CPU number into the expiration time to minimize the possibility of
 * the CPUs getting into lockstep and contending for the global cache chain
 * lock.
 */
576
static void start_cpu_timer(int cpu)
L
Linus Torvalds 已提交
577
{
578
	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
L
Linus Torvalds 已提交
579 580 581 582 583 584

	/*
	 * When this gets called from do_initcalls via cpucache_init(),
	 * init_workqueues() has already run, so keventd will be setup
	 * at that time.
	 */
585
	if (keventd_up() && reap_work->work.func == NULL) {
586
		init_reap_node(cpu);
587
		INIT_DEFERRABLE_WORK(reap_work, cache_reap);
588 589
		schedule_delayed_work_on(cpu, reap_work,
					__round_jiffies_relative(HZ, cpu));
L
Linus Torvalds 已提交
590 591 592
	}
}

593
static void init_arraycache(struct array_cache *ac, int limit, int batch)
L
Linus Torvalds 已提交
594
{
595 596
	/*
	 * The array_cache structures contain pointers to free object.
L
Lucas De Marchi 已提交
597
	 * However, when such objects are allocated or transferred to another
598 599 600 601
	 * cache the pointers are not cleared and they could be counted as
	 * valid references during a kmemleak scan. Therefore, kmemleak must
	 * not scan such objects.
	 */
602 603 604 605 606 607
	kmemleak_no_scan(ac);
	if (ac) {
		ac->avail = 0;
		ac->limit = limit;
		ac->batchcount = batch;
		ac->touched = 0;
L
Linus Torvalds 已提交
608
	}
609 610 611 612 613
}

static struct array_cache *alloc_arraycache(int node, int entries,
					    int batchcount, gfp_t gfp)
{
614
	size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
615 616 617 618 619
	struct array_cache *ac = NULL;

	ac = kmalloc_node(memsize, gfp, node);
	init_arraycache(ac, entries, batchcount);
	return ac;
L
Linus Torvalds 已提交
620 621
}

622
static inline bool is_slab_pfmemalloc(struct page *page)
623 624 625 626 627 628 629 630
{
	return PageSlabPfmemalloc(page);
}

/* Clears pfmemalloc_active if no slabs have pfmalloc set */
static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
						struct array_cache *ac)
{
631
	struct kmem_cache_node *n = get_node(cachep, numa_mem_id());
632
	struct page *page;
633 634 635 636 637
	unsigned long flags;

	if (!pfmemalloc_active)
		return;

638
	spin_lock_irqsave(&n->list_lock, flags);
639 640
	list_for_each_entry(page, &n->slabs_full, lru)
		if (is_slab_pfmemalloc(page))
641 642
			goto out;

643 644
	list_for_each_entry(page, &n->slabs_partial, lru)
		if (is_slab_pfmemalloc(page))
645 646
			goto out;

647 648
	list_for_each_entry(page, &n->slabs_free, lru)
		if (is_slab_pfmemalloc(page))
649 650 651 652
			goto out;

	pfmemalloc_active = false;
out:
653
	spin_unlock_irqrestore(&n->list_lock, flags);
654 655
}

656
static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
657 658 659 660 661 662 663
						gfp_t flags, bool force_refill)
{
	int i;
	void *objp = ac->entry[--ac->avail];

	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
	if (unlikely(is_obj_pfmemalloc(objp))) {
664
		struct kmem_cache_node *n;
665 666 667 668 669 670 671

		if (gfp_pfmemalloc_allowed(flags)) {
			clear_obj_pfmemalloc(&objp);
			return objp;
		}

		/* The caller cannot use PFMEMALLOC objects, find another one */
672
		for (i = 0; i < ac->avail; i++) {
673 674 675 676 677 678 679 680 681 682 683 684 685
			/* If a !PFMEMALLOC object is found, swap them */
			if (!is_obj_pfmemalloc(ac->entry[i])) {
				objp = ac->entry[i];
				ac->entry[i] = ac->entry[ac->avail];
				ac->entry[ac->avail] = objp;
				return objp;
			}
		}

		/*
		 * If there are empty slabs on the slabs_free list and we are
		 * being forced to refill the cache, mark this one !pfmemalloc.
		 */
686
		n = get_node(cachep, numa_mem_id());
687
		if (!list_empty(&n->slabs_free) && force_refill) {
688
			struct page *page = virt_to_head_page(objp);
689
			ClearPageSlabPfmemalloc(page);
690 691 692 693 694 695 696 697 698 699 700 701 702
			clear_obj_pfmemalloc(&objp);
			recheck_pfmemalloc_active(cachep, ac);
			return objp;
		}

		/* No !PFMEMALLOC objects available */
		ac->avail++;
		objp = NULL;
	}

	return objp;
}

703 704 705 706 707 708 709 710 711 712 713 714 715
static inline void *ac_get_obj(struct kmem_cache *cachep,
			struct array_cache *ac, gfp_t flags, bool force_refill)
{
	void *objp;

	if (unlikely(sk_memalloc_socks()))
		objp = __ac_get_obj(cachep, ac, flags, force_refill);
	else
		objp = ac->entry[--ac->avail];

	return objp;
}

J
Joonsoo Kim 已提交
716 717
static noinline void *__ac_put_obj(struct kmem_cache *cachep,
			struct array_cache *ac, void *objp)
718 719 720
{
	if (unlikely(pfmemalloc_active)) {
		/* Some pfmemalloc slabs exist, check if this is one */
721
		struct page *page = virt_to_head_page(objp);
722 723 724 725
		if (PageSlabPfmemalloc(page))
			set_obj_pfmemalloc(&objp);
	}

726 727 728 729 730 731 732 733 734
	return objp;
}

static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
								void *objp)
{
	if (unlikely(sk_memalloc_socks()))
		objp = __ac_put_obj(cachep, ac, objp);

735 736 737
	ac->entry[ac->avail++] = objp;
}

738 739 740 741 742 743 744 745 746 747
/*
 * Transfer objects in one arraycache to another.
 * Locking must be handled by the caller.
 *
 * Return the number of entries transferred.
 */
static int transfer_objects(struct array_cache *to,
		struct array_cache *from, unsigned int max)
{
	/* Figure out how many entries to transfer */
748
	int nr = min3(from->avail, max, to->limit - to->avail);
749 750 751 752 753 754 755 756 757 758 759 760

	if (!nr)
		return 0;

	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
			sizeof(void *) *nr);

	from->avail -= nr;
	to->avail += nr;
	return nr;
}

761 762 763
#ifndef CONFIG_NUMA

#define drain_alien_cache(cachep, alien) do { } while (0)
764
#define reap_alien(cachep, n) do { } while (0)
765

J
Joonsoo Kim 已提交
766 767
static inline struct alien_cache **alloc_alien_cache(int node,
						int limit, gfp_t gfp)
768
{
769
	return (struct alien_cache **)BAD_ALIEN_MAGIC;
770 771
}

J
Joonsoo Kim 已提交
772
static inline void free_alien_cache(struct alien_cache **ac_ptr)
773 774 775 776 777 778 779 780 781 782 783 784 785 786
{
}

static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
{
	return 0;
}

static inline void *alternate_node_alloc(struct kmem_cache *cachep,
		gfp_t flags)
{
	return NULL;
}

787
static inline void *____cache_alloc_node(struct kmem_cache *cachep,
788 789 790 791 792
		 gfp_t flags, int nodeid)
{
	return NULL;
}

D
David Rientjes 已提交
793 794 795 796 797
static inline gfp_t gfp_exact_node(gfp_t flags)
{
	return flags;
}

798 799
#else	/* CONFIG_NUMA */

800
static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
801
static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
802

J
Joonsoo Kim 已提交
803 804 805
static struct alien_cache *__alloc_alien_cache(int node, int entries,
						int batch, gfp_t gfp)
{
806
	size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
J
Joonsoo Kim 已提交
807 808 809 810
	struct alien_cache *alc = NULL;

	alc = kmalloc_node(memsize, gfp, node);
	init_arraycache(&alc->ac, entries, batch);
811
	spin_lock_init(&alc->lock);
J
Joonsoo Kim 已提交
812 813 814 815
	return alc;
}

static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
816
{
J
Joonsoo Kim 已提交
817
	struct alien_cache **alc_ptr;
818
	size_t memsize = sizeof(void *) * nr_node_ids;
819 820 821 822
	int i;

	if (limit > 1)
		limit = 12;
J
Joonsoo Kim 已提交
823 824 825 826 827 828 829 830 831 832 833 834 835
	alc_ptr = kzalloc_node(memsize, gfp, node);
	if (!alc_ptr)
		return NULL;

	for_each_node(i) {
		if (i == node || !node_online(i))
			continue;
		alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
		if (!alc_ptr[i]) {
			for (i--; i >= 0; i--)
				kfree(alc_ptr[i]);
			kfree(alc_ptr);
			return NULL;
836 837
		}
	}
J
Joonsoo Kim 已提交
838
	return alc_ptr;
839 840
}

J
Joonsoo Kim 已提交
841
static void free_alien_cache(struct alien_cache **alc_ptr)
842 843 844
{
	int i;

J
Joonsoo Kim 已提交
845
	if (!alc_ptr)
846 847
		return;
	for_each_node(i)
J
Joonsoo Kim 已提交
848 849
	    kfree(alc_ptr[i]);
	kfree(alc_ptr);
850 851
}

852
static void __drain_alien_cache(struct kmem_cache *cachep,
853 854
				struct array_cache *ac, int node,
				struct list_head *list)
855
{
856
	struct kmem_cache_node *n = get_node(cachep, node);
857 858

	if (ac->avail) {
859
		spin_lock(&n->list_lock);
860 861 862 863 864
		/*
		 * Stuff objects into the remote nodes shared array first.
		 * That way we could avoid the overhead of putting the objects
		 * into the free lists and getting them back later.
		 */
865 866
		if (n->shared)
			transfer_objects(n->shared, ac, ac->limit);
867

868
		free_block(cachep, ac->entry, ac->avail, node, list);
869
		ac->avail = 0;
870
		spin_unlock(&n->list_lock);
871 872 873
	}
}

874 875 876
/*
 * Called from cache_reap() to regularly drain alien caches round robin.
 */
877
static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
878
{
879
	int node = __this_cpu_read(slab_reap_node);
880

881
	if (n->alien) {
J
Joonsoo Kim 已提交
882 883 884 885 886
		struct alien_cache *alc = n->alien[node];
		struct array_cache *ac;

		if (alc) {
			ac = &alc->ac;
887
			if (ac->avail && spin_trylock_irq(&alc->lock)) {
888 889 890
				LIST_HEAD(list);

				__drain_alien_cache(cachep, ac, node, &list);
891
				spin_unlock_irq(&alc->lock);
892
				slabs_destroy(cachep, &list);
J
Joonsoo Kim 已提交
893
			}
894 895 896 897
		}
	}
}

A
Andrew Morton 已提交
898
static void drain_alien_cache(struct kmem_cache *cachep,
J
Joonsoo Kim 已提交
899
				struct alien_cache **alien)
900
{
P
Pekka Enberg 已提交
901
	int i = 0;
J
Joonsoo Kim 已提交
902
	struct alien_cache *alc;
903 904 905 906
	struct array_cache *ac;
	unsigned long flags;

	for_each_online_node(i) {
J
Joonsoo Kim 已提交
907 908
		alc = alien[i];
		if (alc) {
909 910
			LIST_HEAD(list);

J
Joonsoo Kim 已提交
911
			ac = &alc->ac;
912
			spin_lock_irqsave(&alc->lock, flags);
913
			__drain_alien_cache(cachep, ac, i, &list);
914
			spin_unlock_irqrestore(&alc->lock, flags);
915
			slabs_destroy(cachep, &list);
916 917 918
		}
	}
}
919

920 921
static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
				int node, int page_node)
922
{
923
	struct kmem_cache_node *n;
J
Joonsoo Kim 已提交
924 925
	struct alien_cache *alien = NULL;
	struct array_cache *ac;
926
	LIST_HEAD(list);
P
Pekka Enberg 已提交
927

928
	n = get_node(cachep, node);
929
	STATS_INC_NODEFREES(cachep);
930 931
	if (n->alien && n->alien[page_node]) {
		alien = n->alien[page_node];
J
Joonsoo Kim 已提交
932
		ac = &alien->ac;
933
		spin_lock(&alien->lock);
J
Joonsoo Kim 已提交
934
		if (unlikely(ac->avail == ac->limit)) {
935
			STATS_INC_ACOVERFLOW(cachep);
936
			__drain_alien_cache(cachep, ac, page_node, &list);
937
		}
J
Joonsoo Kim 已提交
938
		ac_put_obj(cachep, ac, objp);
939
		spin_unlock(&alien->lock);
940
		slabs_destroy(cachep, &list);
941
	} else {
942
		n = get_node(cachep, page_node);
943
		spin_lock(&n->list_lock);
944
		free_block(cachep, &objp, 1, page_node, &list);
945
		spin_unlock(&n->list_lock);
946
		slabs_destroy(cachep, &list);
947 948 949
	}
	return 1;
}
950 951 952 953 954 955 956 957 958 959 960 961 962 963

static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
{
	int page_node = page_to_nid(virt_to_page(objp));
	int node = numa_mem_id();
	/*
	 * Make sure we are not freeing a object from another node to the array
	 * cache on this cpu.
	 */
	if (likely(node == page_node))
		return 0;

	return __cache_free_alien(cachep, objp, node, page_node);
}
D
David Rientjes 已提交
964 965

/*
966 967
 * Construct gfp mask to allocate from a specific node but do not direct reclaim
 * or warn about failures. kswapd may still wake to reclaim in the background.
D
David Rientjes 已提交
968 969 970
 */
static inline gfp_t gfp_exact_node(gfp_t flags)
{
971
	return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~__GFP_DIRECT_RECLAIM;
D
David Rientjes 已提交
972
}
973 974
#endif

975
/*
976
 * Allocates and initializes node for a node on each slab cache, used for
977
 * either memory or cpu hotplug.  If memory is being hot-added, the kmem_cache_node
978
 * will be allocated off-node since memory is not yet online for the new node.
979
 * When hotplugging memory or a cpu, existing node are not replaced if
980 981
 * already in use.
 *
982
 * Must hold slab_mutex.
983
 */
984
static int init_cache_node_node(int node)
985 986
{
	struct kmem_cache *cachep;
987
	struct kmem_cache_node *n;
988
	const size_t memsize = sizeof(struct kmem_cache_node);
989

990
	list_for_each_entry(cachep, &slab_caches, list) {
991
		/*
992
		 * Set up the kmem_cache_node for cpu before we can
993 994 995
		 * begin anything. Make sure some other cpu on this
		 * node has not already allocated this
		 */
996 997
		n = get_node(cachep, node);
		if (!n) {
998 999
			n = kmalloc_node(memsize, GFP_KERNEL, node);
			if (!n)
1000
				return -ENOMEM;
1001
			kmem_cache_node_init(n);
1002 1003
			n->next_reap = jiffies + REAPTIMEOUT_NODE +
			    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1004 1005

			/*
1006 1007
			 * The kmem_cache_nodes don't come and go as CPUs
			 * come and go.  slab_mutex is sufficient
1008 1009
			 * protection here.
			 */
1010
			cachep->node[node] = n;
1011 1012
		}

1013 1014
		spin_lock_irq(&n->list_lock);
		n->free_limit =
1015 1016
			(1 + nr_cpus_node(node)) *
			cachep->batchcount + cachep->num;
1017
		spin_unlock_irq(&n->list_lock);
1018 1019 1020 1021
	}
	return 0;
}

1022 1023 1024 1025 1026 1027
static inline int slabs_tofree(struct kmem_cache *cachep,
						struct kmem_cache_node *n)
{
	return (n->free_objects + cachep->num - 1) / cachep->num;
}

1028
static void cpuup_canceled(long cpu)
1029 1030
{
	struct kmem_cache *cachep;
1031
	struct kmem_cache_node *n = NULL;
1032
	int node = cpu_to_mem(cpu);
1033
	const struct cpumask *mask = cpumask_of_node(node);
1034

1035
	list_for_each_entry(cachep, &slab_caches, list) {
1036 1037
		struct array_cache *nc;
		struct array_cache *shared;
J
Joonsoo Kim 已提交
1038
		struct alien_cache **alien;
1039
		LIST_HEAD(list);
1040

1041
		n = get_node(cachep, node);
1042
		if (!n)
1043
			continue;
1044

1045
		spin_lock_irq(&n->list_lock);
1046

1047 1048
		/* Free limit for this kmem_cache_node */
		n->free_limit -= cachep->batchcount;
1049 1050 1051 1052

		/* cpu is dead; no one can alloc from it. */
		nc = per_cpu_ptr(cachep->cpu_cache, cpu);
		if (nc) {
1053
			free_block(cachep, nc->entry, nc->avail, node, &list);
1054 1055
			nc->avail = 0;
		}
1056

1057
		if (!cpumask_empty(mask)) {
1058
			spin_unlock_irq(&n->list_lock);
1059
			goto free_slab;
1060 1061
		}

1062
		shared = n->shared;
1063 1064
		if (shared) {
			free_block(cachep, shared->entry,
1065
				   shared->avail, node, &list);
1066
			n->shared = NULL;
1067 1068
		}

1069 1070
		alien = n->alien;
		n->alien = NULL;
1071

1072
		spin_unlock_irq(&n->list_lock);
1073 1074 1075 1076 1077 1078

		kfree(shared);
		if (alien) {
			drain_alien_cache(cachep, alien);
			free_alien_cache(alien);
		}
1079 1080

free_slab:
1081
		slabs_destroy(cachep, &list);
1082 1083 1084 1085 1086 1087
	}
	/*
	 * In the previous loop, all the objects were freed to
	 * the respective cache's slabs,  now we can go ahead and
	 * shrink each nodelist to its limit.
	 */
1088
	list_for_each_entry(cachep, &slab_caches, list) {
1089
		n = get_node(cachep, node);
1090
		if (!n)
1091
			continue;
1092
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1093 1094 1095
	}
}

1096
static int cpuup_prepare(long cpu)
L
Linus Torvalds 已提交
1097
{
1098
	struct kmem_cache *cachep;
1099
	struct kmem_cache_node *n = NULL;
1100
	int node = cpu_to_mem(cpu);
1101
	int err;
L
Linus Torvalds 已提交
1102

1103 1104 1105 1106
	/*
	 * We need to do this right in the beginning since
	 * alloc_arraycache's are going to use this list.
	 * kmalloc_node allows us to add the slab to the right
1107
	 * kmem_cache_node and not this cpu's kmem_cache_node
1108
	 */
1109
	err = init_cache_node_node(node);
1110 1111
	if (err < 0)
		goto bad;
1112 1113 1114 1115 1116

	/*
	 * Now we can go ahead with allocating the shared arrays and
	 * array caches
	 */
1117
	list_for_each_entry(cachep, &slab_caches, list) {
1118
		struct array_cache *shared = NULL;
J
Joonsoo Kim 已提交
1119
		struct alien_cache **alien = NULL;
1120 1121 1122 1123

		if (cachep->shared) {
			shared = alloc_arraycache(node,
				cachep->shared * cachep->batchcount,
1124
				0xbaadf00d, GFP_KERNEL);
1125
			if (!shared)
L
Linus Torvalds 已提交
1126
				goto bad;
1127 1128
		}
		if (use_alien_caches) {
1129
			alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
1130 1131
			if (!alien) {
				kfree(shared);
1132
				goto bad;
1133
			}
1134
		}
1135
		n = get_node(cachep, node);
1136
		BUG_ON(!n);
1137

1138 1139
		spin_lock_irq(&n->list_lock);
		if (!n->shared) {
1140 1141 1142 1143
			/*
			 * We are serialised from CPU_DEAD or
			 * CPU_UP_CANCELLED by the cpucontrol lock
			 */
1144
			n->shared = shared;
1145 1146
			shared = NULL;
		}
1147
#ifdef CONFIG_NUMA
1148 1149
		if (!n->alien) {
			n->alien = alien;
1150
			alien = NULL;
L
Linus Torvalds 已提交
1151
		}
1152
#endif
1153
		spin_unlock_irq(&n->list_lock);
1154 1155 1156
		kfree(shared);
		free_alien_cache(alien);
	}
1157

1158 1159
	return 0;
bad:
1160
	cpuup_canceled(cpu);
1161 1162 1163
	return -ENOMEM;
}

1164
static int cpuup_callback(struct notifier_block *nfb,
1165 1166 1167 1168 1169 1170 1171 1172
				    unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
	int err = 0;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
1173
		mutex_lock(&slab_mutex);
1174
		err = cpuup_prepare(cpu);
1175
		mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
1176 1177
		break;
	case CPU_ONLINE:
1178
	case CPU_ONLINE_FROZEN:
L
Linus Torvalds 已提交
1179 1180 1181
		start_cpu_timer(cpu);
		break;
#ifdef CONFIG_HOTPLUG_CPU
1182
  	case CPU_DOWN_PREPARE:
1183
  	case CPU_DOWN_PREPARE_FROZEN:
1184
		/*
1185
		 * Shutdown cache reaper. Note that the slab_mutex is
1186 1187 1188 1189
		 * held so that if cache_reap() is invoked it cannot do
		 * anything expensive but will only modify reap_work
		 * and reschedule the timer.
		*/
1190
		cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1191
		/* Now the cache_reaper is guaranteed to be not running. */
1192
		per_cpu(slab_reap_work, cpu).work.func = NULL;
1193 1194
  		break;
  	case CPU_DOWN_FAILED:
1195
  	case CPU_DOWN_FAILED_FROZEN:
1196 1197
		start_cpu_timer(cpu);
  		break;
L
Linus Torvalds 已提交
1198
	case CPU_DEAD:
1199
	case CPU_DEAD_FROZEN:
1200 1201
		/*
		 * Even if all the cpus of a node are down, we don't free the
1202
		 * kmem_cache_node of any cache. This to avoid a race between
1203
		 * cpu_down, and a kmalloc allocation from another cpu for
1204
		 * memory from the node of the cpu going down.  The node
1205 1206 1207
		 * structure is usually allocated from kmem_cache_create() and
		 * gets destroyed at kmem_cache_destroy().
		 */
S
Simon Arlott 已提交
1208
		/* fall through */
1209
#endif
L
Linus Torvalds 已提交
1210
	case CPU_UP_CANCELED:
1211
	case CPU_UP_CANCELED_FROZEN:
1212
		mutex_lock(&slab_mutex);
1213
		cpuup_canceled(cpu);
1214
		mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
1215 1216
		break;
	}
1217
	return notifier_from_errno(err);
L
Linus Torvalds 已提交
1218 1219
}

1220
static struct notifier_block cpucache_notifier = {
1221 1222
	&cpuup_callback, NULL, 0
};
L
Linus Torvalds 已提交
1223

1224 1225 1226 1227 1228 1229
#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
/*
 * Drains freelist for a node on each slab cache, used for memory hot-remove.
 * Returns -EBUSY if all objects cannot be drained so that the node is not
 * removed.
 *
1230
 * Must hold slab_mutex.
1231
 */
1232
static int __meminit drain_cache_node_node(int node)
1233 1234 1235 1236
{
	struct kmem_cache *cachep;
	int ret = 0;

1237
	list_for_each_entry(cachep, &slab_caches, list) {
1238
		struct kmem_cache_node *n;
1239

1240
		n = get_node(cachep, node);
1241
		if (!n)
1242 1243
			continue;

1244
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1245

1246 1247
		if (!list_empty(&n->slabs_full) ||
		    !list_empty(&n->slabs_partial)) {
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
			ret = -EBUSY;
			break;
		}
	}
	return ret;
}

static int __meminit slab_memory_callback(struct notifier_block *self,
					unsigned long action, void *arg)
{
	struct memory_notify *mnb = arg;
	int ret = 0;
	int nid;

	nid = mnb->status_change_nid;
	if (nid < 0)
		goto out;

	switch (action) {
	case MEM_GOING_ONLINE:
1268
		mutex_lock(&slab_mutex);
1269
		ret = init_cache_node_node(nid);
1270
		mutex_unlock(&slab_mutex);
1271 1272
		break;
	case MEM_GOING_OFFLINE:
1273
		mutex_lock(&slab_mutex);
1274
		ret = drain_cache_node_node(nid);
1275
		mutex_unlock(&slab_mutex);
1276 1277 1278 1279 1280 1281 1282 1283
		break;
	case MEM_ONLINE:
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
out:
1284
	return notifier_from_errno(ret);
1285 1286 1287
}
#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */

1288
/*
1289
 * swap the static kmem_cache_node with kmalloced memory
1290
 */
1291
static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
1292
				int nodeid)
1293
{
1294
	struct kmem_cache_node *ptr;
1295

1296
	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1297 1298
	BUG_ON(!ptr);

1299
	memcpy(ptr, list, sizeof(struct kmem_cache_node));
1300 1301 1302 1303 1304
	/*
	 * Do not assume that spinlocks can be initialized via memcpy:
	 */
	spin_lock_init(&ptr->list_lock);

1305
	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1306
	cachep->node[nodeid] = ptr;
1307 1308
}

1309
/*
1310 1311
 * For setting up all the kmem_cache_node for cache whose buffer_size is same as
 * size of kmem_cache_node.
1312
 */
1313
static void __init set_up_node(struct kmem_cache *cachep, int index)
1314 1315 1316 1317
{
	int node;

	for_each_online_node(node) {
1318
		cachep->node[node] = &init_kmem_cache_node[index + node];
1319
		cachep->node[node]->next_reap = jiffies +
1320 1321
		    REAPTIMEOUT_NODE +
		    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1322 1323 1324
	}
}

A
Andrew Morton 已提交
1325 1326 1327
/*
 * Initialisation.  Called after the page allocator have been initialised and
 * before smp_init().
L
Linus Torvalds 已提交
1328 1329 1330
 */
void __init kmem_cache_init(void)
{
1331 1332
	int i;

1333 1334
	BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
					sizeof(struct rcu_head));
1335 1336
	kmem_cache = &kmem_cache_boot;

1337
	if (num_possible_nodes() == 1)
1338 1339
		use_alien_caches = 0;

C
Christoph Lameter 已提交
1340
	for (i = 0; i < NUM_INIT_LISTS; i++)
1341
		kmem_cache_node_init(&init_kmem_cache_node[i]);
C
Christoph Lameter 已提交
1342

L
Linus Torvalds 已提交
1343 1344
	/*
	 * Fragmentation resistance on low memory - only use bigger
1345 1346
	 * page orders on machines with more than 32MB of memory if
	 * not overridden on the command line.
L
Linus Torvalds 已提交
1347
	 */
1348
	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1349
		slab_max_order = SLAB_MAX_ORDER_HI;
L
Linus Torvalds 已提交
1350 1351 1352

	/* Bootstrap is tricky, because several objects are allocated
	 * from caches that do not exist yet:
1353 1354 1355
	 * 1) initialize the kmem_cache cache: it contains the struct
	 *    kmem_cache structures of all caches, except kmem_cache itself:
	 *    kmem_cache is statically allocated.
1356
	 *    Initially an __init data area is used for the head array and the
1357
	 *    kmem_cache_node structures, it's replaced with a kmalloc allocated
1358
	 *    array at the end of the bootstrap.
L
Linus Torvalds 已提交
1359
	 * 2) Create the first kmalloc cache.
1360
	 *    The struct kmem_cache for the new cache is allocated normally.
1361 1362 1363
	 *    An __init data area is used for the head array.
	 * 3) Create the remaining kmalloc caches, with minimally sized
	 *    head arrays.
1364
	 * 4) Replace the __init data head arrays for kmem_cache and the first
L
Linus Torvalds 已提交
1365
	 *    kmalloc cache with kmalloc allocated arrays.
1366
	 * 5) Replace the __init data for kmem_cache_node for kmem_cache and
1367 1368
	 *    the other cache's with kmalloc allocated memory.
	 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
L
Linus Torvalds 已提交
1369 1370
	 */

1371
	/* 1) create the kmem_cache */
L
Linus Torvalds 已提交
1372

E
Eric Dumazet 已提交
1373
	/*
1374
	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
E
Eric Dumazet 已提交
1375
	 */
1376
	create_boot_cache(kmem_cache, "kmem_cache",
1377
		offsetof(struct kmem_cache, node) +
1378
				  nr_node_ids * sizeof(struct kmem_cache_node *),
1379 1380
				  SLAB_HWCACHE_ALIGN);
	list_add(&kmem_cache->list, &slab_caches);
1381
	slab_state = PARTIAL;
L
Linus Torvalds 已提交
1382

A
Andrew Morton 已提交
1383
	/*
1384 1385
	 * Initialize the caches that provide memory for the  kmem_cache_node
	 * structures first.  Without this, further allocations will bug.
1386
	 */
1387
	kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
1388
				kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
1389
	slab_state = PARTIAL_NODE;
1390
	setup_kmalloc_cache_index_table();
1391

1392 1393
	slab_early_init = 0;

1394
	/* 5) Replace the bootstrap kmem_cache_node */
1395
	{
P
Pekka Enberg 已提交
1396 1397
		int nid;

1398
		for_each_online_node(nid) {
1399
			init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1400

1401
			init_list(kmalloc_caches[INDEX_NODE],
1402
					  &init_kmem_cache_node[SIZE_NODE + nid], nid);
1403 1404
		}
	}
L
Linus Torvalds 已提交
1405

1406
	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1407 1408 1409 1410 1411 1412
}

void __init kmem_cache_init_late(void)
{
	struct kmem_cache *cachep;

1413
	slab_state = UP;
P
Peter Zijlstra 已提交
1414

1415
	/* 6) resize the head arrays to their final sizes */
1416 1417
	mutex_lock(&slab_mutex);
	list_for_each_entry(cachep, &slab_caches, list)
1418 1419
		if (enable_cpucache(cachep, GFP_NOWAIT))
			BUG();
1420
	mutex_unlock(&slab_mutex);
1421

1422 1423 1424
	/* Done! */
	slab_state = FULL;

A
Andrew Morton 已提交
1425 1426 1427
	/*
	 * Register a cpu startup notifier callback that initializes
	 * cpu_cache_get for all new cpus
L
Linus Torvalds 已提交
1428 1429 1430
	 */
	register_cpu_notifier(&cpucache_notifier);

1431 1432 1433
#ifdef CONFIG_NUMA
	/*
	 * Register a memory hotplug callback that initializes and frees
1434
	 * node.
1435 1436 1437 1438
	 */
	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
#endif

A
Andrew Morton 已提交
1439 1440 1441
	/*
	 * The reap timers are started later, with a module init call: That part
	 * of the kernel is not yet operational.
L
Linus Torvalds 已提交
1442 1443 1444 1445 1446 1447 1448
	 */
}

static int __init cpucache_init(void)
{
	int cpu;

A
Andrew Morton 已提交
1449 1450
	/*
	 * Register the timers that return unneeded pages to the page allocator
L
Linus Torvalds 已提交
1451
	 */
1452
	for_each_online_cpu(cpu)
A
Andrew Morton 已提交
1453
		start_cpu_timer(cpu);
1454 1455

	/* Done! */
1456
	slab_state = FULL;
L
Linus Torvalds 已提交
1457 1458 1459 1460
	return 0;
}
__initcall(cpucache_init);

1461 1462 1463
static noinline void
slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
{
1464
#if DEBUG
1465
	struct kmem_cache_node *n;
1466
	struct page *page;
1467 1468
	unsigned long flags;
	int node;
1469 1470 1471 1472 1473
	static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
		return;
1474 1475 1476 1477 1478

	printk(KERN_WARNING
		"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
		nodeid, gfpflags);
	printk(KERN_WARNING "  cache: %s, object size: %d, order: %d\n",
1479
		cachep->name, cachep->size, cachep->gfporder);
1480

1481
	for_each_kmem_cache_node(cachep, node, n) {
1482 1483 1484
		unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
		unsigned long active_slabs = 0, num_slabs = 0;

1485
		spin_lock_irqsave(&n->list_lock, flags);
1486
		list_for_each_entry(page, &n->slabs_full, lru) {
1487 1488 1489
			active_objs += cachep->num;
			active_slabs++;
		}
1490 1491
		list_for_each_entry(page, &n->slabs_partial, lru) {
			active_objs += page->active;
1492 1493
			active_slabs++;
		}
1494
		list_for_each_entry(page, &n->slabs_free, lru)
1495 1496
			num_slabs++;

1497 1498
		free_objects += n->free_objects;
		spin_unlock_irqrestore(&n->list_lock, flags);
1499 1500 1501 1502 1503 1504 1505 1506

		num_slabs += active_slabs;
		num_objs = num_slabs * cachep->num;
		printk(KERN_WARNING
			"  node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
			node, active_slabs, num_slabs, active_objs, num_objs,
			free_objects);
	}
1507
#endif
1508 1509
}

L
Linus Torvalds 已提交
1510
/*
W
Wang Sheng-Hui 已提交
1511 1512
 * Interface to system's page allocator. No need to hold the
 * kmem_cache_node ->list_lock.
L
Linus Torvalds 已提交
1513 1514 1515 1516 1517
 *
 * If we requested dmaable memory, we will get it. Even if we
 * did not request dmaable memory, we might get it, but that
 * would be relatively rare and ignorable.
 */
1518 1519
static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
								int nodeid)
L
Linus Torvalds 已提交
1520 1521
{
	struct page *page;
1522
	int nr_pages;
1523

1524
	flags |= cachep->allocflags;
1525 1526
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
		flags |= __GFP_RECLAIMABLE;
1527

1528
	page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
1529
	if (!page) {
1530
		slab_out_of_memory(cachep, flags, nodeid);
L
Linus Torvalds 已提交
1531
		return NULL;
1532
	}
L
Linus Torvalds 已提交
1533

1534 1535 1536 1537 1538
	if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
		__free_pages(page, cachep->gfporder);
		return NULL;
	}

1539
	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1540
	if (page_is_pfmemalloc(page))
1541 1542
		pfmemalloc_active = true;

1543
	nr_pages = (1 << cachep->gfporder);
L
Linus Torvalds 已提交
1544
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1545 1546 1547 1548 1549
		add_zone_page_state(page_zone(page),
			NR_SLAB_RECLAIMABLE, nr_pages);
	else
		add_zone_page_state(page_zone(page),
			NR_SLAB_UNRECLAIMABLE, nr_pages);
1550
	__SetPageSlab(page);
1551
	if (page_is_pfmemalloc(page))
1552
		SetPageSlabPfmemalloc(page);
1553

1554 1555 1556 1557 1558 1559 1560 1561
	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
		kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);

		if (cachep->ctor)
			kmemcheck_mark_uninitialized_pages(page, nr_pages);
		else
			kmemcheck_mark_unallocated_pages(page, nr_pages);
	}
P
Pekka Enberg 已提交
1562

1563
	return page;
L
Linus Torvalds 已提交
1564 1565 1566 1567 1568
}

/*
 * Interface to system's page release.
 */
1569
static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
L
Linus Torvalds 已提交
1570
{
1571
	const unsigned long nr_freed = (1 << cachep->gfporder);
L
Linus Torvalds 已提交
1572

1573
	kmemcheck_free_shadow(page, cachep->gfporder);
P
Pekka Enberg 已提交
1574

1575 1576 1577 1578 1579 1580
	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
		sub_zone_page_state(page_zone(page),
				NR_SLAB_RECLAIMABLE, nr_freed);
	else
		sub_zone_page_state(page_zone(page),
				NR_SLAB_UNRECLAIMABLE, nr_freed);
J
Joonsoo Kim 已提交
1581

1582
	BUG_ON(!PageSlab(page));
J
Joonsoo Kim 已提交
1583
	__ClearPageSlabPfmemalloc(page);
1584
	__ClearPageSlab(page);
1585 1586
	page_mapcount_reset(page);
	page->mapping = NULL;
G
Glauber Costa 已提交
1587

L
Linus Torvalds 已提交
1588 1589
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += nr_freed;
1590
	__free_kmem_pages(page, cachep->gfporder);
L
Linus Torvalds 已提交
1591 1592 1593 1594
}

static void kmem_rcu_free(struct rcu_head *head)
{
1595 1596
	struct kmem_cache *cachep;
	struct page *page;
L
Linus Torvalds 已提交
1597

1598 1599 1600 1601
	page = container_of(head, struct page, rcu_head);
	cachep = page->slab_cache;

	kmem_freepages(cachep, page);
L
Linus Torvalds 已提交
1602 1603 1604
}

#if DEBUG
1605 1606 1607 1608 1609 1610 1611 1612
static bool is_debug_pagealloc_cache(struct kmem_cache *cachep)
{
	if (debug_pagealloc_enabled() && OFF_SLAB(cachep) &&
		(cachep->size % PAGE_SIZE) == 0)
		return true;

	return false;
}
L
Linus Torvalds 已提交
1613 1614

#ifdef CONFIG_DEBUG_PAGEALLOC
1615
static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
P
Pekka Enberg 已提交
1616
			    unsigned long caller)
L
Linus Torvalds 已提交
1617
{
1618
	int size = cachep->object_size;
L
Linus Torvalds 已提交
1619

1620
	addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
L
Linus Torvalds 已提交
1621

P
Pekka Enberg 已提交
1622
	if (size < 5 * sizeof(unsigned long))
L
Linus Torvalds 已提交
1623 1624
		return;

P
Pekka Enberg 已提交
1625 1626 1627 1628
	*addr++ = 0x12345678;
	*addr++ = caller;
	*addr++ = smp_processor_id();
	size -= 3 * sizeof(unsigned long);
L
Linus Torvalds 已提交
1629 1630 1631 1632 1633 1634 1635
	{
		unsigned long *sptr = &caller;
		unsigned long svalue;

		while (!kstack_end(sptr)) {
			svalue = *sptr++;
			if (kernel_text_address(svalue)) {
P
Pekka Enberg 已提交
1636
				*addr++ = svalue;
L
Linus Torvalds 已提交
1637 1638 1639 1640 1641 1642 1643
				size -= sizeof(unsigned long);
				if (size <= sizeof(unsigned long))
					break;
			}
		}

	}
P
Pekka Enberg 已提交
1644
	*addr++ = 0x87654321;
L
Linus Torvalds 已提交
1645
}
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662

static void slab_kernel_map(struct kmem_cache *cachep, void *objp,
				int map, unsigned long caller)
{
	if (!is_debug_pagealloc_cache(cachep))
		return;

	if (caller)
		store_stackinfo(cachep, objp, caller);

	kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
}

#else
static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,
				int map, unsigned long caller) {}

L
Linus Torvalds 已提交
1663 1664
#endif

1665
static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
L
Linus Torvalds 已提交
1666
{
1667
	int size = cachep->object_size;
1668
	addr = &((char *)addr)[obj_offset(cachep)];
L
Linus Torvalds 已提交
1669 1670

	memset(addr, val, size);
P
Pekka Enberg 已提交
1671
	*(unsigned char *)(addr + size - 1) = POISON_END;
L
Linus Torvalds 已提交
1672 1673 1674 1675 1676
}

static void dump_line(char *data, int offset, int limit)
{
	int i;
D
Dave Jones 已提交
1677 1678 1679
	unsigned char error = 0;
	int bad_count = 0;

1680
	printk(KERN_ERR "%03x: ", offset);
D
Dave Jones 已提交
1681 1682 1683 1684 1685 1686
	for (i = 0; i < limit; i++) {
		if (data[offset + i] != POISON_FREE) {
			error = data[offset + i];
			bad_count++;
		}
	}
1687 1688
	print_hex_dump(KERN_CONT, "", 0, 16, 1,
			&data[offset], limit, 1);
D
Dave Jones 已提交
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702

	if (bad_count == 1) {
		error ^= POISON_FREE;
		if (!(error & (error - 1))) {
			printk(KERN_ERR "Single bit error detected. Probably "
					"bad RAM.\n");
#ifdef CONFIG_X86
			printk(KERN_ERR "Run memtest86+ or a similar memory "
					"test tool.\n");
#else
			printk(KERN_ERR "Run a memory test tool.\n");
#endif
		}
	}
L
Linus Torvalds 已提交
1703 1704 1705 1706 1707
}
#endif

#if DEBUG

1708
static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
L
Linus Torvalds 已提交
1709 1710 1711 1712 1713
{
	int i, size;
	char *realobj;

	if (cachep->flags & SLAB_RED_ZONE) {
1714
		printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
A
Andrew Morton 已提交
1715 1716
			*dbg_redzone1(cachep, objp),
			*dbg_redzone2(cachep, objp));
L
Linus Torvalds 已提交
1717 1718 1719
	}

	if (cachep->flags & SLAB_STORE_USER) {
J
Joe Perches 已提交
1720 1721 1722
		printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
		       *dbg_userword(cachep, objp),
		       *dbg_userword(cachep, objp));
L
Linus Torvalds 已提交
1723
	}
1724
	realobj = (char *)objp + obj_offset(cachep);
1725
	size = cachep->object_size;
P
Pekka Enberg 已提交
1726
	for (i = 0; i < size && lines; i += 16, lines--) {
L
Linus Torvalds 已提交
1727 1728
		int limit;
		limit = 16;
P
Pekka Enberg 已提交
1729 1730
		if (i + limit > size)
			limit = size - i;
L
Linus Torvalds 已提交
1731 1732 1733 1734
		dump_line(realobj, i, limit);
	}
}

1735
static void check_poison_obj(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
1736 1737 1738 1739 1740
{
	char *realobj;
	int size, i;
	int lines = 0;

1741 1742 1743
	if (is_debug_pagealloc_cache(cachep))
		return;

1744
	realobj = (char *)objp + obj_offset(cachep);
1745
	size = cachep->object_size;
L
Linus Torvalds 已提交
1746

P
Pekka Enberg 已提交
1747
	for (i = 0; i < size; i++) {
L
Linus Torvalds 已提交
1748
		char exp = POISON_FREE;
P
Pekka Enberg 已提交
1749
		if (i == size - 1)
L
Linus Torvalds 已提交
1750 1751 1752 1753 1754 1755
			exp = POISON_END;
		if (realobj[i] != exp) {
			int limit;
			/* Mismatch ! */
			/* Print header */
			if (lines == 0) {
P
Pekka Enberg 已提交
1756
				printk(KERN_ERR
1757 1758
					"Slab corruption (%s): %s start=%p, len=%d\n",
					print_tainted(), cachep->name, realobj, size);
L
Linus Torvalds 已提交
1759 1760 1761
				print_objinfo(cachep, objp, 0);
			}
			/* Hexdump the affected line */
P
Pekka Enberg 已提交
1762
			i = (i / 16) * 16;
L
Linus Torvalds 已提交
1763
			limit = 16;
P
Pekka Enberg 已提交
1764 1765
			if (i + limit > size)
				limit = size - i;
L
Linus Torvalds 已提交
1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
			dump_line(realobj, i, limit);
			i += 16;
			lines++;
			/* Limit to 5 lines */
			if (lines > 5)
				break;
		}
	}
	if (lines != 0) {
		/* Print some data about the neighboring objects, if they
		 * exist:
		 */
1778
		struct page *page = virt_to_head_page(objp);
1779
		unsigned int objnr;
L
Linus Torvalds 已提交
1780

1781
		objnr = obj_to_index(cachep, page, objp);
L
Linus Torvalds 已提交
1782
		if (objnr) {
1783
			objp = index_to_obj(cachep, page, objnr - 1);
1784
			realobj = (char *)objp + obj_offset(cachep);
L
Linus Torvalds 已提交
1785
			printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
P
Pekka Enberg 已提交
1786
			       realobj, size);
L
Linus Torvalds 已提交
1787 1788
			print_objinfo(cachep, objp, 2);
		}
P
Pekka Enberg 已提交
1789
		if (objnr + 1 < cachep->num) {
1790
			objp = index_to_obj(cachep, page, objnr + 1);
1791
			realobj = (char *)objp + obj_offset(cachep);
L
Linus Torvalds 已提交
1792
			printk(KERN_ERR "Next obj: start=%p, len=%d\n",
P
Pekka Enberg 已提交
1793
			       realobj, size);
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798 1799
			print_objinfo(cachep, objp, 2);
		}
	}
}
#endif

1800
#if DEBUG
1801 1802
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
						struct page *page)
L
Linus Torvalds 已提交
1803 1804 1805
{
	int i;
	for (i = 0; i < cachep->num; i++) {
1806
		void *objp = index_to_obj(cachep, page, i);
L
Linus Torvalds 已提交
1807 1808 1809

		if (cachep->flags & SLAB_POISON) {
			check_poison_obj(cachep, objp);
1810
			slab_kernel_map(cachep, objp, 1, 0);
L
Linus Torvalds 已提交
1811 1812 1813 1814
		}
		if (cachep->flags & SLAB_RED_ZONE) {
			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "start of a freed object "
P
Pekka Enberg 已提交
1815
					   "was overwritten");
L
Linus Torvalds 已提交
1816 1817
			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "end of a freed object "
P
Pekka Enberg 已提交
1818
					   "was overwritten");
L
Linus Torvalds 已提交
1819 1820
		}
	}
1821
}
L
Linus Torvalds 已提交
1822
#else
1823 1824
static void slab_destroy_debugcheck(struct kmem_cache *cachep,
						struct page *page)
1825 1826
{
}
L
Linus Torvalds 已提交
1827 1828
#endif

1829 1830 1831
/**
 * slab_destroy - destroy and release all objects in a slab
 * @cachep: cache pointer being destroyed
1832
 * @page: page pointer being destroyed
1833
 *
W
Wang Sheng-Hui 已提交
1834 1835 1836
 * Destroy all the objs in a slab page, and release the mem back to the system.
 * Before calling the slab page must have been unlinked from the cache. The
 * kmem_cache_node ->list_lock is not held/needed.
1837
 */
1838
static void slab_destroy(struct kmem_cache *cachep, struct page *page)
1839
{
1840
	void *freelist;
1841

1842 1843
	freelist = page->freelist;
	slab_destroy_debugcheck(cachep, page);
1844 1845 1846
	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
		call_rcu(&page->rcu_head, kmem_rcu_free);
	else
1847
		kmem_freepages(cachep, page);
1848 1849

	/*
1850
	 * From now on, we don't use freelist
1851 1852 1853
	 * although actual page can be freed in rcu context
	 */
	if (OFF_SLAB(cachep))
1854
		kmem_cache_free(cachep->freelist_cache, freelist);
L
Linus Torvalds 已提交
1855 1856
}

1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
{
	struct page *page, *n;

	list_for_each_entry_safe(page, n, list, lru) {
		list_del(&page->lru);
		slab_destroy(cachep, page);
	}
}

1867
/**
1868 1869 1870 1871 1872 1873
 * calculate_slab_order - calculate size (page order) of slabs
 * @cachep: pointer to the cache that is being created
 * @size: size of objects to be created in this cache.
 * @flags: slab allocation flags
 *
 * Also calculates the number of objects per slab.
1874 1875 1876 1877 1878
 *
 * This could be made much more intelligent.  For now, try to avoid using
 * high order pages for slabs.  When the gfp() functions are more friendly
 * towards high-order requests, this should be changed.
 */
A
Andrew Morton 已提交
1879
static size_t calculate_slab_order(struct kmem_cache *cachep,
1880
				size_t size, unsigned long flags)
1881
{
1882
	unsigned long offslab_limit;
1883
	size_t left_over = 0;
1884
	int gfporder;
1885

1886
	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
1887 1888 1889
		unsigned int num;
		size_t remainder;

1890
		cache_estimate(gfporder, size, flags, &remainder, &num);
1891 1892
		if (!num)
			continue;
1893

1894 1895 1896 1897
		/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
		if (num > SLAB_OBJ_MAX_NUM)
			break;

1898 1899 1900 1901 1902 1903
		if (flags & CFLGS_OFF_SLAB) {
			/*
			 * Max number of objs-per-slab for caches which
			 * use off-slab slabs. Needed to avoid a possible
			 * looping condition in cache_grow().
			 */
1904
			offslab_limit = size;
1905
			offslab_limit /= sizeof(freelist_idx_t);
1906 1907 1908 1909

 			if (num > offslab_limit)
				break;
		}
1910

1911
		/* Found something acceptable - save it away */
1912
		cachep->num = num;
1913
		cachep->gfporder = gfporder;
1914 1915
		left_over = remainder;

1916 1917 1918 1919 1920 1921 1922 1923
		/*
		 * A VFS-reclaimable slab tends to have most allocations
		 * as GFP_NOFS and we really don't want to have to be allocating
		 * higher-order pages when we are unable to shrink dcache.
		 */
		if (flags & SLAB_RECLAIM_ACCOUNT)
			break;

1924 1925 1926 1927
		/*
		 * Large number of objects is good, but very large slabs are
		 * currently bad for the gfp()s.
		 */
1928
		if (gfporder >= slab_max_order)
1929 1930
			break;

1931 1932 1933
		/*
		 * Acceptable internal fragmentation?
		 */
A
Andrew Morton 已提交
1934
		if (left_over * 8 <= (PAGE_SIZE << gfporder))
1935 1936 1937 1938 1939
			break;
	}
	return left_over;
}

1940 1941 1942 1943 1944 1945 1946 1947
static struct array_cache __percpu *alloc_kmem_cache_cpus(
		struct kmem_cache *cachep, int entries, int batchcount)
{
	int cpu;
	size_t size;
	struct array_cache __percpu *cpu_cache;

	size = sizeof(void *) * entries + sizeof(struct array_cache);
1948
	cpu_cache = __alloc_percpu(size, sizeof(void *));
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960

	if (!cpu_cache)
		return NULL;

	for_each_possible_cpu(cpu) {
		init_arraycache(per_cpu_ptr(cpu_cache, cpu),
				entries, batchcount);
	}

	return cpu_cache;
}

1961
static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
1962
{
1963
	if (slab_state >= FULL)
1964
		return enable_cpucache(cachep, gfp);
1965

1966 1967 1968 1969
	cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
	if (!cachep->cpu_cache)
		return 1;

1970
	if (slab_state == DOWN) {
1971 1972
		/* Creation of first cache (kmem_cache). */
		set_up_node(kmem_cache, CACHE_CACHE);
1973
	} else if (slab_state == PARTIAL) {
1974 1975
		/* For kmem_cache_node */
		set_up_node(cachep, SIZE_NODE);
1976
	} else {
1977
		int node;
1978

1979 1980 1981 1982 1983
		for_each_online_node(node) {
			cachep->node[node] = kmalloc_node(
				sizeof(struct kmem_cache_node), gfp, node);
			BUG_ON(!cachep->node[node]);
			kmem_cache_node_init(cachep->node[node]);
1984 1985
		}
	}
1986

1987
	cachep->node[numa_mem_id()]->next_reap =
1988 1989
			jiffies + REAPTIMEOUT_NODE +
			((unsigned long)cachep) % REAPTIMEOUT_NODE;
1990 1991 1992 1993 1994 1995 1996

	cpu_cache_get(cachep)->avail = 0;
	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
	cpu_cache_get(cachep)->batchcount = 1;
	cpu_cache_get(cachep)->touched = 0;
	cachep->batchcount = 1;
	cachep->limit = BOOT_CPUCACHE_ENTRIES;
1997
	return 0;
1998 1999
}

J
Joonsoo Kim 已提交
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
unsigned long kmem_cache_flags(unsigned long object_size,
	unsigned long flags, const char *name,
	void (*ctor)(void *))
{
	return flags;
}

struct kmem_cache *
__kmem_cache_alias(const char *name, size_t size, size_t align,
		   unsigned long flags, void (*ctor)(void *))
{
	struct kmem_cache *cachep;

	cachep = find_mergeable(size, align, flags, name, ctor);
	if (cachep) {
		cachep->refcount++;

		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
		cachep->object_size = max_t(int, cachep->object_size, size);
	}
	return cachep;
}

L
Linus Torvalds 已提交
2026
/**
2027
 * __kmem_cache_create - Create a cache.
R
Randy Dunlap 已提交
2028
 * @cachep: cache management descriptor
L
Linus Torvalds 已提交
2029 2030 2031 2032
 * @flags: SLAB flags
 *
 * Returns a ptr to the cache on success, NULL on failure.
 * Cannot be called within a int, but can be interrupted.
2033
 * The @ctor is run when new pages are allocated by the cache.
L
Linus Torvalds 已提交
2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
 *
 * The flags are
 *
 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
 * to catch references to uninitialised memory.
 *
 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
 * for buffer overruns.
 *
 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
 * cacheline.  This can be beneficial if you're counting cycles as closely
 * as davem.
 */
2047
int
2048
__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
L
Linus Torvalds 已提交
2049
{
2050 2051
	size_t left_over, freelist_size;
	size_t ralign = BYTES_PER_WORD;
2052
	gfp_t gfp;
2053
	int err;
2054
	size_t size = cachep->size;
L
Linus Torvalds 已提交
2055 2056 2057 2058 2059 2060 2061 2062 2063

#if DEBUG
#if FORCED_DEBUG
	/*
	 * Enable redzoning and last user accounting, except for caches with
	 * large objects, if the increased size would increase the object size
	 * above the next power of two: caches with object sizes just above a
	 * power of two have a significant amount of internal fragmentation.
	 */
D
David Woodhouse 已提交
2064 2065
	if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
						2 * sizeof(unsigned long long)))
P
Pekka Enberg 已提交
2066
		flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
L
Linus Torvalds 已提交
2067 2068 2069 2070 2071
	if (!(flags & SLAB_DESTROY_BY_RCU))
		flags |= SLAB_POISON;
#endif
#endif

A
Andrew Morton 已提交
2072 2073
	/*
	 * Check that size is in terms of words.  This is needed to avoid
L
Linus Torvalds 已提交
2074 2075 2076
	 * unaligned accesses for some archs when redzoning is used, and makes
	 * sure any on-slab bufctl's are also correctly aligned.
	 */
P
Pekka Enberg 已提交
2077 2078 2079
	if (size & (BYTES_PER_WORD - 1)) {
		size += (BYTES_PER_WORD - 1);
		size &= ~(BYTES_PER_WORD - 1);
L
Linus Torvalds 已提交
2080 2081
	}

D
David Woodhouse 已提交
2082 2083 2084 2085 2086 2087 2088
	if (flags & SLAB_RED_ZONE) {
		ralign = REDZONE_ALIGN;
		/* If redzoning, ensure that the second redzone is suitably
		 * aligned, by adjusting the object size accordingly. */
		size += REDZONE_ALIGN - 1;
		size &= ~(REDZONE_ALIGN - 1);
	}
2089

2090
	/* 3) caller mandated alignment */
2091 2092
	if (ralign < cachep->align) {
		ralign = cachep->align;
L
Linus Torvalds 已提交
2093
	}
2094 2095
	/* disable debug if necessary */
	if (ralign > __alignof__(unsigned long long))
2096
		flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
A
Andrew Morton 已提交
2097
	/*
2098
	 * 4) Store it.
L
Linus Torvalds 已提交
2099
	 */
2100
	cachep->align = ralign;
L
Linus Torvalds 已提交
2101

2102 2103 2104 2105 2106
	if (slab_is_available())
		gfp = GFP_KERNEL;
	else
		gfp = GFP_NOWAIT;

L
Linus Torvalds 已提交
2107 2108
#if DEBUG

2109 2110 2111 2112
	/*
	 * Both debugging options require word-alignment which is calculated
	 * into align above.
	 */
L
Linus Torvalds 已提交
2113 2114
	if (flags & SLAB_RED_ZONE) {
		/* add space for red zone words */
2115 2116
		cachep->obj_offset += sizeof(unsigned long long);
		size += 2 * sizeof(unsigned long long);
L
Linus Torvalds 已提交
2117 2118
	}
	if (flags & SLAB_STORE_USER) {
2119
		/* user store requires one word storage behind the end of
D
David Woodhouse 已提交
2120 2121
		 * the real object. But if the second red zone needs to be
		 * aligned to 64 bits, we must allow that much space.
L
Linus Torvalds 已提交
2122
		 */
D
David Woodhouse 已提交
2123 2124 2125 2126
		if (flags & SLAB_RED_ZONE)
			size += REDZONE_ALIGN;
		else
			size += BYTES_PER_WORD;
L
Linus Torvalds 已提交
2127
	}
2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
#endif

	size = ALIGN(size, cachep->align);
	/*
	 * We should restrict the number of objects in a slab to implement
	 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
	 */
	if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
		size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);

#if DEBUG
2139 2140 2141 2142 2143 2144 2145
	/*
	 * To activate debug pagealloc, off-slab management is necessary
	 * requirement. In early phase of initialization, small sized slab
	 * doesn't get initialized so it would not be possible. So, we need
	 * to check size >= 256. It guarantees that all necessary small
	 * sized slab is initialized in current slab initialization sequence.
	 */
2146
	if (debug_pagealloc_enabled() && (flags & SLAB_POISON) &&
2147
		!slab_early_init && size >= kmalloc_size(INDEX_NODE) &&
2148
		size >= 256 && cachep->object_size > cache_line_size() &&
2149 2150
		size < PAGE_SIZE) {
		cachep->obj_offset += PAGE_SIZE - size;
L
Linus Torvalds 已提交
2151 2152 2153 2154
		size = PAGE_SIZE;
	}
#endif

2155 2156 2157
	/*
	 * Determine if the slab management is 'on' or 'off' slab.
	 * (bootstrapping cannot cope with offslab caches so don't do
2158 2159
	 * it too early on. Always use on-slab management when
	 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
2160
	 */
2161
	if (size >= OFF_SLAB_MIN_SIZE && !slab_early_init &&
2162
	    !(flags & SLAB_NOLEAKTRACE)) {
L
Linus Torvalds 已提交
2163 2164 2165 2166 2167
		/*
		 * Size is large, assume best to place the slab management obj
		 * off-slab (should allow better packing of objs).
		 */
		flags |= CFLGS_OFF_SLAB;
2168
	}
L
Linus Torvalds 已提交
2169

2170
	left_over = calculate_slab_order(cachep, size, flags);
L
Linus Torvalds 已提交
2171

2172
	if (!cachep->num)
2173
		return -E2BIG;
L
Linus Torvalds 已提交
2174

2175
	freelist_size = cachep->num * sizeof(freelist_idx_t);
L
Linus Torvalds 已提交
2176 2177 2178 2179 2180

	/*
	 * If the slab has been placed off-slab, and we have enough space then
	 * move it on-slab. This is at the expense of any extra colouring.
	 */
2181
	if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
L
Linus Torvalds 已提交
2182
		flags &= ~CFLGS_OFF_SLAB;
2183
		left_over -= freelist_size;
L
Linus Torvalds 已提交
2184 2185 2186 2187
	}

	cachep->colour_off = cache_line_size();
	/* Offset must be a multiple of the alignment. */
2188 2189
	if (cachep->colour_off < cachep->align)
		cachep->colour_off = cachep->align;
P
Pekka Enberg 已提交
2190
	cachep->colour = left_over / cachep->colour_off;
2191
	cachep->freelist_size = freelist_size;
L
Linus Torvalds 已提交
2192
	cachep->flags = flags;
2193
	cachep->allocflags = __GFP_COMP;
2194
	if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
2195
		cachep->allocflags |= GFP_DMA;
2196
	cachep->size = size;
2197
	cachep->reciprocal_buffer_size = reciprocal_value(size);
L
Linus Torvalds 已提交
2198

2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
#if DEBUG
	/*
	 * If we're going to use the generic kernel_map_pages()
	 * poisoning, then it's going to smash the contents of
	 * the redzone and userword anyhow, so switch them off.
	 */
	if (IS_ENABLED(CONFIG_PAGE_POISONING) &&
		(cachep->flags & SLAB_POISON) &&
		is_debug_pagealloc_cache(cachep))
		cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
#endif

	if (OFF_SLAB(cachep)) {
2212
		cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
2213
		/*
2214
		 * This is a possibility for one of the kmalloc_{dma,}_caches.
2215
		 * But since we go off slab only for object size greater than
2216
		 * OFF_SLAB_MIN_SIZE, and kmalloc_{dma,}_caches get created
2217
		 * in ascending order,this should not happen at all.
2218 2219
		 * But leave a BUG_ON for some lucky dude.
		 */
2220
		BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
2221
	}
L
Linus Torvalds 已提交
2222

2223 2224
	err = setup_cpu_cache(cachep, gfp);
	if (err) {
2225
		__kmem_cache_release(cachep);
2226
		return err;
2227
	}
L
Linus Torvalds 已提交
2228

2229
	return 0;
L
Linus Torvalds 已提交
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
}

#if DEBUG
static void check_irq_off(void)
{
	BUG_ON(!irqs_disabled());
}

static void check_irq_on(void)
{
	BUG_ON(irqs_disabled());
}

2243
static void check_spinlock_acquired(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2244 2245 2246
{
#ifdef CONFIG_SMP
	check_irq_off();
2247
	assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
L
Linus Torvalds 已提交
2248 2249
#endif
}
2250

2251
static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2252 2253 2254
{
#ifdef CONFIG_SMP
	check_irq_off();
2255
	assert_spin_locked(&get_node(cachep, node)->list_lock);
2256 2257 2258
#endif
}

L
Linus Torvalds 已提交
2259 2260 2261 2262
#else
#define check_irq_off()	do { } while(0)
#define check_irq_on()	do { } while(0)
#define check_spinlock_acquired(x) do { } while(0)
2263
#define check_spinlock_acquired_node(x, y) do { } while(0)
L
Linus Torvalds 已提交
2264 2265
#endif

2266
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
2267 2268 2269
			struct array_cache *ac,
			int force, int node);

L
Linus Torvalds 已提交
2270 2271
static void do_drain(void *arg)
{
A
Andrew Morton 已提交
2272
	struct kmem_cache *cachep = arg;
L
Linus Torvalds 已提交
2273
	struct array_cache *ac;
2274
	int node = numa_mem_id();
2275
	struct kmem_cache_node *n;
2276
	LIST_HEAD(list);
L
Linus Torvalds 已提交
2277 2278

	check_irq_off();
2279
	ac = cpu_cache_get(cachep);
2280 2281
	n = get_node(cachep, node);
	spin_lock(&n->list_lock);
2282
	free_block(cachep, ac->entry, ac->avail, node, &list);
2283
	spin_unlock(&n->list_lock);
2284
	slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
2285 2286 2287
	ac->avail = 0;
}

2288
static void drain_cpu_caches(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2289
{
2290
	struct kmem_cache_node *n;
2291 2292
	int node;

2293
	on_each_cpu(do_drain, cachep, 1);
L
Linus Torvalds 已提交
2294
	check_irq_on();
2295 2296
	for_each_kmem_cache_node(cachep, node, n)
		if (n->alien)
2297
			drain_alien_cache(cachep, n->alien);
2298

2299 2300
	for_each_kmem_cache_node(cachep, node, n)
		drain_array(cachep, n, n->shared, 1, node);
L
Linus Torvalds 已提交
2301 2302
}

2303 2304 2305 2306 2307 2308 2309
/*
 * Remove slabs from the list of free slabs.
 * Specify the number of slabs to drain in tofree.
 *
 * Returns the actual number of slabs released.
 */
static int drain_freelist(struct kmem_cache *cache,
2310
			struct kmem_cache_node *n, int tofree)
L
Linus Torvalds 已提交
2311
{
2312 2313
	struct list_head *p;
	int nr_freed;
2314
	struct page *page;
L
Linus Torvalds 已提交
2315

2316
	nr_freed = 0;
2317
	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
L
Linus Torvalds 已提交
2318

2319 2320 2321 2322
		spin_lock_irq(&n->list_lock);
		p = n->slabs_free.prev;
		if (p == &n->slabs_free) {
			spin_unlock_irq(&n->list_lock);
2323 2324
			goto out;
		}
L
Linus Torvalds 已提交
2325

2326 2327
		page = list_entry(p, struct page, lru);
		list_del(&page->lru);
2328 2329 2330 2331
		/*
		 * Safe to drop the lock. The slab is no longer linked
		 * to the cache.
		 */
2332 2333
		n->free_objects -= cache->num;
		spin_unlock_irq(&n->list_lock);
2334
		slab_destroy(cache, page);
2335
		nr_freed++;
L
Linus Torvalds 已提交
2336
	}
2337 2338
out:
	return nr_freed;
L
Linus Torvalds 已提交
2339 2340
}

2341
int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
2342
{
2343 2344
	int ret = 0;
	int node;
2345
	struct kmem_cache_node *n;
2346 2347 2348 2349

	drain_cpu_caches(cachep);

	check_irq_on();
2350
	for_each_kmem_cache_node(cachep, node, n) {
2351
		drain_freelist(cachep, n, slabs_tofree(cachep, n));
2352

2353 2354
		ret += !list_empty(&n->slabs_full) ||
			!list_empty(&n->slabs_partial);
2355 2356 2357 2358
	}
	return (ret ? 1 : 0);
}

2359
int __kmem_cache_shutdown(struct kmem_cache *cachep)
2360 2361 2362 2363 2364
{
	return __kmem_cache_shrink(cachep, false);
}

void __kmem_cache_release(struct kmem_cache *cachep)
L
Linus Torvalds 已提交
2365
{
2366
	int i;
2367
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2368

2369
	free_percpu(cachep->cpu_cache);
L
Linus Torvalds 已提交
2370

2371
	/* NUMA: free the node structures */
2372 2373 2374 2375 2376
	for_each_kmem_cache_node(cachep, i, n) {
		kfree(n->shared);
		free_alien_cache(n->alien);
		kfree(n);
		cachep->node[i] = NULL;
2377
	}
L
Linus Torvalds 已提交
2378 2379
}

2380 2381
/*
 * Get the memory for a slab management obj.
2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392
 *
 * For a slab cache when the slab descriptor is off-slab, the
 * slab descriptor can't come from the same cache which is being created,
 * Because if it is the case, that means we defer the creation of
 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
 * And we eventually call down to __kmem_cache_create(), which
 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
 * This is a "chicken-and-egg" problem.
 *
 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
 * which are all initialized during kmem_cache_init().
2393
 */
2394
static void *alloc_slabmgmt(struct kmem_cache *cachep,
2395 2396
				   struct page *page, int colour_off,
				   gfp_t local_flags, int nodeid)
L
Linus Torvalds 已提交
2397
{
2398
	void *freelist;
2399
	void *addr = page_address(page);
P
Pekka Enberg 已提交
2400

2401 2402 2403
	page->s_mem = addr + colour_off;
	page->active = 0;

L
Linus Torvalds 已提交
2404 2405
	if (OFF_SLAB(cachep)) {
		/* Slab management obj is off-slab. */
2406
		freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2407
					      local_flags, nodeid);
2408
		if (!freelist)
L
Linus Torvalds 已提交
2409 2410
			return NULL;
	} else {
2411 2412 2413
		/* We will use last bytes at the slab for freelist */
		freelist = addr + (PAGE_SIZE << cachep->gfporder) -
				cachep->freelist_size;
L
Linus Torvalds 已提交
2414
	}
2415

2416
	return freelist;
L
Linus Torvalds 已提交
2417 2418
}

2419
static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
L
Linus Torvalds 已提交
2420
{
2421
	return ((freelist_idx_t *)page->freelist)[idx];
2422 2423 2424
}

static inline void set_free_obj(struct page *page,
2425
					unsigned int idx, freelist_idx_t val)
2426
{
2427
	((freelist_idx_t *)(page->freelist))[idx] = val;
L
Linus Torvalds 已提交
2428 2429
}

2430
static void cache_init_objs(struct kmem_cache *cachep,
2431
			    struct page *page)
L
Linus Torvalds 已提交
2432 2433 2434 2435
{
	int i;

	for (i = 0; i < cachep->num; i++) {
2436
		void *objp = index_to_obj(cachep, page, i);
L
Linus Torvalds 已提交
2437 2438 2439 2440 2441 2442 2443 2444 2445
#if DEBUG
		if (cachep->flags & SLAB_STORE_USER)
			*dbg_userword(cachep, objp) = NULL;

		if (cachep->flags & SLAB_RED_ZONE) {
			*dbg_redzone1(cachep, objp) = RED_INACTIVE;
			*dbg_redzone2(cachep, objp) = RED_INACTIVE;
		}
		/*
A
Andrew Morton 已提交
2446 2447 2448
		 * Constructors are not allowed to allocate memory from the same
		 * cache which they are a constructor for.  Otherwise, deadlock.
		 * They must also be threaded.
L
Linus Torvalds 已提交
2449 2450
		 */
		if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2451
			cachep->ctor(objp + obj_offset(cachep));
L
Linus Torvalds 已提交
2452 2453 2454 2455

		if (cachep->flags & SLAB_RED_ZONE) {
			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "constructor overwrote the"
P
Pekka Enberg 已提交
2456
					   " end of an object");
L
Linus Torvalds 已提交
2457 2458
			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
				slab_error(cachep, "constructor overwrote the"
P
Pekka Enberg 已提交
2459
					   " start of an object");
L
Linus Torvalds 已提交
2460
		}
2461 2462 2463 2464 2465
		/* need to poison the objs? */
		if (cachep->flags & SLAB_POISON) {
			poison_obj(cachep, objp, POISON_FREE);
			slab_kernel_map(cachep, objp, 0, 0);
		}
L
Linus Torvalds 已提交
2466 2467
#else
		if (cachep->ctor)
2468
			cachep->ctor(objp);
L
Linus Torvalds 已提交
2469
#endif
2470
		set_free_obj(page, i, i);
L
Linus Torvalds 已提交
2471 2472 2473
	}
}

2474
static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
2475
{
2476 2477
	if (CONFIG_ZONE_DMA_FLAG) {
		if (flags & GFP_DMA)
2478
			BUG_ON(!(cachep->allocflags & GFP_DMA));
2479
		else
2480
			BUG_ON(cachep->allocflags & GFP_DMA);
2481
	}
L
Linus Torvalds 已提交
2482 2483
}

2484
static void *slab_get_obj(struct kmem_cache *cachep, struct page *page)
2485
{
2486
	void *objp;
2487

2488
	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2489
	page->active++;
2490

2491 2492 2493 2494 2495
#if DEBUG
	if (cachep->flags & SLAB_STORE_USER)
		set_store_user_dirty(cachep);
#endif

2496 2497 2498
	return objp;
}

2499 2500
static void slab_put_obj(struct kmem_cache *cachep,
			struct page *page, void *objp)
2501
{
2502
	unsigned int objnr = obj_to_index(cachep, page, objp);
2503
#if DEBUG
J
Joonsoo Kim 已提交
2504
	unsigned int i;
2505 2506

	/* Verify double free bug */
2507
	for (i = page->active; i < cachep->num; i++) {
2508
		if (get_free_obj(page, i) == objnr) {
2509 2510 2511 2512
			printk(KERN_ERR "slab: double free detected in cache "
					"'%s', objp %p\n", cachep->name, objp);
			BUG();
		}
2513 2514
	}
#endif
2515
	page->active--;
2516
	set_free_obj(page, page->active, objnr);
2517 2518
}

2519 2520 2521
/*
 * Map pages beginning at addr to the given cache and slab. This is required
 * for the slab allocator to be able to lookup the cache and slab of a
2522
 * virtual address for kfree, ksize, and slab debugging.
2523
 */
2524
static void slab_map_pages(struct kmem_cache *cache, struct page *page,
2525
			   void *freelist)
L
Linus Torvalds 已提交
2526
{
2527
	page->slab_cache = cache;
2528
	page->freelist = freelist;
L
Linus Torvalds 已提交
2529 2530 2531 2532 2533 2534
}

/*
 * Grow (by 1) the number of slabs within a cache.  This is called by
 * kmem_cache_alloc() when there are no active objs left in a cache.
 */
2535
static int cache_grow(struct kmem_cache *cachep,
2536
		gfp_t flags, int nodeid, struct page *page)
L
Linus Torvalds 已提交
2537
{
2538
	void *freelist;
P
Pekka Enberg 已提交
2539 2540
	size_t offset;
	gfp_t local_flags;
2541
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2542

A
Andrew Morton 已提交
2543 2544 2545
	/*
	 * Be lazy and only check for valid flags here,  keeping it out of the
	 * critical path in kmem_cache_alloc().
L
Linus Torvalds 已提交
2546
	 */
2547 2548 2549 2550
	if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
		pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
		BUG();
	}
C
Christoph Lameter 已提交
2551
	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
L
Linus Torvalds 已提交
2552

2553
	/* Take the node list lock to change the colour_next on this node */
L
Linus Torvalds 已提交
2554
	check_irq_off();
2555
	n = get_node(cachep, nodeid);
2556
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2557 2558

	/* Get colour for the slab, and cal the next value. */
2559 2560 2561 2562 2563
	offset = n->colour_next;
	n->colour_next++;
	if (n->colour_next >= cachep->colour)
		n->colour_next = 0;
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2564

2565
	offset *= cachep->colour_off;
L
Linus Torvalds 已提交
2566

2567
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
		local_irq_enable();

	/*
	 * The test for missing atomic flag is performed here, rather than
	 * the more obvious place, simply to reduce the critical path length
	 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
	 * will eventually be caught here (where it matters).
	 */
	kmem_flagcheck(cachep, flags);

A
Andrew Morton 已提交
2578 2579 2580
	/*
	 * Get mem for the objs.  Attempt to allocate a physical page from
	 * 'nodeid'.
2581
	 */
2582 2583 2584
	if (!page)
		page = kmem_getpages(cachep, local_flags, nodeid);
	if (!page)
L
Linus Torvalds 已提交
2585 2586 2587
		goto failed;

	/* Get slab management. */
2588
	freelist = alloc_slabmgmt(cachep, page, offset,
C
Christoph Lameter 已提交
2589
			local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
2590
	if (!freelist)
L
Linus Torvalds 已提交
2591 2592
		goto opps1;

2593
	slab_map_pages(cachep, page, freelist);
L
Linus Torvalds 已提交
2594

2595
	cache_init_objs(cachep, page);
L
Linus Torvalds 已提交
2596

2597
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2598 2599
		local_irq_disable();
	check_irq_off();
2600
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2601 2602

	/* Make slab active. */
2603
	list_add_tail(&page->lru, &(n->slabs_free));
L
Linus Torvalds 已提交
2604
	STATS_INC_GROWN(cachep);
2605 2606
	n->free_objects += cachep->num;
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2607
	return 1;
A
Andrew Morton 已提交
2608
opps1:
2609
	kmem_freepages(cachep, page);
A
Andrew Morton 已提交
2610
failed:
2611
	if (gfpflags_allow_blocking(local_flags))
L
Linus Torvalds 已提交
2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626
		local_irq_disable();
	return 0;
}

#if DEBUG

/*
 * Perform extra freeing checks:
 * - detect bad pointers.
 * - POISON/RED_ZONE checking
 */
static void kfree_debugcheck(const void *objp)
{
	if (!virt_addr_valid(objp)) {
		printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
P
Pekka Enberg 已提交
2627 2628
		       (unsigned long)objp);
		BUG();
L
Linus Torvalds 已提交
2629 2630 2631
	}
}

2632 2633
static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
{
2634
	unsigned long long redzone1, redzone2;
2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649

	redzone1 = *dbg_redzone1(cache, obj);
	redzone2 = *dbg_redzone2(cache, obj);

	/*
	 * Redzone is ok.
	 */
	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
		return;

	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
		slab_error(cache, "double free detected");
	else
		slab_error(cache, "memory outside object was overwritten");

2650
	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
2651 2652 2653
			obj, redzone1, redzone2);
}

2654
static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
2655
				   unsigned long caller)
L
Linus Torvalds 已提交
2656 2657
{
	unsigned int objnr;
2658
	struct page *page;
L
Linus Torvalds 已提交
2659

2660 2661
	BUG_ON(virt_to_cache(objp) != cachep);

2662
	objp -= obj_offset(cachep);
L
Linus Torvalds 已提交
2663
	kfree_debugcheck(objp);
2664
	page = virt_to_head_page(objp);
L
Linus Torvalds 已提交
2665 2666

	if (cachep->flags & SLAB_RED_ZONE) {
2667
		verify_redzone_free(cachep, objp);
L
Linus Torvalds 已提交
2668 2669 2670
		*dbg_redzone1(cachep, objp) = RED_INACTIVE;
		*dbg_redzone2(cachep, objp) = RED_INACTIVE;
	}
2671 2672
	if (cachep->flags & SLAB_STORE_USER) {
		set_store_user_dirty(cachep);
2673
		*dbg_userword(cachep, objp) = (void *)caller;
2674
	}
L
Linus Torvalds 已提交
2675

2676
	objnr = obj_to_index(cachep, page, objp);
L
Linus Torvalds 已提交
2677 2678

	BUG_ON(objnr >= cachep->num);
2679
	BUG_ON(objp != index_to_obj(cachep, page, objnr));
L
Linus Torvalds 已提交
2680 2681 2682

	if (cachep->flags & SLAB_POISON) {
		poison_obj(cachep, objp, POISON_FREE);
2683
		slab_kernel_map(cachep, objp, 0, caller);
L
Linus Torvalds 已提交
2684 2685 2686 2687 2688 2689 2690 2691 2692
	}
	return objp;
}

#else
#define kfree_debugcheck(x) do { } while(0)
#define cache_free_debugcheck(x,objp,z) (objp)
#endif

2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
static struct page *get_first_slab(struct kmem_cache_node *n)
{
	struct page *page;

	page = list_first_entry_or_null(&n->slabs_partial,
			struct page, lru);
	if (!page) {
		n->free_touched = 1;
		page = list_first_entry_or_null(&n->slabs_free,
				struct page, lru);
	}

	return page;
}

2708 2709
static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
							bool force_refill)
L
Linus Torvalds 已提交
2710 2711
{
	int batchcount;
2712
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
2713
	struct array_cache *ac;
P
Pekka Enberg 已提交
2714 2715
	int node;

L
Linus Torvalds 已提交
2716
	check_irq_off();
2717
	node = numa_mem_id();
2718 2719 2720
	if (unlikely(force_refill))
		goto force_grow;
retry:
2721
	ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
2722 2723
	batchcount = ac->batchcount;
	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
A
Andrew Morton 已提交
2724 2725 2726 2727
		/*
		 * If there was little recent activity on this cache, then
		 * perform only a partial refill.  Otherwise we could generate
		 * refill bouncing.
L
Linus Torvalds 已提交
2728 2729 2730
		 */
		batchcount = BATCHREFILL_LIMIT;
	}
2731
	n = get_node(cachep, node);
2732

2733 2734
	BUG_ON(ac->avail > 0 || !n);
	spin_lock(&n->list_lock);
L
Linus Torvalds 已提交
2735

2736
	/* See if we can refill from the shared array */
2737 2738
	if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
		n->shared->touched = 1;
2739
		goto alloc_done;
2740
	}
2741

L
Linus Torvalds 已提交
2742
	while (batchcount > 0) {
2743
		struct page *page;
L
Linus Torvalds 已提交
2744
		/* Get slab alloc is to come from. */
2745 2746 2747
		page = get_first_slab(n);
		if (!page)
			goto must_grow;
L
Linus Torvalds 已提交
2748 2749

		check_spinlock_acquired(cachep);
2750 2751 2752 2753 2754 2755

		/*
		 * The slab was either on partial or free list so
		 * there must be at least one object available for
		 * allocation.
		 */
2756
		BUG_ON(page->active >= cachep->num);
2757

2758
		while (page->active < cachep->num && batchcount--) {
L
Linus Torvalds 已提交
2759 2760 2761 2762
			STATS_INC_ALLOCED(cachep);
			STATS_INC_ACTIVE(cachep);
			STATS_SET_HIGH(cachep);

2763
			ac_put_obj(cachep, ac, slab_get_obj(cachep, page));
L
Linus Torvalds 已提交
2764 2765 2766
		}

		/* move slabp to correct slabp list: */
2767 2768
		list_del(&page->lru);
		if (page->active == cachep->num)
2769
			list_add(&page->lru, &n->slabs_full);
L
Linus Torvalds 已提交
2770
		else
2771
			list_add(&page->lru, &n->slabs_partial);
L
Linus Torvalds 已提交
2772 2773
	}

A
Andrew Morton 已提交
2774
must_grow:
2775
	n->free_objects -= ac->avail;
A
Andrew Morton 已提交
2776
alloc_done:
2777
	spin_unlock(&n->list_lock);
L
Linus Torvalds 已提交
2778 2779 2780

	if (unlikely(!ac->avail)) {
		int x;
2781
force_grow:
D
David Rientjes 已提交
2782
		x = cache_grow(cachep, gfp_exact_node(flags), node, NULL);
2783

A
Andrew Morton 已提交
2784
		/* cache_grow can reenable interrupts, then ac could change. */
2785
		ac = cpu_cache_get(cachep);
2786
		node = numa_mem_id();
2787 2788 2789

		/* no objects in sight? abort */
		if (!x && (ac->avail == 0 || force_refill))
L
Linus Torvalds 已提交
2790 2791
			return NULL;

A
Andrew Morton 已提交
2792
		if (!ac->avail)		/* objects refilled by interrupt? */
L
Linus Torvalds 已提交
2793 2794 2795
			goto retry;
	}
	ac->touched = 1;
2796 2797

	return ac_get_obj(cachep, ac, flags, force_refill);
L
Linus Torvalds 已提交
2798 2799
}

A
Andrew Morton 已提交
2800 2801
static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
						gfp_t flags)
L
Linus Torvalds 已提交
2802
{
2803
	might_sleep_if(gfpflags_allow_blocking(flags));
L
Linus Torvalds 已提交
2804 2805 2806 2807 2808 2809
#if DEBUG
	kmem_flagcheck(cachep, flags);
#endif
}

#if DEBUG
A
Andrew Morton 已提交
2810
static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
2811
				gfp_t flags, void *objp, unsigned long caller)
L
Linus Torvalds 已提交
2812
{
P
Pekka Enberg 已提交
2813
	if (!objp)
L
Linus Torvalds 已提交
2814
		return objp;
P
Pekka Enberg 已提交
2815
	if (cachep->flags & SLAB_POISON) {
L
Linus Torvalds 已提交
2816
		check_poison_obj(cachep, objp);
2817
		slab_kernel_map(cachep, objp, 1, 0);
L
Linus Torvalds 已提交
2818 2819 2820
		poison_obj(cachep, objp, POISON_INUSE);
	}
	if (cachep->flags & SLAB_STORE_USER)
2821
		*dbg_userword(cachep, objp) = (void *)caller;
L
Linus Torvalds 已提交
2822 2823

	if (cachep->flags & SLAB_RED_ZONE) {
A
Andrew Morton 已提交
2824 2825 2826 2827
		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
			slab_error(cachep, "double free, or memory outside"
						" object was overwritten");
P
Pekka Enberg 已提交
2828
			printk(KERN_ERR
2829
				"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
A
Andrew Morton 已提交
2830 2831
				objp, *dbg_redzone1(cachep, objp),
				*dbg_redzone2(cachep, objp));
L
Linus Torvalds 已提交
2832 2833 2834 2835
		}
		*dbg_redzone1(cachep, objp) = RED_ACTIVE;
		*dbg_redzone2(cachep, objp) = RED_ACTIVE;
	}
2836

2837
	objp += obj_offset(cachep);
2838
	if (cachep->ctor && cachep->flags & SLAB_POISON)
2839
		cachep->ctor(objp);
T
Tetsuo Handa 已提交
2840 2841
	if (ARCH_SLAB_MINALIGN &&
	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
2842
		printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
H
Hugh Dickins 已提交
2843
		       objp, (int)ARCH_SLAB_MINALIGN);
2844
	}
L
Linus Torvalds 已提交
2845 2846 2847 2848 2849 2850
	return objp;
}
#else
#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
#endif

2851
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
2852
{
P
Pekka Enberg 已提交
2853
	void *objp;
L
Linus Torvalds 已提交
2854
	struct array_cache *ac;
2855
	bool force_refill = false;
L
Linus Torvalds 已提交
2856

2857
	check_irq_off();
2858

2859
	ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
2860 2861
	if (likely(ac->avail)) {
		ac->touched = 1;
2862 2863
		objp = ac_get_obj(cachep, ac, flags, false);

2864
		/*
2865 2866
		 * Allow for the possibility all avail objects are not allowed
		 * by the current flags
2867
		 */
2868 2869 2870 2871 2872
		if (objp) {
			STATS_INC_ALLOCHIT(cachep);
			goto out;
		}
		force_refill = true;
L
Linus Torvalds 已提交
2873
	}
2874 2875 2876 2877 2878 2879 2880 2881 2882 2883

	STATS_INC_ALLOCMISS(cachep);
	objp = cache_alloc_refill(cachep, flags, force_refill);
	/*
	 * the 'ac' may be updated by cache_alloc_refill(),
	 * and kmemleak_erase() requires its correct value.
	 */
	ac = cpu_cache_get(cachep);

out:
2884 2885 2886 2887 2888
	/*
	 * To avoid a false negative, if an object that is in one of the
	 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
	 * treat the array pointers as a reference to the object.
	 */
2889 2890
	if (objp)
		kmemleak_erase(&ac->entry[ac->avail]);
2891 2892 2893
	return objp;
}

2894
#ifdef CONFIG_NUMA
2895
/*
2896
 * Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
2897 2898 2899 2900 2901 2902 2903 2904
 *
 * If we are in_interrupt, then process context, including cpusets and
 * mempolicy, may not apply and should not be used for allocation policy.
 */
static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
{
	int nid_alloc, nid_here;

2905
	if (in_interrupt() || (flags & __GFP_THISNODE))
2906
		return NULL;
2907
	nid_alloc = nid_here = numa_mem_id();
2908
	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
2909
		nid_alloc = cpuset_slab_spread_node();
2910
	else if (current->mempolicy)
2911
		nid_alloc = mempolicy_slab_node();
2912
	if (nid_alloc != nid_here)
2913
		return ____cache_alloc_node(cachep, flags, nid_alloc);
2914 2915 2916
	return NULL;
}

2917 2918
/*
 * Fallback function if there was no memory available and no objects on a
2919
 * certain node and fall back is permitted. First we scan all the
2920
 * available node for available objects. If that fails then we
2921 2922 2923
 * perform an allocation without specifying a node. This allows the page
 * allocator to do its reclaim / fallback magic. We then insert the
 * slab into the proper nodelist and then allocate from it.
2924
 */
2925
static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
2926
{
2927 2928
	struct zonelist *zonelist;
	gfp_t local_flags;
2929
	struct zoneref *z;
2930 2931
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
2932
	void *obj = NULL;
2933
	int nid;
2934
	unsigned int cpuset_mems_cookie;
2935 2936 2937 2938

	if (flags & __GFP_THISNODE)
		return NULL;

C
Christoph Lameter 已提交
2939
	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
2940

2941
retry_cpuset:
2942
	cpuset_mems_cookie = read_mems_allowed_begin();
2943
	zonelist = node_zonelist(mempolicy_slab_node(), flags);
2944

2945 2946 2947 2948 2949
retry:
	/*
	 * Look through allowed nodes for objects available
	 * from existing per node queues.
	 */
2950 2951
	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
		nid = zone_to_nid(zone);
2952

2953
		if (cpuset_zone_allowed(zone, flags) &&
2954 2955
			get_node(cache, nid) &&
			get_node(cache, nid)->free_objects) {
2956
				obj = ____cache_alloc_node(cache,
D
David Rientjes 已提交
2957
					gfp_exact_node(flags), nid);
2958 2959 2960
				if (obj)
					break;
		}
2961 2962
	}

2963
	if (!obj) {
2964 2965 2966 2967 2968 2969
		/*
		 * This allocation will be performed within the constraints
		 * of the current cpuset / memory policy requirements.
		 * We may trigger various forms of reclaim on the allowed
		 * set and go into memory reserves if necessary.
		 */
2970 2971
		struct page *page;

2972
		if (gfpflags_allow_blocking(local_flags))
2973 2974
			local_irq_enable();
		kmem_flagcheck(cache, flags);
2975
		page = kmem_getpages(cache, local_flags, numa_mem_id());
2976
		if (gfpflags_allow_blocking(local_flags))
2977
			local_irq_disable();
2978
		if (page) {
2979 2980 2981
			/*
			 * Insert into the appropriate per node queues
			 */
2982 2983
			nid = page_to_nid(page);
			if (cache_grow(cache, flags, nid, page)) {
2984
				obj = ____cache_alloc_node(cache,
D
David Rientjes 已提交
2985
					gfp_exact_node(flags), nid);
2986 2987 2988 2989 2990 2991 2992 2993
				if (!obj)
					/*
					 * Another processor may allocate the
					 * objects in the slab since we are
					 * not holding any locks.
					 */
					goto retry;
			} else {
2994
				/* cache_grow already freed obj */
2995 2996 2997
				obj = NULL;
			}
		}
2998
	}
2999

3000
	if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
3001
		goto retry_cpuset;
3002 3003 3004
	return obj;
}

3005 3006
/*
 * A interface to enable slab creation on nodeid
L
Linus Torvalds 已提交
3007
 */
3008
static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
A
Andrew Morton 已提交
3009
				int nodeid)
3010
{
3011
	struct page *page;
3012
	struct kmem_cache_node *n;
P
Pekka Enberg 已提交
3013 3014 3015
	void *obj;
	int x;

3016
	VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
3017
	n = get_node(cachep, nodeid);
3018
	BUG_ON(!n);
P
Pekka Enberg 已提交
3019

A
Andrew Morton 已提交
3020
retry:
3021
	check_irq_off();
3022
	spin_lock(&n->list_lock);
3023 3024 3025
	page = get_first_slab(n);
	if (!page)
		goto must_grow;
P
Pekka Enberg 已提交
3026 3027 3028 3029 3030 3031 3032

	check_spinlock_acquired_node(cachep, nodeid);

	STATS_INC_NODEALLOCS(cachep);
	STATS_INC_ACTIVE(cachep);
	STATS_SET_HIGH(cachep);

3033
	BUG_ON(page->active == cachep->num);
P
Pekka Enberg 已提交
3034

3035
	obj = slab_get_obj(cachep, page);
3036
	n->free_objects--;
P
Pekka Enberg 已提交
3037
	/* move slabp to correct slabp list: */
3038
	list_del(&page->lru);
P
Pekka Enberg 已提交
3039

3040 3041
	if (page->active == cachep->num)
		list_add(&page->lru, &n->slabs_full);
A
Andrew Morton 已提交
3042
	else
3043
		list_add(&page->lru, &n->slabs_partial);
3044

3045
	spin_unlock(&n->list_lock);
P
Pekka Enberg 已提交
3046
	goto done;
3047

A
Andrew Morton 已提交
3048
must_grow:
3049
	spin_unlock(&n->list_lock);
D
David Rientjes 已提交
3050
	x = cache_grow(cachep, gfp_exact_node(flags), nodeid, NULL);
3051 3052
	if (x)
		goto retry;
L
Linus Torvalds 已提交
3053

3054
	return fallback_alloc(cachep, flags);
3055

A
Andrew Morton 已提交
3056
done:
P
Pekka Enberg 已提交
3057
	return obj;
3058
}
3059 3060

static __always_inline void *
3061
slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3062
		   unsigned long caller)
3063 3064 3065
{
	unsigned long save_flags;
	void *ptr;
3066
	int slab_node = numa_mem_id();
3067

3068
	flags &= gfp_allowed_mask;
3069 3070
	cachep = slab_pre_alloc_hook(cachep, flags);
	if (unlikely(!cachep))
3071 3072
		return NULL;

3073 3074 3075
	cache_alloc_debugcheck_before(cachep, flags);
	local_irq_save(save_flags);

A
Andrew Morton 已提交
3076
	if (nodeid == NUMA_NO_NODE)
3077
		nodeid = slab_node;
3078

3079
	if (unlikely(!get_node(cachep, nodeid))) {
3080 3081 3082 3083 3084
		/* Node not bootstrapped yet */
		ptr = fallback_alloc(cachep, flags);
		goto out;
	}

3085
	if (nodeid == slab_node) {
3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101
		/*
		 * Use the locally cached objects if possible.
		 * However ____cache_alloc does not allow fallback
		 * to other nodes. It may fail while we still have
		 * objects on other nodes available.
		 */
		ptr = ____cache_alloc(cachep, flags);
		if (ptr)
			goto out;
	}
	/* ___cache_alloc_node can fall back to other nodes */
	ptr = ____cache_alloc_node(cachep, flags, nodeid);
  out:
	local_irq_restore(save_flags);
	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);

3102 3103
	if (unlikely(flags & __GFP_ZERO) && ptr)
		memset(ptr, 0, cachep->object_size);
3104

3105
	slab_post_alloc_hook(cachep, flags, 1, &ptr);
3106 3107 3108 3109 3110 3111 3112 3113
	return ptr;
}

static __always_inline void *
__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
{
	void *objp;

3114
	if (current->mempolicy || cpuset_do_slab_mem_spread()) {
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124
		objp = alternate_node_alloc(cache, flags);
		if (objp)
			goto out;
	}
	objp = ____cache_alloc(cache, flags);

	/*
	 * We may just have run out of memory on the local node.
	 * ____cache_alloc_node() knows how to locate memory on other nodes
	 */
3125 3126
	if (!objp)
		objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141

  out:
	return objp;
}
#else

static __always_inline void *
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
{
	return ____cache_alloc(cachep, flags);
}

#endif /* CONFIG_NUMA */

static __always_inline void *
3142
slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3143 3144 3145 3146
{
	unsigned long save_flags;
	void *objp;

3147
	flags &= gfp_allowed_mask;
3148 3149
	cachep = slab_pre_alloc_hook(cachep, flags);
	if (unlikely(!cachep))
3150 3151
		return NULL;

3152 3153 3154 3155 3156 3157 3158
	cache_alloc_debugcheck_before(cachep, flags);
	local_irq_save(save_flags);
	objp = __do_cache_alloc(cachep, flags);
	local_irq_restore(save_flags);
	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
	prefetchw(objp);

3159 3160
	if (unlikely(flags & __GFP_ZERO) && objp)
		memset(objp, 0, cachep->object_size);
3161

3162
	slab_post_alloc_hook(cachep, flags, 1, &objp);
3163 3164
	return objp;
}
3165 3166

/*
3167
 * Caller needs to acquire correct kmem_cache_node's list_lock
3168
 * @list: List of detached free slabs should be freed by caller
3169
 */
3170 3171
static void free_block(struct kmem_cache *cachep, void **objpp,
			int nr_objects, int node, struct list_head *list)
L
Linus Torvalds 已提交
3172 3173
{
	int i;
3174
	struct kmem_cache_node *n = get_node(cachep, node);
L
Linus Torvalds 已提交
3175 3176

	for (i = 0; i < nr_objects; i++) {
3177
		void *objp;
3178
		struct page *page;
L
Linus Torvalds 已提交
3179

3180 3181 3182
		clear_obj_pfmemalloc(&objpp[i]);
		objp = objpp[i];

3183 3184
		page = virt_to_head_page(objp);
		list_del(&page->lru);
3185
		check_spinlock_acquired_node(cachep, node);
3186
		slab_put_obj(cachep, page, objp);
L
Linus Torvalds 已提交
3187
		STATS_DEC_ACTIVE(cachep);
3188
		n->free_objects++;
L
Linus Torvalds 已提交
3189 3190

		/* fixup slab chains */
3191
		if (page->active == 0) {
3192 3193
			if (n->free_objects > n->free_limit) {
				n->free_objects -= cachep->num;
3194
				list_add_tail(&page->lru, list);
L
Linus Torvalds 已提交
3195
			} else {
3196
				list_add(&page->lru, &n->slabs_free);
L
Linus Torvalds 已提交
3197 3198 3199 3200 3201 3202
			}
		} else {
			/* Unconditionally move a slab to the end of the
			 * partial list on free - maximum time for the
			 * other objects to be freed, too.
			 */
3203
			list_add_tail(&page->lru, &n->slabs_partial);
L
Linus Torvalds 已提交
3204 3205 3206 3207
		}
	}
}

3208
static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
L
Linus Torvalds 已提交
3209 3210
{
	int batchcount;
3211
	struct kmem_cache_node *n;
3212
	int node = numa_mem_id();
3213
	LIST_HEAD(list);
L
Linus Torvalds 已提交
3214 3215

	batchcount = ac->batchcount;
3216

L
Linus Torvalds 已提交
3217
	check_irq_off();
3218
	n = get_node(cachep, node);
3219 3220 3221
	spin_lock(&n->list_lock);
	if (n->shared) {
		struct array_cache *shared_array = n->shared;
P
Pekka Enberg 已提交
3222
		int max = shared_array->limit - shared_array->avail;
L
Linus Torvalds 已提交
3223 3224 3225
		if (max) {
			if (batchcount > max)
				batchcount = max;
3226
			memcpy(&(shared_array->entry[shared_array->avail]),
P
Pekka Enberg 已提交
3227
			       ac->entry, sizeof(void *) * batchcount);
L
Linus Torvalds 已提交
3228 3229 3230 3231 3232
			shared_array->avail += batchcount;
			goto free_done;
		}
	}

3233
	free_block(cachep, ac->entry, batchcount, node, &list);
A
Andrew Morton 已提交
3234
free_done:
L
Linus Torvalds 已提交
3235 3236 3237
#if STATS
	{
		int i = 0;
3238
		struct page *page;
L
Linus Torvalds 已提交
3239

3240
		list_for_each_entry(page, &n->slabs_free, lru) {
3241
			BUG_ON(page->active);
L
Linus Torvalds 已提交
3242 3243 3244 3245 3246 3247

			i++;
		}
		STATS_SET_FREEABLE(cachep, i);
	}
#endif
3248
	spin_unlock(&n->list_lock);
3249
	slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3250
	ac->avail -= batchcount;
A
Andrew Morton 已提交
3251
	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
L
Linus Torvalds 已提交
3252 3253 3254
}

/*
A
Andrew Morton 已提交
3255 3256
 * Release an obj back to its cache. If the obj has a constructed state, it must
 * be in this state _before_ it is released.  Called with disabled ints.
L
Linus Torvalds 已提交
3257
 */
3258
static inline void __cache_free(struct kmem_cache *cachep, void *objp,
3259
				unsigned long caller)
L
Linus Torvalds 已提交
3260
{
3261
	struct array_cache *ac = cpu_cache_get(cachep);
L
Linus Torvalds 已提交
3262 3263

	check_irq_off();
3264
	kmemleak_free_recursive(objp, cachep->flags);
3265
	objp = cache_free_debugcheck(cachep, objp, caller);
L
Linus Torvalds 已提交
3266

3267
	kmemcheck_slab_free(cachep, objp, cachep->object_size);
P
Pekka Enberg 已提交
3268

3269 3270 3271 3272 3273 3274 3275
	/*
	 * Skip calling cache_free_alien() when the platform is not numa.
	 * This will avoid cache misses that happen while accessing slabp (which
	 * is per page memory  reference) to get nodeid. Instead use a global
	 * variable to skip the call, which is mostly likely to be present in
	 * the cache.
	 */
3276
	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3277 3278
		return;

3279
	if (ac->avail < ac->limit) {
L
Linus Torvalds 已提交
3280 3281 3282 3283 3284
		STATS_INC_FREEHIT(cachep);
	} else {
		STATS_INC_FREEMISS(cachep);
		cache_flusharray(cachep, ac);
	}
Z
Zhao Jin 已提交
3285

3286
	ac_put_obj(cachep, ac, objp);
L
Linus Torvalds 已提交
3287 3288 3289 3290 3291 3292 3293 3294 3295 3296
}

/**
 * kmem_cache_alloc - Allocate an object
 * @cachep: The cache to allocate from.
 * @flags: See kmalloc().
 *
 * Allocate an object from this cache.  The flags are only relevant
 * if the cache has no available objects.
 */
3297
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
L
Linus Torvalds 已提交
3298
{
3299
	void *ret = slab_alloc(cachep, flags, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3300

3301
	trace_kmem_cache_alloc(_RET_IP_, ret,
3302
			       cachep->object_size, cachep->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3303 3304

	return ret;
L
Linus Torvalds 已提交
3305 3306 3307
}
EXPORT_SYMBOL(kmem_cache_alloc);

3308 3309 3310 3311 3312 3313 3314 3315 3316 3317
static __always_inline void
cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
				  size_t size, void **p, unsigned long caller)
{
	size_t i;

	for (i = 0; i < size; i++)
		p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
}

3318
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3319
			  void **p)
3320
{
3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338
	size_t i;

	s = slab_pre_alloc_hook(s, flags);
	if (!s)
		return 0;

	cache_alloc_debugcheck_before(s, flags);

	local_irq_disable();
	for (i = 0; i < size; i++) {
		void *objp = __do_cache_alloc(s, flags);

		if (unlikely(!objp))
			goto error;
		p[i] = objp;
	}
	local_irq_enable();

3339 3340
	cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);

3341 3342 3343 3344 3345 3346 3347 3348 3349 3350
	/* Clear memory outside IRQ disabled section */
	if (unlikely(flags & __GFP_ZERO))
		for (i = 0; i < size; i++)
			memset(p[i], 0, s->object_size);

	slab_post_alloc_hook(s, flags, size, p);
	/* FIXME: Trace call missing. Christoph would like a bulk variant */
	return size;
error:
	local_irq_enable();
3351
	cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
3352 3353 3354
	slab_post_alloc_hook(s, flags, i, p);
	__kmem_cache_free_bulk(s, i, p);
	return 0;
3355 3356 3357
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk);

3358
#ifdef CONFIG_TRACING
3359
void *
3360
kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
E
Eduard - Gabriel Munteanu 已提交
3361
{
3362 3363
	void *ret;

3364
	ret = slab_alloc(cachep, flags, _RET_IP_);
3365 3366

	trace_kmalloc(_RET_IP_, ret,
3367
		      size, cachep->size, flags);
3368
	return ret;
E
Eduard - Gabriel Munteanu 已提交
3369
}
3370
EXPORT_SYMBOL(kmem_cache_alloc_trace);
E
Eduard - Gabriel Munteanu 已提交
3371 3372
#endif

L
Linus Torvalds 已提交
3373
#ifdef CONFIG_NUMA
3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384
/**
 * kmem_cache_alloc_node - Allocate an object on the specified node
 * @cachep: The cache to allocate from.
 * @flags: See kmalloc().
 * @nodeid: node number of the target node.
 *
 * Identical to kmem_cache_alloc but it will allocate memory on the given
 * node, which can improve the performance for cpu bound structures.
 *
 * Fallback to other node is possible if __GFP_THISNODE is not set.
 */
3385 3386
void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
{
3387
	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3388

3389
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3390
				    cachep->object_size, cachep->size,
3391
				    flags, nodeid);
E
Eduard - Gabriel Munteanu 已提交
3392 3393

	return ret;
3394
}
L
Linus Torvalds 已提交
3395 3396
EXPORT_SYMBOL(kmem_cache_alloc_node);

3397
#ifdef CONFIG_TRACING
3398
void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
3399
				  gfp_t flags,
3400 3401
				  int nodeid,
				  size_t size)
E
Eduard - Gabriel Munteanu 已提交
3402
{
3403 3404
	void *ret;

3405
	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3406

3407
	trace_kmalloc_node(_RET_IP_, ret,
3408
			   size, cachep->size,
3409 3410
			   flags, nodeid);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
3411
}
3412
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
E
Eduard - Gabriel Munteanu 已提交
3413 3414
#endif

3415
static __always_inline void *
3416
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3417
{
3418
	struct kmem_cache *cachep;
3419

3420
	cachep = kmalloc_slab(size, flags);
3421 3422
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
		return cachep;
3423
	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
3424
}
3425 3426 3427

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
3428
	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3429
}
3430
EXPORT_SYMBOL(__kmalloc_node);
3431 3432

void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3433
		int node, unsigned long caller)
3434
{
3435
	return __do_kmalloc_node(size, flags, node, caller);
3436 3437 3438
}
EXPORT_SYMBOL(__kmalloc_node_track_caller);
#endif /* CONFIG_NUMA */
L
Linus Torvalds 已提交
3439 3440

/**
3441
 * __do_kmalloc - allocate memory
L
Linus Torvalds 已提交
3442
 * @size: how many bytes of memory are required.
3443
 * @flags: the type of memory to allocate (see kmalloc).
3444
 * @caller: function caller for debug tracking of the caller
L
Linus Torvalds 已提交
3445
 */
3446
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3447
					  unsigned long caller)
L
Linus Torvalds 已提交
3448
{
3449
	struct kmem_cache *cachep;
E
Eduard - Gabriel Munteanu 已提交
3450
	void *ret;
L
Linus Torvalds 已提交
3451

3452
	cachep = kmalloc_slab(size, flags);
3453 3454
	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
		return cachep;
3455
	ret = slab_alloc(cachep, flags, caller);
E
Eduard - Gabriel Munteanu 已提交
3456

3457
	trace_kmalloc(caller, ret,
3458
		      size, cachep->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3459 3460

	return ret;
3461 3462 3463 3464
}

void *__kmalloc(size_t size, gfp_t flags)
{
3465
	return __do_kmalloc(size, flags, _RET_IP_);
L
Linus Torvalds 已提交
3466 3467 3468
}
EXPORT_SYMBOL(__kmalloc);

3469
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3470
{
3471
	return __do_kmalloc(size, flags, caller);
3472 3473
}
EXPORT_SYMBOL(__kmalloc_track_caller);
3474

L
Linus Torvalds 已提交
3475 3476 3477 3478 3479 3480 3481 3482
/**
 * kmem_cache_free - Deallocate an object
 * @cachep: The cache the allocation was from.
 * @objp: The previously allocated object.
 *
 * Free an object which was previously allocated from this
 * cache.
 */
3483
void kmem_cache_free(struct kmem_cache *cachep, void *objp)
L
Linus Torvalds 已提交
3484 3485
{
	unsigned long flags;
3486 3487 3488
	cachep = cache_from_obj(cachep, objp);
	if (!cachep)
		return;
L
Linus Torvalds 已提交
3489 3490

	local_irq_save(flags);
3491
	debug_check_no_locks_freed(objp, cachep->object_size);
3492
	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3493
		debug_check_no_obj_freed(objp, cachep->object_size);
3494
	__cache_free(cachep, objp, _RET_IP_);
L
Linus Torvalds 已提交
3495
	local_irq_restore(flags);
E
Eduard - Gabriel Munteanu 已提交
3496

3497
	trace_kmem_cache_free(_RET_IP_, objp);
L
Linus Torvalds 已提交
3498 3499 3500
}
EXPORT_SYMBOL(kmem_cache_free);

3501 3502 3503 3504 3505 3506 3507 3508 3509
void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
{
	struct kmem_cache *s;
	size_t i;

	local_irq_disable();
	for (i = 0; i < size; i++) {
		void *objp = p[i];

3510 3511 3512 3513
		if (!orig_s) /* called via kfree_bulk */
			s = virt_to_cache(objp);
		else
			s = cache_from_obj(orig_s, objp);
3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526

		debug_check_no_locks_freed(objp, s->object_size);
		if (!(s->flags & SLAB_DEBUG_OBJECTS))
			debug_check_no_obj_freed(objp, s->object_size);

		__cache_free(s, objp, _RET_IP_);
	}
	local_irq_enable();

	/* FIXME: add tracing */
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

L
Linus Torvalds 已提交
3527 3528 3529 3530
/**
 * kfree - free previously allocated memory
 * @objp: pointer returned by kmalloc.
 *
3531 3532
 * If @objp is NULL, no operation is performed.
 *
L
Linus Torvalds 已提交
3533 3534 3535 3536 3537
 * Don't free memory not originally allocated by kmalloc()
 * or you will run into trouble.
 */
void kfree(const void *objp)
{
3538
	struct kmem_cache *c;
L
Linus Torvalds 已提交
3539 3540
	unsigned long flags;

3541 3542
	trace_kfree(_RET_IP_, objp);

3543
	if (unlikely(ZERO_OR_NULL_PTR(objp)))
L
Linus Torvalds 已提交
3544 3545 3546
		return;
	local_irq_save(flags);
	kfree_debugcheck(objp);
3547
	c = virt_to_cache(objp);
3548 3549 3550
	debug_check_no_locks_freed(objp, c->object_size);

	debug_check_no_obj_freed(objp, c->object_size);
3551
	__cache_free(c, (void *)objp, _RET_IP_);
L
Linus Torvalds 已提交
3552 3553 3554 3555
	local_irq_restore(flags);
}
EXPORT_SYMBOL(kfree);

3556
/*
3557
 * This initializes kmem_cache_node or resizes various caches for all nodes.
3558
 */
3559
static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp)
3560 3561
{
	int node;
3562
	struct kmem_cache_node *n;
3563
	struct array_cache *new_shared;
J
Joonsoo Kim 已提交
3564
	struct alien_cache **new_alien = NULL;
3565

3566
	for_each_online_node(node) {
3567

3568 3569 3570 3571 3572
		if (use_alien_caches) {
			new_alien = alloc_alien_cache(node, cachep->limit, gfp);
			if (!new_alien)
				goto fail;
		}
3573

3574 3575 3576
		new_shared = NULL;
		if (cachep->shared) {
			new_shared = alloc_arraycache(node,
3577
				cachep->shared*cachep->batchcount,
3578
					0xbaadf00d, gfp);
3579 3580 3581 3582
			if (!new_shared) {
				free_alien_cache(new_alien);
				goto fail;
			}
3583
		}
3584

3585
		n = get_node(cachep, node);
3586 3587
		if (n) {
			struct array_cache *shared = n->shared;
3588
			LIST_HEAD(list);
3589

3590
			spin_lock_irq(&n->list_lock);
3591

3592
			if (shared)
3593
				free_block(cachep, shared->entry,
3594
						shared->avail, node, &list);
3595

3596 3597 3598
			n->shared = new_shared;
			if (!n->alien) {
				n->alien = new_alien;
3599 3600
				new_alien = NULL;
			}
3601
			n->free_limit = (1 + nr_cpus_node(node)) *
A
Andrew Morton 已提交
3602
					cachep->batchcount + cachep->num;
3603
			spin_unlock_irq(&n->list_lock);
3604
			slabs_destroy(cachep, &list);
3605
			kfree(shared);
3606 3607 3608
			free_alien_cache(new_alien);
			continue;
		}
3609 3610
		n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
		if (!n) {
3611 3612
			free_alien_cache(new_alien);
			kfree(new_shared);
3613
			goto fail;
3614
		}
3615

3616
		kmem_cache_node_init(n);
3617 3618
		n->next_reap = jiffies + REAPTIMEOUT_NODE +
				((unsigned long)cachep) % REAPTIMEOUT_NODE;
3619 3620 3621
		n->shared = new_shared;
		n->alien = new_alien;
		n->free_limit = (1 + nr_cpus_node(node)) *
A
Andrew Morton 已提交
3622
					cachep->batchcount + cachep->num;
3623
		cachep->node[node] = n;
3624
	}
3625
	return 0;
3626

A
Andrew Morton 已提交
3627
fail:
3628
	if (!cachep->list.next) {
3629 3630 3631
		/* Cache is not active yet. Roll back what we did */
		node--;
		while (node >= 0) {
3632 3633
			n = get_node(cachep, node);
			if (n) {
3634 3635 3636
				kfree(n->shared);
				free_alien_cache(n->alien);
				kfree(n);
3637
				cachep->node[node] = NULL;
3638 3639 3640 3641
			}
			node--;
		}
	}
3642
	return -ENOMEM;
3643 3644
}

3645
/* Always called with the slab_mutex held */
G
Glauber Costa 已提交
3646
static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3647
				int batchcount, int shared, gfp_t gfp)
L
Linus Torvalds 已提交
3648
{
3649 3650
	struct array_cache __percpu *cpu_cache, *prev;
	int cpu;
L
Linus Torvalds 已提交
3651

3652 3653
	cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
	if (!cpu_cache)
3654 3655
		return -ENOMEM;

3656 3657 3658
	prev = cachep->cpu_cache;
	cachep->cpu_cache = cpu_cache;
	kick_all_cpus_sync();
3659

L
Linus Torvalds 已提交
3660 3661 3662
	check_irq_on();
	cachep->batchcount = batchcount;
	cachep->limit = limit;
3663
	cachep->shared = shared;
L
Linus Torvalds 已提交
3664

3665 3666 3667 3668
	if (!prev)
		goto alloc_node;

	for_each_online_cpu(cpu) {
3669
		LIST_HEAD(list);
3670 3671
		int node;
		struct kmem_cache_node *n;
3672
		struct array_cache *ac = per_cpu_ptr(prev, cpu);
3673

3674
		node = cpu_to_mem(cpu);
3675 3676
		n = get_node(cachep, node);
		spin_lock_irq(&n->list_lock);
3677
		free_block(cachep, ac->entry, ac->avail, node, &list);
3678
		spin_unlock_irq(&n->list_lock);
3679
		slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3680
	}
3681 3682 3683
	free_percpu(prev);

alloc_node:
3684
	return alloc_kmem_cache_node(cachep, gfp);
L
Linus Torvalds 已提交
3685 3686
}

G
Glauber Costa 已提交
3687 3688 3689 3690
static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
				int batchcount, int shared, gfp_t gfp)
{
	int ret;
3691
	struct kmem_cache *c;
G
Glauber Costa 已提交
3692 3693 3694 3695 3696 3697 3698 3699 3700

	ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);

	if (slab_state < FULL)
		return ret;

	if ((ret < 0) || !is_root_cache(cachep))
		return ret;

3701 3702 3703 3704
	lockdep_assert_held(&slab_mutex);
	for_each_memcg_cache(c, cachep) {
		/* return value determined by the root cache only */
		__do_tune_cpucache(c, limit, batchcount, shared, gfp);
G
Glauber Costa 已提交
3705 3706 3707 3708 3709
	}

	return ret;
}

3710
/* Called with slab_mutex held always */
3711
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
L
Linus Torvalds 已提交
3712 3713
{
	int err;
G
Glauber Costa 已提交
3714 3715 3716 3717 3718 3719 3720 3721 3722 3723
	int limit = 0;
	int shared = 0;
	int batchcount = 0;

	if (!is_root_cache(cachep)) {
		struct kmem_cache *root = memcg_root_cache(cachep);
		limit = root->limit;
		shared = root->shared;
		batchcount = root->batchcount;
	}
L
Linus Torvalds 已提交
3724

G
Glauber Costa 已提交
3725 3726
	if (limit && shared && batchcount)
		goto skip_setup;
A
Andrew Morton 已提交
3727 3728
	/*
	 * The head array serves three purposes:
L
Linus Torvalds 已提交
3729 3730
	 * - create a LIFO ordering, i.e. return objects that are cache-warm
	 * - reduce the number of spinlock operations.
A
Andrew Morton 已提交
3731
	 * - reduce the number of linked list operations on the slab and
L
Linus Torvalds 已提交
3732 3733 3734 3735
	 *   bufctl chains: array operations are cheaper.
	 * The numbers are guessed, we should auto-tune as described by
	 * Bonwick.
	 */
3736
	if (cachep->size > 131072)
L
Linus Torvalds 已提交
3737
		limit = 1;
3738
	else if (cachep->size > PAGE_SIZE)
L
Linus Torvalds 已提交
3739
		limit = 8;
3740
	else if (cachep->size > 1024)
L
Linus Torvalds 已提交
3741
		limit = 24;
3742
	else if (cachep->size > 256)
L
Linus Torvalds 已提交
3743 3744 3745 3746
		limit = 54;
	else
		limit = 120;

A
Andrew Morton 已提交
3747 3748
	/*
	 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
L
Linus Torvalds 已提交
3749 3750 3751 3752 3753 3754 3755 3756
	 * allocation behaviour: Most allocs on one cpu, most free operations
	 * on another cpu. For these cases, an efficient object passing between
	 * cpus is necessary. This is provided by a shared array. The array
	 * replaces Bonwick's magazine layer.
	 * On uniprocessor, it's functionally equivalent (but less efficient)
	 * to a larger limit. Thus disabled by default.
	 */
	shared = 0;
3757
	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
L
Linus Torvalds 已提交
3758 3759 3760
		shared = 8;

#if DEBUG
A
Andrew Morton 已提交
3761 3762 3763
	/*
	 * With debugging enabled, large batchcount lead to excessively long
	 * periods with disabled local interrupts. Limit the batchcount
L
Linus Torvalds 已提交
3764 3765 3766 3767
	 */
	if (limit > 32)
		limit = 32;
#endif
G
Glauber Costa 已提交
3768 3769 3770
	batchcount = (limit + 1) / 2;
skip_setup:
	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
L
Linus Torvalds 已提交
3771 3772
	if (err)
		printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
P
Pekka Enberg 已提交
3773
		       cachep->name, -err);
3774
	return err;
L
Linus Torvalds 已提交
3775 3776
}

3777
/*
3778 3779
 * Drain an array if it contains any elements taking the node lock only if
 * necessary. Note that the node listlock also protects the array_cache
3780
 * if drain_array() is used on the shared array.
3781
 */
3782
static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
3783
			 struct array_cache *ac, int force, int node)
L
Linus Torvalds 已提交
3784
{
3785
	LIST_HEAD(list);
L
Linus Torvalds 已提交
3786 3787
	int tofree;

3788 3789
	if (!ac || !ac->avail)
		return;
L
Linus Torvalds 已提交
3790 3791
	if (ac->touched && !force) {
		ac->touched = 0;
3792
	} else {
3793
		spin_lock_irq(&n->list_lock);
3794 3795 3796 3797
		if (ac->avail) {
			tofree = force ? ac->avail : (ac->limit + 4) / 5;
			if (tofree > ac->avail)
				tofree = (ac->avail + 1) / 2;
3798
			free_block(cachep, ac->entry, tofree, node, &list);
3799 3800 3801 3802
			ac->avail -= tofree;
			memmove(ac->entry, &(ac->entry[tofree]),
				sizeof(void *) * ac->avail);
		}
3803
		spin_unlock_irq(&n->list_lock);
3804
		slabs_destroy(cachep, &list);
L
Linus Torvalds 已提交
3805 3806 3807 3808 3809
	}
}

/**
 * cache_reap - Reclaim memory from caches.
3810
 * @w: work descriptor
L
Linus Torvalds 已提交
3811 3812 3813 3814 3815 3816
 *
 * Called from workqueue/eventd every few seconds.
 * Purpose:
 * - clear the per-cpu caches for this CPU.
 * - return freeable pages to the main free memory pool.
 *
A
Andrew Morton 已提交
3817 3818
 * If we cannot acquire the cache chain mutex then just give up - we'll try
 * again on the next iteration.
L
Linus Torvalds 已提交
3819
 */
3820
static void cache_reap(struct work_struct *w)
L
Linus Torvalds 已提交
3821
{
3822
	struct kmem_cache *searchp;
3823
	struct kmem_cache_node *n;
3824
	int node = numa_mem_id();
3825
	struct delayed_work *work = to_delayed_work(w);
L
Linus Torvalds 已提交
3826

3827
	if (!mutex_trylock(&slab_mutex))
L
Linus Torvalds 已提交
3828
		/* Give up. Setup the next iteration. */
3829
		goto out;
L
Linus Torvalds 已提交
3830

3831
	list_for_each_entry(searchp, &slab_caches, list) {
L
Linus Torvalds 已提交
3832 3833
		check_irq_on();

3834
		/*
3835
		 * We only take the node lock if absolutely necessary and we
3836 3837 3838
		 * have established with reasonable certainty that
		 * we can do some work if the lock was obtained.
		 */
3839
		n = get_node(searchp, node);
3840

3841
		reap_alien(searchp, n);
L
Linus Torvalds 已提交
3842

3843
		drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
L
Linus Torvalds 已提交
3844

3845 3846 3847 3848
		/*
		 * These are racy checks but it does not matter
		 * if we skip one check or scan twice.
		 */
3849
		if (time_after(n->next_reap, jiffies))
3850
			goto next;
L
Linus Torvalds 已提交
3851

3852
		n->next_reap = jiffies + REAPTIMEOUT_NODE;
L
Linus Torvalds 已提交
3853

3854
		drain_array(searchp, n, n->shared, 0, node);
L
Linus Torvalds 已提交
3855

3856 3857
		if (n->free_touched)
			n->free_touched = 0;
3858 3859
		else {
			int freed;
L
Linus Torvalds 已提交
3860

3861
			freed = drain_freelist(searchp, n, (n->free_limit +
3862 3863 3864
				5 * searchp->num - 1) / (5 * searchp->num));
			STATS_ADD_REAPED(searchp, freed);
		}
3865
next:
L
Linus Torvalds 已提交
3866 3867 3868
		cond_resched();
	}
	check_irq_on();
3869
	mutex_unlock(&slab_mutex);
3870
	next_reap_node();
3871
out:
A
Andrew Morton 已提交
3872
	/* Set up the next iteration */
3873
	schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
L
Linus Torvalds 已提交
3874 3875
}

3876
#ifdef CONFIG_SLABINFO
3877
void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
L
Linus Torvalds 已提交
3878
{
3879
	struct page *page;
P
Pekka Enberg 已提交
3880 3881 3882 3883
	unsigned long active_objs;
	unsigned long num_objs;
	unsigned long active_slabs = 0;
	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
3884
	const char *name;
L
Linus Torvalds 已提交
3885
	char *error = NULL;
3886
	int node;
3887
	struct kmem_cache_node *n;
L
Linus Torvalds 已提交
3888 3889 3890

	active_objs = 0;
	num_slabs = 0;
3891
	for_each_kmem_cache_node(cachep, node, n) {
3892

3893
		check_irq_on();
3894
		spin_lock_irq(&n->list_lock);
3895

3896 3897
		list_for_each_entry(page, &n->slabs_full, lru) {
			if (page->active != cachep->num && !error)
3898 3899 3900 3901
				error = "slabs_full accounting error";
			active_objs += cachep->num;
			active_slabs++;
		}
3902 3903
		list_for_each_entry(page, &n->slabs_partial, lru) {
			if (page->active == cachep->num && !error)
3904
				error = "slabs_partial accounting error";
3905
			if (!page->active && !error)
3906
				error = "slabs_partial accounting error";
3907
			active_objs += page->active;
3908 3909
			active_slabs++;
		}
3910 3911
		list_for_each_entry(page, &n->slabs_free, lru) {
			if (page->active && !error)
3912
				error = "slabs_free accounting error";
3913 3914
			num_slabs++;
		}
3915 3916 3917
		free_objects += n->free_objects;
		if (n->shared)
			shared_avail += n->shared->avail;
3918

3919
		spin_unlock_irq(&n->list_lock);
L
Linus Torvalds 已提交
3920
	}
P
Pekka Enberg 已提交
3921 3922
	num_slabs += active_slabs;
	num_objs = num_slabs * cachep->num;
3923
	if (num_objs - active_objs != free_objects && !error)
L
Linus Torvalds 已提交
3924 3925
		error = "free_objects accounting error";

P
Pekka Enberg 已提交
3926
	name = cachep->name;
L
Linus Torvalds 已提交
3927 3928 3929
	if (error)
		printk(KERN_ERR "slab: cache %s error: %s\n", name, error);

3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943
	sinfo->active_objs = active_objs;
	sinfo->num_objs = num_objs;
	sinfo->active_slabs = active_slabs;
	sinfo->num_slabs = num_slabs;
	sinfo->shared_avail = shared_avail;
	sinfo->limit = cachep->limit;
	sinfo->batchcount = cachep->batchcount;
	sinfo->shared = cachep->shared;
	sinfo->objects_per_slab = cachep->num;
	sinfo->cache_order = cachep->gfporder;
}

void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
{
L
Linus Torvalds 已提交
3944
#if STATS
3945
	{			/* node stats */
L
Linus Torvalds 已提交
3946 3947 3948 3949 3950 3951 3952
		unsigned long high = cachep->high_mark;
		unsigned long allocs = cachep->num_allocations;
		unsigned long grown = cachep->grown;
		unsigned long reaped = cachep->reaped;
		unsigned long errors = cachep->errors;
		unsigned long max_freeable = cachep->max_freeable;
		unsigned long node_allocs = cachep->node_allocs;
3953
		unsigned long node_frees = cachep->node_frees;
3954
		unsigned long overflows = cachep->node_overflow;
L
Linus Torvalds 已提交
3955

J
Joe Perches 已提交
3956 3957 3958 3959 3960
		seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
			   "%4lu %4lu %4lu %4lu %4lu",
			   allocs, high, grown,
			   reaped, errors, max_freeable, node_allocs,
			   node_frees, overflows);
L
Linus Torvalds 已提交
3961 3962 3963 3964 3965 3966 3967 3968 3969
	}
	/* cpu stats */
	{
		unsigned long allochit = atomic_read(&cachep->allochit);
		unsigned long allocmiss = atomic_read(&cachep->allocmiss);
		unsigned long freehit = atomic_read(&cachep->freehit);
		unsigned long freemiss = atomic_read(&cachep->freemiss);

		seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
P
Pekka Enberg 已提交
3970
			   allochit, allocmiss, freehit, freemiss);
L
Linus Torvalds 已提交
3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982
	}
#endif
}

#define MAX_SLABINFO_WRITE 128
/**
 * slabinfo_write - Tuning for the slab allocator
 * @file: unused
 * @buffer: user buffer
 * @count: data length
 * @ppos: unused
 */
3983
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
P
Pekka Enberg 已提交
3984
		       size_t count, loff_t *ppos)
L
Linus Torvalds 已提交
3985
{
P
Pekka Enberg 已提交
3986
	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
L
Linus Torvalds 已提交
3987
	int limit, batchcount, shared, res;
3988
	struct kmem_cache *cachep;
P
Pekka Enberg 已提交
3989

L
Linus Torvalds 已提交
3990 3991 3992 3993
	if (count > MAX_SLABINFO_WRITE)
		return -EINVAL;
	if (copy_from_user(&kbuf, buffer, count))
		return -EFAULT;
P
Pekka Enberg 已提交
3994
	kbuf[MAX_SLABINFO_WRITE] = '\0';
L
Linus Torvalds 已提交
3995 3996 3997 3998 3999 4000 4001 4002 4003 4004

	tmp = strchr(kbuf, ' ');
	if (!tmp)
		return -EINVAL;
	*tmp = '\0';
	tmp++;
	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
		return -EINVAL;

	/* Find the cache in the chain of caches. */
4005
	mutex_lock(&slab_mutex);
L
Linus Torvalds 已提交
4006
	res = -EINVAL;
4007
	list_for_each_entry(cachep, &slab_caches, list) {
L
Linus Torvalds 已提交
4008
		if (!strcmp(cachep->name, kbuf)) {
A
Andrew Morton 已提交
4009 4010
			if (limit < 1 || batchcount < 1 ||
					batchcount > limit || shared < 0) {
4011
				res = 0;
L
Linus Torvalds 已提交
4012
			} else {
4013
				res = do_tune_cpucache(cachep, limit,
4014 4015
						       batchcount, shared,
						       GFP_KERNEL);
L
Linus Torvalds 已提交
4016 4017 4018 4019
			}
			break;
		}
	}
4020
	mutex_unlock(&slab_mutex);
L
Linus Torvalds 已提交
4021 4022 4023 4024
	if (res >= 0)
		res = count;
	return res;
}
4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057

#ifdef CONFIG_DEBUG_SLAB_LEAK

static inline int add_caller(unsigned long *n, unsigned long v)
{
	unsigned long *p;
	int l;
	if (!v)
		return 1;
	l = n[1];
	p = n + 2;
	while (l) {
		int i = l/2;
		unsigned long *q = p + 2 * i;
		if (*q == v) {
			q[1]++;
			return 1;
		}
		if (*q > v) {
			l = i;
		} else {
			p = q + 2;
			l -= i + 1;
		}
	}
	if (++n[1] == n[0])
		return 0;
	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
	p[0] = v;
	p[1] = 1;
	return 1;
}

4058 4059
static void handle_slab(unsigned long *n, struct kmem_cache *c,
						struct page *page)
4060 4061
{
	void *p;
4062 4063
	int i, j;
	unsigned long v;
4064

4065 4066
	if (n[0] == n[1])
		return;
4067
	for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
		bool active = true;

		for (j = page->active; j < c->num; j++) {
			if (get_free_obj(page, j) == i) {
				active = false;
				break;
			}
		}

		if (!active)
4078
			continue;
4079

4080 4081 4082 4083 4084 4085 4086 4087 4088 4089
		/*
		 * probe_kernel_read() is used for DEBUG_PAGEALLOC. page table
		 * mapping is established when actual object allocation and
		 * we could mistakenly access the unmapped object in the cpu
		 * cache.
		 */
		if (probe_kernel_read(&v, dbg_userword(c, p), sizeof(v)))
			continue;

		if (!add_caller(n, v))
4090 4091 4092 4093 4094 4095 4096 4097
			return;
	}
}

static void show_symbol(struct seq_file *m, unsigned long address)
{
#ifdef CONFIG_KALLSYMS
	unsigned long offset, size;
4098
	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4099

4100
	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4101
		seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4102
		if (modname[0])
4103 4104 4105 4106 4107 4108 4109 4110 4111
			seq_printf(m, " [%s]", modname);
		return;
	}
#endif
	seq_printf(m, "%p", (void *)address);
}

static int leaks_show(struct seq_file *m, void *p)
{
4112
	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4113
	struct page *page;
4114
	struct kmem_cache_node *n;
4115
	const char *name;
4116
	unsigned long *x = m->private;
4117 4118 4119 4120 4121 4122 4123 4124
	int node;
	int i;

	if (!(cachep->flags & SLAB_STORE_USER))
		return 0;
	if (!(cachep->flags & SLAB_RED_ZONE))
		return 0;

4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135
	/*
	 * Set store_user_clean and start to grab stored user information
	 * for all objects on this cache. If some alloc/free requests comes
	 * during the processing, information would be wrong so restart
	 * whole processing.
	 */
	do {
		set_store_user_clean(cachep);
		drain_cpu_caches(cachep);

		x[1] = 0;
4136

4137
		for_each_kmem_cache_node(cachep, node, n) {
4138

4139 4140
			check_irq_on();
			spin_lock_irq(&n->list_lock);
4141

4142 4143 4144 4145 4146 4147 4148
			list_for_each_entry(page, &n->slabs_full, lru)
				handle_slab(x, cachep, page);
			list_for_each_entry(page, &n->slabs_partial, lru)
				handle_slab(x, cachep, page);
			spin_unlock_irq(&n->list_lock);
		}
	} while (!is_store_user_clean(cachep));
4149 4150

	name = cachep->name;
4151
	if (x[0] == x[1]) {
4152
		/* Increase the buffer size */
4153
		mutex_unlock(&slab_mutex);
4154
		m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4155 4156
		if (!m->private) {
			/* Too bad, we are really out */
4157
			m->private = x;
4158
			mutex_lock(&slab_mutex);
4159 4160
			return -ENOMEM;
		}
4161 4162
		*(unsigned long *)m->private = x[0] * 2;
		kfree(x);
4163
		mutex_lock(&slab_mutex);
4164 4165 4166 4167
		/* Now make sure this entry will be retried */
		m->count = m->size;
		return 0;
	}
4168 4169 4170
	for (i = 0; i < x[1]; i++) {
		seq_printf(m, "%s: %lu ", name, x[2*i+3]);
		show_symbol(m, x[2*i+2]);
4171 4172
		seq_putc(m, '\n');
	}
4173

4174 4175 4176
	return 0;
}

4177
static const struct seq_operations slabstats_op = {
4178
	.start = slab_start,
4179 4180
	.next = slab_next,
	.stop = slab_stop,
4181 4182
	.show = leaks_show,
};
4183 4184 4185

static int slabstats_open(struct inode *inode, struct file *file)
{
4186 4187 4188 4189 4190 4191 4192 4193 4194
	unsigned long *n;

	n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
	if (!n)
		return -ENOMEM;

	*n = PAGE_SIZE / (2 * sizeof(unsigned long));

	return 0;
4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208
}

static const struct file_operations proc_slabstats_operations = {
	.open		= slabstats_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release_private,
};
#endif

static int __init slab_proc_init(void)
{
#ifdef CONFIG_DEBUG_SLAB_LEAK
	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4209
#endif
4210 4211 4212
	return 0;
}
module_init(slab_proc_init);
L
Linus Torvalds 已提交
4213 4214
#endif

4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226
/**
 * ksize - get the actual amount of memory allocated for a given object
 * @objp: Pointer to the object
 *
 * kmalloc may internally round up allocations and return more memory
 * than requested. ksize() can be used to determine the actual amount of
 * memory allocated. The caller may use this additional memory, even though
 * a smaller amount of memory was initially specified with the kmalloc call.
 * The caller must guarantee that objp points to a valid object previously
 * allocated with either kmalloc() or kmem_cache_alloc(). The object
 * must not be freed during the duration of the call.
 */
P
Pekka Enberg 已提交
4227
size_t ksize(const void *objp)
L
Linus Torvalds 已提交
4228
{
4229 4230
	BUG_ON(!objp);
	if (unlikely(objp == ZERO_SIZE_PTR))
4231
		return 0;
L
Linus Torvalds 已提交
4232

4233
	return virt_to_cache(objp)->object_size;
L
Linus Torvalds 已提交
4234
}
K
Kirill A. Shutemov 已提交
4235
EXPORT_SYMBOL(ksize);