slub.c 129.3 KB
Newer Older
C
Christoph Lameter 已提交
1 2 3 4
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
5 6
 * The allocator synchronizes using per slab locks or atomic operatios
 * and only uses a centralized lock to manage a pool of partial slabs.
C
Christoph Lameter 已提交
7
 *
C
Christoph Lameter 已提交
8
 * (C) 2007 SGI, Christoph Lameter
9
 * (C) 2011 Linux Foundation, Christoph Lameter
C
Christoph Lameter 已提交
10 11 12
 */

#include <linux/mm.h>
N
Nick Piggin 已提交
13
#include <linux/swap.h> /* struct reclaim_state */
C
Christoph Lameter 已提交
14 15 16 17 18
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/slab.h>
19
#include <linux/proc_fs.h>
C
Christoph Lameter 已提交
20
#include <linux/seq_file.h>
V
Vegard Nossum 已提交
21
#include <linux/kmemcheck.h>
C
Christoph Lameter 已提交
22 23 24 25
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
26
#include <linux/debugobjects.h>
C
Christoph Lameter 已提交
27
#include <linux/kallsyms.h>
28
#include <linux/memory.h>
R
Roman Zippel 已提交
29
#include <linux/math64.h>
A
Akinobu Mita 已提交
30
#include <linux/fault-inject.h>
31
#include <linux/stacktrace.h>
32
#include <linux/prefetch.h>
C
Christoph Lameter 已提交
33

34 35
#include <trace/events/kmem.h>

C
Christoph Lameter 已提交
36 37
/*
 * Lock order:
38 39 40
 *   1. slub_lock (Global Semaphore)
 *   2. node->list_lock
 *   3. slab_lock(page) (Only on some arches and for debugging)
C
Christoph Lameter 已提交
41
 *
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
 *   slub_lock
 *
 *   The role of the slub_lock is to protect the list of all the slabs
 *   and to synchronize major metadata changes to slab cache structures.
 *
 *   The slab_lock is only used for debugging and on arches that do not
 *   have the ability to do a cmpxchg_double. It only protects the second
 *   double word in the page struct. Meaning
 *	A. page->freelist	-> List of object free in a page
 *	B. page->counters	-> Counters of objects
 *	C. page->frozen		-> frozen state
 *
 *   If a slab is frozen then it is exempt from list management. It is not
 *   on any list. The processor that froze the slab is the one who can
 *   perform list operations on the page. Other processors may put objects
 *   onto the freelist but the processor that froze the slab is the only
 *   one that can retrieve the objects from the page's freelist.
C
Christoph Lameter 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
 *
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
 *   Interrupts are disabled during allocation and deallocation in order to
 *   make the slab allocator safe to use in the context of an irq. In addition
 *   interrupts are disabled to ensure that the processor does not change
 *   while handling per_cpu slabs, due to kernel preemption.
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
C
Christoph Lameter 已提交
79 80
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
C
Christoph Lameter 已提交
81
 * freed then the slab will show up again on the partial lists.
C
Christoph Lameter 已提交
82 83
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
C
Christoph Lameter 已提交
84 85 86 87 88 89 90
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
 * Overloading of page flags that are otherwise used for LRU management.
 *
91 92 93 94 95 96 97 98 99 100 101 102
 * PageActive 		The slab is frozen and exempt from list processing.
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
103
 * 			freelist that allows lockless access to
104 105
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
C
Christoph Lameter 已提交
106 107 108
 *
 * PageError		Slab requires special handling due to debug
 * 			options set. This moves	slab handling out of
109
 * 			the fast path and disables lockless freelists.
C
Christoph Lameter 已提交
110 111
 */

112 113 114 115 116
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
		SLAB_TRACE | SLAB_DEBUG_FREE)

static inline int kmem_cache_debug(struct kmem_cache *s)
{
117
#ifdef CONFIG_SLUB_DEBUG
118
	return unlikely(s->flags & SLAB_DEBUG_FLAGS);
119
#else
120
	return 0;
121
#endif
122
}
123

C
Christoph Lameter 已提交
124 125 126 127 128 129 130 131 132 133 134
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST

135 136 137
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG

138 139 140 141
/*
 * Mininum number of partial slabs. These will be left on the partial
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
C
Christoph Lameter 已提交
142
#define MIN_PARTIAL 5
C
Christoph Lameter 已提交
143

144 145 146 147 148 149 150
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
 * sort the partial list by the number of objects in the.
 */
#define MAX_PARTIAL 10

C
Christoph Lameter 已提交
151 152
#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)
C
Christoph Lameter 已提交
153

154
/*
155 156 157
 * Debugging flags that require metadata to be stored in the slab.  These get
 * disabled when slub_debug=O is used and a cache's min order increases with
 * metadata.
158
 */
159
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
160

C
Christoph Lameter 已提交
161 162 163 164
/*
 * Set of flags that will prevent slab merging
 */
#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
165 166
		SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
		SLAB_FAILSLAB)
C
Christoph Lameter 已提交
167 168

#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
V
Vegard Nossum 已提交
169
		SLAB_CACHE_DMA | SLAB_NOTRACK)
C
Christoph Lameter 已提交
170

171 172
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
173
#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */
174

C
Christoph Lameter 已提交
175
/* Internal SLUB flags */
C
Christoph Lameter 已提交
176
#define __OBJECT_POISON		0x80000000UL /* Poison object */
177
#define __CMPXCHG_DOUBLE	0x40000000UL /* Use cmpxchg_double */
C
Christoph Lameter 已提交
178 179 180 181 182 183 184 185 186

static int kmem_size = sizeof(struct kmem_cache);

#ifdef CONFIG_SMP
static struct notifier_block slab_notifier;
#endif

static enum {
	DOWN,		/* No slab functionality available */
187
	PARTIAL,	/* Kmem_cache_node works */
C
Christoph Lameter 已提交
188
	UP,		/* Everything works but does not show up in sysfs */
C
Christoph Lameter 已提交
189 190 191 192 193
	SYSFS		/* Sysfs up */
} slab_state = DOWN;

/* A list of all slab caches on the system */
static DECLARE_RWSEM(slub_lock);
A
Adrian Bunk 已提交
194
static LIST_HEAD(slab_caches);
C
Christoph Lameter 已提交
195

196 197 198
/*
 * Tracking user of a slab.
 */
199
#define TRACK_ADDRS_COUNT 16
200
struct track {
201
	unsigned long addr;	/* Called from address */
202 203 204
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
205 206 207 208 209 210 211
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

212
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
213 214 215
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
static void sysfs_slab_remove(struct kmem_cache *);
216

C
Christoph Lameter 已提交
217
#else
218 219 220
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
C
Christoph Lameter 已提交
221 222
static inline void sysfs_slab_remove(struct kmem_cache *s)
{
P
Pekka Enberg 已提交
223
	kfree(s->name);
C
Christoph Lameter 已提交
224 225
	kfree(s);
}
226

C
Christoph Lameter 已提交
227 228
#endif

229
static inline void stat(const struct kmem_cache *s, enum stat_item si)
230 231
{
#ifdef CONFIG_SLUB_STATS
232
	__this_cpu_inc(s->cpu_slab->stat[si]);
233 234 235
#endif
}

C
Christoph Lameter 已提交
236 237 238 239 240 241 242 243 244 245 246 247 248 249
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

int slab_is_available(void)
{
	return slab_state >= UP;
}

static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
	return s->node[node];
}

C
Christoph Lameter 已提交
250
/* Verify that a pointer has an address that is valid within a slab page */
251 252 253 254 255
static inline int check_valid_pointer(struct kmem_cache *s,
				struct page *page, const void *object)
{
	void *base;

256
	if (!object)
257 258
		return 1;

259
	base = page_address(page);
260
	if (object < base || object >= base + page->objects * s->size ||
261 262 263 264 265 266 267
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

268 269 270 271 272
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
	return *(void **)(object + s->offset);
}

273 274 275 276 277
static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{
	prefetch(object + s->offset);
}

278 279 280 281 282 283 284 285 286 287 288 289
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
	void *p;

#ifdef CONFIG_DEBUG_PAGEALLOC
	probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
#else
	p = get_freepointer(s, object);
#endif
	return p;
}

290 291 292 293 294 295
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
	*(void **)(object + s->offset) = fp;
}

/* Loop over all objects in a slab */
296 297
#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
298 299 300 301 302 303 304 305
			__p += (__s)->size)

/* Determine object index from a given position */
static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
{
	return (p - addr) / s->size;
}

306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
static inline size_t slab_ksize(const struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_DEBUG
	/*
	 * Debugging requires use of the padding between object
	 * and whatever may come after it.
	 */
	if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
		return s->objsize;

#endif
	/*
	 * If we have the need to store the freelist pointer
	 * back there or track user information then we can
	 * only use the space before that information.
	 */
	if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
		return s->inuse;
	/*
	 * Else we can use all the padding etc for the allocation
	 */
	return s->size;
}

330 331 332 333 334
static inline int order_objects(int order, unsigned long size, int reserved)
{
	return ((PAGE_SIZE << order) - reserved) / size;
}

335
static inline struct kmem_cache_order_objects oo_make(int order,
336
		unsigned long size, int reserved)
337 338
{
	struct kmem_cache_order_objects x = {
339
		(order << OO_SHIFT) + order_objects(order, size, reserved)
340 341 342 343 344 345 346
	};

	return x;
}

static inline int oo_order(struct kmem_cache_order_objects x)
{
347
	return x.x >> OO_SHIFT;
348 349 350 351
}

static inline int oo_objects(struct kmem_cache_order_objects x)
{
352
	return x.x & OO_MASK;
353 354
}

355 356 357 358 359 360 361 362 363 364 365 366 367
/*
 * Per slab locking using the pagelock
 */
static __always_inline void slab_lock(struct page *page)
{
	bit_spin_lock(PG_locked, &page->flags);
}

static __always_inline void slab_unlock(struct page *page)
{
	__bit_spin_unlock(PG_locked, &page->flags);
}

368 369 370 371 372 373 374
/* Interrupts must be disabled (for the fallback code to work right) */
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
	VM_BUG_ON(!irqs_disabled());
375 376
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
377
	if (s->flags & __CMPXCHG_DOUBLE) {
378
		if (cmpxchg_double(&page->freelist, &page->counters,
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
			freelist_old, counters_old,
			freelist_new, counters_new))
		return 1;
	} else
#endif
	{
		slab_lock(page);
		if (page->freelist == freelist_old && page->counters == counters_old) {
			page->freelist = freelist_new;
			page->counters = counters_new;
			slab_unlock(page);
			return 1;
		}
		slab_unlock(page);
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
	printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
#endif

	return 0;
}

405 406 407 408 409
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
410 411
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
412
	if (s->flags & __CMPXCHG_DOUBLE) {
413
		if (cmpxchg_double(&page->freelist, &page->counters,
414 415 416 417 418 419
			freelist_old, counters_old,
			freelist_new, counters_new))
		return 1;
	} else
#endif
	{
420 421 422
		unsigned long flags;

		local_irq_save(flags);
423
		slab_lock(page);
424 425 426
		if (page->freelist == freelist_old && page->counters == counters_old) {
			page->freelist = freelist_new;
			page->counters = counters_new;
427
			slab_unlock(page);
428
			local_irq_restore(flags);
429 430
			return 1;
		}
431
		slab_unlock(page);
432
		local_irq_restore(flags);
433 434 435 436 437 438 439 440 441 442 443 444
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
	printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
#endif

	return 0;
}

C
Christoph Lameter 已提交
445
#ifdef CONFIG_SLUB_DEBUG
446 447 448
/*
 * Determine a map of object in use on a page.
 *
449
 * Node listlock must be held to guarantee that the page does
450 451 452 453 454 455 456 457 458 459 460
 * not vanish from under us.
 */
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
{
	void *p;
	void *addr = page_address(page);

	for (p = page->freelist; p; p = get_freepointer(s, p))
		set_bit(slab_index(p, s, addr), map);
}

C
Christoph Lameter 已提交
461 462 463
/*
 * Debug settings:
 */
464 465 466
#ifdef CONFIG_SLUB_DEBUG_ON
static int slub_debug = DEBUG_DEFAULT_FLAGS;
#else
C
Christoph Lameter 已提交
467
static int slub_debug;
468
#endif
C
Christoph Lameter 已提交
469 470

static char *slub_debug_slabs;
471
static int disable_higher_order_debug;
C
Christoph Lameter 已提交
472

C
Christoph Lameter 已提交
473 474 475 476 477
/*
 * Object debugging
 */
static void print_section(char *text, u8 *addr, unsigned int length)
{
478 479
	print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
			length, 1);
C
Christoph Lameter 已提交
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
}

static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
496
			enum track_item alloc, unsigned long addr)
C
Christoph Lameter 已提交
497
{
A
Akinobu Mita 已提交
498
	struct track *p = get_track(s, object, alloc);
C
Christoph Lameter 已提交
499 500

	if (addr) {
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
#ifdef CONFIG_STACKTRACE
		struct stack_trace trace;
		int i;

		trace.nr_entries = 0;
		trace.max_entries = TRACK_ADDRS_COUNT;
		trace.entries = p->addrs;
		trace.skip = 3;
		save_stack_trace(&trace);

		/* See rant in lockdep.c */
		if (trace.nr_entries != 0 &&
		    trace.entries[trace.nr_entries - 1] == ULONG_MAX)
			trace.nr_entries--;

		for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
			p->addrs[i] = 0;
#endif
C
Christoph Lameter 已提交
519 520
		p->addr = addr;
		p->cpu = smp_processor_id();
A
Alexey Dobriyan 已提交
521
		p->pid = current->pid;
C
Christoph Lameter 已提交
522 523 524 525 526 527 528
		p->when = jiffies;
	} else
		memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
529 530 531
	if (!(s->flags & SLAB_STORE_USER))
		return;

532 533
	set_track(s, object, TRACK_FREE, 0UL);
	set_track(s, object, TRACK_ALLOC, 0UL);
C
Christoph Lameter 已提交
534 535 536 537 538 539 540
}

static void print_track(const char *s, struct track *t)
{
	if (!t->addr)
		return;

541
	printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
542
		s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
543 544 545 546 547 548 549 550 551 552
#ifdef CONFIG_STACKTRACE
	{
		int i;
		for (i = 0; i < TRACK_ADDRS_COUNT; i++)
			if (t->addrs[i])
				printk(KERN_ERR "\t%pS\n", (void *)t->addrs[i]);
			else
				break;
	}
#endif
553 554 555 556 557 558 559 560 561 562 563 564 565
}

static void print_tracking(struct kmem_cache *s, void *object)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	print_track("Allocated", get_track(s, object, TRACK_ALLOC));
	print_track("Freed", get_track(s, object, TRACK_FREE));
}

static void print_page_info(struct page *page)
{
566 567
	printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
		page, page->objects, page->inuse, page->freelist, page->flags);
568 569 570 571 572 573 574 575 576 577 578 579 580

}

static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[100];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "========================================"
			"=====================================\n");
581
	printk(KERN_ERR "BUG %s (%s): %s\n", s->name, print_tainted(), buf);
582 583
	printk(KERN_ERR "----------------------------------------"
			"-------------------------------------\n\n");
C
Christoph Lameter 已提交
584 585
}

586 587 588 589 590 591 592 593 594 595 596 597
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[100];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
}

static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
C
Christoph Lameter 已提交
598 599
{
	unsigned int off;	/* Offset of last byte */
600
	u8 *addr = page_address(page);
601 602 603 604 605 606 607 608 609

	print_tracking(s, p);

	print_page_info(page);

	printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
			p, p - addr, get_freepointer(s, p));

	if (p > addr + 16)
610
		print_section("Bytes b4 ", p - 16, 16);
C
Christoph Lameter 已提交
611

612 613
	print_section("Object ", p, min_t(unsigned long, s->objsize,
				PAGE_SIZE));
C
Christoph Lameter 已提交
614
	if (s->flags & SLAB_RED_ZONE)
615
		print_section("Redzone ", p + s->objsize,
C
Christoph Lameter 已提交
616 617 618 619 620 621 622
			s->inuse - s->objsize);

	if (s->offset)
		off = s->offset + sizeof(void *);
	else
		off = s->inuse;

623
	if (s->flags & SLAB_STORE_USER)
C
Christoph Lameter 已提交
624 625 626 627
		off += 2 * sizeof(struct track);

	if (off != s->size)
		/* Beginning of the filler is the free pointer */
628
		print_section("Padding ", p + off, s->size - off);
629 630

	dump_stack();
C
Christoph Lameter 已提交
631 632 633 634 635
}

static void object_err(struct kmem_cache *s, struct page *page,
			u8 *object, char *reason)
{
636
	slab_bug(s, "%s", reason);
637
	print_trailer(s, page, object);
C
Christoph Lameter 已提交
638 639
}

640
static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
C
Christoph Lameter 已提交
641 642 643 644
{
	va_list args;
	char buf[100];

645 646
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
C
Christoph Lameter 已提交
647
	va_end(args);
648
	slab_bug(s, "%s", buf);
649
	print_page_info(page);
C
Christoph Lameter 已提交
650 651 652
	dump_stack();
}

653
static void init_object(struct kmem_cache *s, void *object, u8 val)
C
Christoph Lameter 已提交
654 655 656 657 658
{
	u8 *p = object;

	if (s->flags & __OBJECT_POISON) {
		memset(p, POISON_FREE, s->objsize - 1);
P
Pekka Enberg 已提交
659
		p[s->objsize - 1] = POISON_END;
C
Christoph Lameter 已提交
660 661 662
	}

	if (s->flags & SLAB_RED_ZONE)
663
		memset(p + s->objsize, val, s->inuse - s->objsize);
C
Christoph Lameter 已提交
664 665
}

666 667 668 669 670 671 672 673 674
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
	memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
			u8 *object, char *what,
P
Pekka Enberg 已提交
675
			u8 *start, unsigned int value, unsigned int bytes)
676 677 678 679
{
	u8 *fault;
	u8 *end;

680
	fault = memchr_inv(start, value, bytes);
681 682 683 684 685 686 687 688 689 690 691 692 693 694
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
	printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
					fault, end - 1, fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
C
Christoph Lameter 已提交
695 696 697 698 699 700 701 702 703
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
 * 	pointer is the first word of the object.
C
Christoph Lameter 已提交
704
 *
C
Christoph Lameter 已提交
705 706 707 708 709
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
 * object + s->objsize
 * 	Padding to reach word boundary. This is also used for Redzoning.
C
Christoph Lameter 已提交
710 711 712
 * 	Padding is extended by another word if Redzoning is enabled and
 * 	objsize == inuse.
 *
C
Christoph Lameter 已提交
713 714 715 716
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
C
Christoph Lameter 已提交
717 718
 * 	Meta data starts here.
 *
C
Christoph Lameter 已提交
719 720
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
C
Christoph Lameter 已提交
721
 * 	C. Padding to reach required alignment boundary or at mininum
C
Christoph Lameter 已提交
722
 * 		one word if debugging is on to be able to detect writes
C
Christoph Lameter 已提交
723 724 725
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
C
Christoph Lameter 已提交
726 727
 *
 * object + s->size
C
Christoph Lameter 已提交
728
 * 	Nothing is used beyond s->size.
C
Christoph Lameter 已提交
729
 *
C
Christoph Lameter 已提交
730 731
 * If slabcaches are merged then the objsize and inuse boundaries are mostly
 * ignored. And therefore no slab options that rely on these boundaries
C
Christoph Lameter 已提交
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749
 * may be used with merged slabcaches.
 */

static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
{
	unsigned long off = s->inuse;	/* The end of info */

	if (s->offset)
		/* Freepointer is placed after the object. */
		off += sizeof(void *);

	if (s->flags & SLAB_STORE_USER)
		/* We also have user information there */
		off += 2 * sizeof(struct track);

	if (s->size == off)
		return 1;

750 751
	return check_bytes_and_report(s, page, p, "Object padding",
				p + off, POISON_INUSE, s->size - off);
C
Christoph Lameter 已提交
752 753
}

754
/* Check the pad bytes at the end of a slab page */
C
Christoph Lameter 已提交
755 756
static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
757 758 759 760 761
	u8 *start;
	u8 *fault;
	u8 *end;
	int length;
	int remainder;
C
Christoph Lameter 已提交
762 763 764 765

	if (!(s->flags & SLAB_POISON))
		return 1;

766
	start = page_address(page);
767
	length = (PAGE_SIZE << compound_order(page)) - s->reserved;
768 769
	end = start + length;
	remainder = length % s->size;
C
Christoph Lameter 已提交
770 771 772
	if (!remainder)
		return 1;

773
	fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
774 775 776 777 778 779
	if (!fault)
		return 1;
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
780
	print_section("Padding ", end - remainder, remainder);
781

E
Eric Dumazet 已提交
782
	restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
783
	return 0;
C
Christoph Lameter 已提交
784 785 786
}

static int check_object(struct kmem_cache *s, struct page *page,
787
					void *object, u8 val)
C
Christoph Lameter 已提交
788 789 790 791 792
{
	u8 *p = object;
	u8 *endobject = object + s->objsize;

	if (s->flags & SLAB_RED_ZONE) {
793
		if (!check_bytes_and_report(s, page, object, "Redzone",
794
			endobject, val, s->inuse - s->objsize))
C
Christoph Lameter 已提交
795 796
			return 0;
	} else {
I
Ingo Molnar 已提交
797 798 799 800
		if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
			check_bytes_and_report(s, page, p, "Alignment padding",
				endobject, POISON_INUSE, s->inuse - s->objsize);
		}
C
Christoph Lameter 已提交
801 802 803
	}

	if (s->flags & SLAB_POISON) {
804
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
805 806 807
			(!check_bytes_and_report(s, page, p, "Poison", p,
					POISON_FREE, s->objsize - 1) ||
			 !check_bytes_and_report(s, page, p, "Poison",
P
Pekka Enberg 已提交
808
				p + s->objsize - 1, POISON_END, 1)))
C
Christoph Lameter 已提交
809 810 811 812 813 814 815
			return 0;
		/*
		 * check_pad_bytes cleans up on its own.
		 */
		check_pad_bytes(s, page, p);
	}

816
	if (!s->offset && val == SLUB_RED_ACTIVE)
C
Christoph Lameter 已提交
817 818 819 820 821 822 823 824 825 826
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
	if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
		object_err(s, page, p, "Freepointer corrupt");
		/*
N
Nick Andrew 已提交
827
		 * No choice but to zap it and thus lose the remainder
C
Christoph Lameter 已提交
828
		 * of the free objects in this slab. May cause
C
Christoph Lameter 已提交
829
		 * another error because the object count is now wrong.
C
Christoph Lameter 已提交
830
		 */
831
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
832 833 834 835 836 837 838
		return 0;
	}
	return 1;
}

static int check_slab(struct kmem_cache *s, struct page *page)
{
839 840
	int maxobj;

C
Christoph Lameter 已提交
841 842 843
	VM_BUG_ON(!irqs_disabled());

	if (!PageSlab(page)) {
844
		slab_err(s, page, "Not a valid slab page");
C
Christoph Lameter 已提交
845 846
		return 0;
	}
847

848
	maxobj = order_objects(compound_order(page), s->size, s->reserved);
849 850 851 852 853 854
	if (page->objects > maxobj) {
		slab_err(s, page, "objects %u > max %u",
			s->name, page->objects, maxobj);
		return 0;
	}
	if (page->inuse > page->objects) {
855
		slab_err(s, page, "inuse %u > max %u",
856
			s->name, page->inuse, page->objects);
C
Christoph Lameter 已提交
857 858 859 860 861 862 863 864
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
	slab_pad_check(s, page);
	return 1;
}

/*
C
Christoph Lameter 已提交
865 866
 * Determine if a certain object on a page is on the freelist. Must hold the
 * slab lock to guarantee that the chains are in a consistent state.
C
Christoph Lameter 已提交
867 868 869 870
 */
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
	int nr = 0;
871
	void *fp;
C
Christoph Lameter 已提交
872
	void *object = NULL;
873
	unsigned long max_objects;
C
Christoph Lameter 已提交
874

875
	fp = page->freelist;
876
	while (fp && nr <= page->objects) {
C
Christoph Lameter 已提交
877 878 879 880 881 882
		if (fp == search)
			return 1;
		if (!check_valid_pointer(s, page, fp)) {
			if (object) {
				object_err(s, page, object,
					"Freechain corrupt");
883
				set_freepointer(s, object, NULL);
C
Christoph Lameter 已提交
884 885
				break;
			} else {
886
				slab_err(s, page, "Freepointer corrupt");
887
				page->freelist = NULL;
888
				page->inuse = page->objects;
889
				slab_fix(s, "Freelist cleared");
C
Christoph Lameter 已提交
890 891 892 893 894 895 896 897 898
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

899
	max_objects = order_objects(compound_order(page), s->size, s->reserved);
900 901
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
902 903 904 905 906 907 908

	if (page->objects != max_objects) {
		slab_err(s, page, "Wrong number of objects. Found %d but "
			"should be %d", page->objects, max_objects);
		page->objects = max_objects;
		slab_fix(s, "Number of objects adjusted.");
	}
909
	if (page->inuse != page->objects - nr) {
910
		slab_err(s, page, "Wrong object count. Counter is %d but "
911 912
			"counted were %d", page->inuse, page->objects - nr);
		page->inuse = page->objects - nr;
913
		slab_fix(s, "Object count adjusted.");
C
Christoph Lameter 已提交
914 915 916 917
	}
	return search == NULL;
}

918 919
static void trace(struct kmem_cache *s, struct page *page, void *object,
								int alloc)
C
Christoph Lameter 已提交
920 921 922 923 924 925 926 927 928
{
	if (s->flags & SLAB_TRACE) {
		printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
929
			print_section("Object ", (void *)object, s->objsize);
C
Christoph Lameter 已提交
930 931 932 933 934

		dump_stack();
	}
}

935 936 937 938 939 940
/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
 */
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
941
	flags &= gfp_allowed_mask;
942 943 944 945 946 947 948 949
	lockdep_trace_alloc(flags);
	might_sleep_if(flags & __GFP_WAIT);

	return should_failslab(s->objsize, flags, s->flags);
}

static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{
950
	flags &= gfp_allowed_mask;
951
	kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
952 953 954 955 956 957 958
	kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
}

static inline void slab_free_hook(struct kmem_cache *s, void *x)
{
	kmemleak_free_recursive(x, s->flags);

959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
	/*
	 * Trouble is that we may no longer disable interupts in the fast path
	 * So in order to make the debug calls that expect irqs to be
	 * disabled we need to disable interrupts temporarily.
	 */
#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
	{
		unsigned long flags;

		local_irq_save(flags);
		kmemcheck_slab_free(s, x, s->objsize);
		debug_check_no_locks_freed(x, s->objsize);
		local_irq_restore(flags);
	}
#endif
974 975
	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->objsize);
976 977
}

978
/*
C
Christoph Lameter 已提交
979
 * Tracking of fully allocated slabs for debugging purposes.
980 981
 *
 * list_lock must be held.
982
 */
983 984
static void add_full(struct kmem_cache *s,
	struct kmem_cache_node *n, struct page *page)
985
{
986 987 988
	if (!(s->flags & SLAB_STORE_USER))
		return;

989 990 991
	list_add(&page->lru, &n->full);
}

992 993 994
/*
 * list_lock must be held.
 */
995 996 997 998 999 1000 1001 1002
static void remove_full(struct kmem_cache *s, struct page *page)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	list_del(&page->lru);
}

1003 1004 1005 1006 1007 1008 1009 1010
/* Tracking of the number of slabs for debugging purposes */
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
{
	struct kmem_cache_node *n = get_node(s, node);

	return atomic_long_read(&n->nr_slabs);
}

1011 1012 1013 1014 1015
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

1016
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1017 1018 1019 1020 1021 1022 1023 1024 1025
{
	struct kmem_cache_node *n = get_node(s, node);

	/*
	 * May be called early in order to allocate a slab for the
	 * kmem_cache_node structure. Solve the chicken-egg
	 * dilemma by deferring the increment of the count during
	 * bootstrap (see early_kmem_cache_node_alloc).
	 */
1026
	if (n) {
1027
		atomic_long_inc(&n->nr_slabs);
1028 1029
		atomic_long_add(objects, &n->total_objects);
	}
1030
}
1031
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1032 1033 1034 1035
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
1036
	atomic_long_sub(objects, &n->total_objects);
1037 1038 1039
}

/* Object debug checks for alloc/free paths */
C
Christoph Lameter 已提交
1040 1041 1042 1043 1044 1045
static void setup_object_debug(struct kmem_cache *s, struct page *page,
								void *object)
{
	if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
		return;

1046
	init_object(s, object, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1047 1048 1049
	init_tracking(s, object);
}

1050
static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *page,
1051
					void *object, unsigned long addr)
C
Christoph Lameter 已提交
1052 1053 1054 1055 1056 1057
{
	if (!check_slab(s, page))
		goto bad;

	if (!check_valid_pointer(s, page, object)) {
		object_err(s, page, object, "Freelist Pointer check fails");
1058
		goto bad;
C
Christoph Lameter 已提交
1059 1060
	}

1061
	if (!check_object(s, page, object, SLUB_RED_INACTIVE))
C
Christoph Lameter 已提交
1062 1063
		goto bad;

C
Christoph Lameter 已提交
1064 1065 1066 1067
	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
	trace(s, page, object, 1);
1068
	init_object(s, object, SLUB_RED_ACTIVE);
C
Christoph Lameter 已提交
1069
	return 1;
C
Christoph Lameter 已提交
1070

C
Christoph Lameter 已提交
1071 1072 1073 1074 1075
bad:
	if (PageSlab(page)) {
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
C
Christoph Lameter 已提交
1076
		 * as used avoids touching the remaining objects.
C
Christoph Lameter 已提交
1077
		 */
1078
		slab_fix(s, "Marking all objects used");
1079
		page->inuse = page->objects;
1080
		page->freelist = NULL;
C
Christoph Lameter 已提交
1081 1082 1083 1084
	}
	return 0;
}

1085 1086
static noinline int free_debug_processing(struct kmem_cache *s,
		 struct page *page, void *object, unsigned long addr)
C
Christoph Lameter 已提交
1087
{
1088 1089 1090 1091
	unsigned long flags;
	int rc = 0;

	local_irq_save(flags);
1092 1093
	slab_lock(page);

C
Christoph Lameter 已提交
1094 1095 1096 1097
	if (!check_slab(s, page))
		goto fail;

	if (!check_valid_pointer(s, page, object)) {
1098
		slab_err(s, page, "Invalid object pointer 0x%p", object);
C
Christoph Lameter 已提交
1099 1100 1101 1102
		goto fail;
	}

	if (on_freelist(s, page, object)) {
1103
		object_err(s, page, object, "Object already free");
C
Christoph Lameter 已提交
1104 1105 1106
		goto fail;
	}

1107
	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1108
		goto out;
C
Christoph Lameter 已提交
1109 1110

	if (unlikely(s != page->slab)) {
I
Ingo Molnar 已提交
1111
		if (!PageSlab(page)) {
1112 1113
			slab_err(s, page, "Attempt to free object(0x%p) "
				"outside of slab", object);
I
Ingo Molnar 已提交
1114
		} else if (!page->slab) {
C
Christoph Lameter 已提交
1115
			printk(KERN_ERR
1116
				"SLUB <none>: no slab for object 0x%p.\n",
C
Christoph Lameter 已提交
1117
						object);
1118
			dump_stack();
P
Pekka Enberg 已提交
1119
		} else
1120 1121
			object_err(s, page, object,
					"page slab pointer corrupt.");
C
Christoph Lameter 已提交
1122 1123
		goto fail;
	}
C
Christoph Lameter 已提交
1124 1125 1126 1127

	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
1128
	init_object(s, object, SLUB_RED_INACTIVE);
1129 1130
	rc = 1;
out:
1131
	slab_unlock(page);
1132 1133
	local_irq_restore(flags);
	return rc;
C
Christoph Lameter 已提交
1134

C
Christoph Lameter 已提交
1135
fail:
1136
	slab_fix(s, "Object at 0x%p not freed", object);
1137
	goto out;
C
Christoph Lameter 已提交
1138 1139
}

C
Christoph Lameter 已提交
1140 1141
static int __init setup_slub_debug(char *str)
{
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
	slub_debug = DEBUG_DEFAULT_FLAGS;
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	if (*str == ',')
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
		goto check_slabs;

1156 1157 1158 1159 1160 1161 1162 1163 1164
	if (tolower(*str) == 'o') {
		/*
		 * Avoid enabling debugging on caches if its minimum order
		 * would increase as a result.
		 */
		disable_higher_order_debug = 1;
		goto out;
	}

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
	slub_debug = 0;
	if (*str == '-')
		/*
		 * Switch off all debugging measures.
		 */
		goto out;

	/*
	 * Determine which debug features should be switched on
	 */
P
Pekka Enberg 已提交
1175
	for (; *str && *str != ','; str++) {
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
		switch (tolower(*str)) {
		case 'f':
			slub_debug |= SLAB_DEBUG_FREE;
			break;
		case 'z':
			slub_debug |= SLAB_RED_ZONE;
			break;
		case 'p':
			slub_debug |= SLAB_POISON;
			break;
		case 'u':
			slub_debug |= SLAB_STORE_USER;
			break;
		case 't':
			slub_debug |= SLAB_TRACE;
			break;
1192 1193 1194
		case 'a':
			slub_debug |= SLAB_FAILSLAB;
			break;
1195 1196
		default:
			printk(KERN_ERR "slub_debug option '%c' "
P
Pekka Enberg 已提交
1197
				"unknown. skipped\n", *str);
1198
		}
C
Christoph Lameter 已提交
1199 1200
	}

1201
check_slabs:
C
Christoph Lameter 已提交
1202 1203
	if (*str == ',')
		slub_debug_slabs = str + 1;
1204
out:
C
Christoph Lameter 已提交
1205 1206 1207 1208 1209
	return 1;
}

__setup("slub_debug", setup_slub_debug);

1210 1211
static unsigned long kmem_cache_flags(unsigned long objsize,
	unsigned long flags, const char *name,
1212
	void (*ctor)(void *))
C
Christoph Lameter 已提交
1213 1214
{
	/*
1215
	 * Enable debugging if selected on the kernel commandline.
C
Christoph Lameter 已提交
1216
	 */
1217
	if (slub_debug && (!slub_debug_slabs ||
1218 1219
		!strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
		flags |= slub_debug;
1220 1221

	return flags;
C
Christoph Lameter 已提交
1222 1223
}
#else
C
Christoph Lameter 已提交
1224 1225
static inline void setup_object_debug(struct kmem_cache *s,
			struct page *page, void *object) {}
C
Christoph Lameter 已提交
1226

C
Christoph Lameter 已提交
1227
static inline int alloc_debug_processing(struct kmem_cache *s,
1228
	struct page *page, void *object, unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1229

C
Christoph Lameter 已提交
1230
static inline int free_debug_processing(struct kmem_cache *s,
1231
	struct page *page, void *object, unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1232 1233 1234 1235

static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
			{ return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page,
1236
			void *object, u8 val) { return 1; }
1237 1238
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
					struct page *page) {}
1239
static inline void remove_full(struct kmem_cache *s, struct page *page) {}
1240 1241
static inline unsigned long kmem_cache_flags(unsigned long objsize,
	unsigned long flags, const char *name,
1242
	void (*ctor)(void *))
1243 1244 1245
{
	return flags;
}
C
Christoph Lameter 已提交
1246
#define slub_debug 0
1247

1248 1249
#define disable_higher_order_debug 0

1250 1251
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }
1252 1253
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1254 1255 1256 1257
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1258 1259 1260 1261 1262 1263 1264 1265 1266

static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
							{ return 0; }

static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
		void *object) {}

static inline void slab_free_hook(struct kmem_cache *s, void *x) {}

1267
#endif /* CONFIG_SLUB_DEBUG */
1268

C
Christoph Lameter 已提交
1269 1270 1271
/*
 * Slab allocation and freeing
 */
1272 1273 1274 1275 1276
static inline struct page *alloc_slab_page(gfp_t flags, int node,
					struct kmem_cache_order_objects oo)
{
	int order = oo_order(oo);

1277 1278
	flags |= __GFP_NOTRACK;

1279
	if (node == NUMA_NO_NODE)
1280 1281
		return alloc_pages(flags, order);
	else
1282
		return alloc_pages_exact_node(node, flags, order);
1283 1284
}

C
Christoph Lameter 已提交
1285 1286
static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
{
P
Pekka Enberg 已提交
1287
	struct page *page;
1288
	struct kmem_cache_order_objects oo = s->oo;
1289
	gfp_t alloc_gfp;
C
Christoph Lameter 已提交
1290

1291 1292 1293 1294 1295
	flags &= gfp_allowed_mask;

	if (flags & __GFP_WAIT)
		local_irq_enable();

1296
	flags |= s->allocflags;
1297

1298 1299 1300 1301 1302 1303 1304
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;

	page = alloc_slab_page(alloc_gfp, node, oo);
1305 1306 1307 1308 1309 1310 1311
	if (unlikely(!page)) {
		oo = s->min;
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
		page = alloc_slab_page(flags, node, oo);
C
Christoph Lameter 已提交
1312

1313 1314
		if (page)
			stat(s, ORDER_FALLBACK);
1315
	}
V
Vegard Nossum 已提交
1316

1317 1318 1319 1320 1321 1322
	if (flags & __GFP_WAIT)
		local_irq_disable();

	if (!page)
		return NULL;

V
Vegard Nossum 已提交
1323
	if (kmemcheck_enabled
1324
		&& !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
		int pages = 1 << oo_order(oo);

		kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);

		/*
		 * Objects from caches that have a constructor don't get
		 * cleared when they're allocated, so we need to do it here.
		 */
		if (s->ctor)
			kmemcheck_mark_uninitialized_pages(page, pages);
		else
			kmemcheck_mark_unallocated_pages(page, pages);
V
Vegard Nossum 已提交
1337 1338
	}

1339
	page->objects = oo_objects(oo);
C
Christoph Lameter 已提交
1340 1341 1342
	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1343
		1 << oo_order(oo));
C
Christoph Lameter 已提交
1344 1345 1346 1347 1348 1349 1350

	return page;
}

static void setup_object(struct kmem_cache *s, struct page *page,
				void *object)
{
C
Christoph Lameter 已提交
1351
	setup_object_debug(s, page, object);
1352
	if (unlikely(s->ctor))
1353
		s->ctor(object);
C
Christoph Lameter 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362
}

static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
	struct page *page;
	void *start;
	void *last;
	void *p;

C
Christoph Lameter 已提交
1363
	BUG_ON(flags & GFP_SLAB_BUG_MASK);
C
Christoph Lameter 已提交
1364

C
Christoph Lameter 已提交
1365 1366
	page = allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
C
Christoph Lameter 已提交
1367 1368 1369
	if (!page)
		goto out;

1370
	inc_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1371 1372 1373 1374 1375 1376
	page->slab = s;
	page->flags |= 1 << PG_slab;

	start = page_address(page);

	if (unlikely(s->flags & SLAB_POISON))
1377
		memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
C
Christoph Lameter 已提交
1378 1379

	last = start;
1380
	for_each_object(p, s, start, page->objects) {
C
Christoph Lameter 已提交
1381 1382 1383 1384 1385
		setup_object(s, page, last);
		set_freepointer(s, last, p);
		last = p;
	}
	setup_object(s, page, last);
1386
	set_freepointer(s, last, NULL);
C
Christoph Lameter 已提交
1387 1388

	page->freelist = start;
1389
	page->inuse = page->objects;
1390
	page->frozen = 1;
C
Christoph Lameter 已提交
1391 1392 1393 1394 1395 1396
out:
	return page;
}

static void __free_slab(struct kmem_cache *s, struct page *page)
{
1397 1398
	int order = compound_order(page);
	int pages = 1 << order;
C
Christoph Lameter 已提交
1399

1400
	if (kmem_cache_debug(s)) {
C
Christoph Lameter 已提交
1401 1402 1403
		void *p;

		slab_pad_check(s, page);
1404 1405
		for_each_object(p, s, page_address(page),
						page->objects)
1406
			check_object(s, page, p, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1407 1408
	}

1409
	kmemcheck_free_shadow(page, compound_order(page));
V
Vegard Nossum 已提交
1410

C
Christoph Lameter 已提交
1411 1412 1413
	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
P
Pekka Enberg 已提交
1414
		-pages);
C
Christoph Lameter 已提交
1415

1416 1417
	__ClearPageSlab(page);
	reset_page_mapcount(page);
N
Nick Piggin 已提交
1418 1419
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
1420
	__free_pages(page, order);
C
Christoph Lameter 已提交
1421 1422
}

1423 1424 1425
#define need_reserve_slab_rcu						\
	(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))

C
Christoph Lameter 已提交
1426 1427 1428 1429
static void rcu_free_slab(struct rcu_head *h)
{
	struct page *page;

1430 1431 1432 1433 1434
	if (need_reserve_slab_rcu)
		page = virt_to_head_page(h);
	else
		page = container_of((struct list_head *)h, struct page, lru);

C
Christoph Lameter 已提交
1435 1436 1437 1438 1439 1440
	__free_slab(page->slab, page);
}

static void free_slab(struct kmem_cache *s, struct page *page)
{
	if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
		struct rcu_head *head;

		if (need_reserve_slab_rcu) {
			int order = compound_order(page);
			int offset = (PAGE_SIZE << order) - s->reserved;

			VM_BUG_ON(s->reserved != sizeof(*head));
			head = page_address(page) + offset;
		} else {
			/*
			 * RCU free overloads the RCU head over the LRU
			 */
			head = (void *)&page->lru;
		}
C
Christoph Lameter 已提交
1455 1456 1457 1458 1459 1460 1461 1462

		call_rcu(head, rcu_free_slab);
	} else
		__free_slab(s, page);
}

static void discard_slab(struct kmem_cache *s, struct page *page)
{
1463
	dec_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1464 1465 1466 1467
	free_slab(s, page);
}

/*
1468 1469 1470
 * Management of partially allocated slabs.
 *
 * list_lock must be held.
C
Christoph Lameter 已提交
1471
 */
1472
static inline void add_partial(struct kmem_cache_node *n,
1473
				struct page *page, int tail)
C
Christoph Lameter 已提交
1474
{
C
Christoph Lameter 已提交
1475
	n->nr_partial++;
1476
	if (tail == DEACTIVATE_TO_TAIL)
1477 1478 1479
		list_add_tail(&page->lru, &n->partial);
	else
		list_add(&page->lru, &n->partial);
C
Christoph Lameter 已提交
1480 1481
}

1482 1483 1484 1485
/*
 * list_lock must be held.
 */
static inline void remove_partial(struct kmem_cache_node *n,
1486 1487 1488 1489 1490 1491
					struct page *page)
{
	list_del(&page->lru);
	n->nr_partial--;
}

C
Christoph Lameter 已提交
1492
/*
1493 1494
 * Lock slab, remove from the partial list and put the object into the
 * per cpu freelist.
C
Christoph Lameter 已提交
1495
 *
1496 1497
 * Returns a list of objects or NULL if it fails.
 *
C
Christoph Lameter 已提交
1498
 * Must hold list_lock.
C
Christoph Lameter 已提交
1499
 */
1500
static inline void *acquire_slab(struct kmem_cache *s,
1501
		struct kmem_cache_node *n, struct page *page,
1502
		int mode)
C
Christoph Lameter 已提交
1503
{
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
	void *freelist;
	unsigned long counters;
	struct page new;

	/*
	 * Zap the freelist and set the frozen bit.
	 * The old freelist is the list of objects for the
	 * per cpu allocation list.
	 */
	do {
		freelist = page->freelist;
		counters = page->counters;
		new.counters = counters;
1517 1518
		if (mode)
			new.inuse = page->objects;
1519 1520 1521 1522

		VM_BUG_ON(new.frozen);
		new.frozen = 1;

1523
	} while (!__cmpxchg_double_slab(s, page,
1524 1525 1526 1527 1528
			freelist, counters,
			NULL, new.counters,
			"lock and freeze"));

	remove_partial(n, page);
1529
	return freelist;
C
Christoph Lameter 已提交
1530 1531
}

1532 1533
static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

C
Christoph Lameter 已提交
1534
/*
C
Christoph Lameter 已提交
1535
 * Try to allocate a partial slab from a specific node.
C
Christoph Lameter 已提交
1536
 */
1537
static void *get_partial_node(struct kmem_cache *s,
1538
		struct kmem_cache_node *n, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1539
{
1540 1541
	struct page *page, *page2;
	void *object = NULL;
C
Christoph Lameter 已提交
1542 1543 1544 1545

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
C
Christoph Lameter 已提交
1546 1547
	 * partial slab and there is none available then get_partials()
	 * will return NULL.
C
Christoph Lameter 已提交
1548 1549 1550 1551 1552
	 */
	if (!n || !n->nr_partial)
		return NULL;

	spin_lock(&n->list_lock);
1553
	list_for_each_entry_safe(page, page2, &n->partial, lru) {
1554
		void *t = acquire_slab(s, n, page, object == NULL);
1555 1556 1557 1558 1559
		int available;

		if (!t)
			break;

1560
		if (!object) {
1561 1562 1563 1564 1565 1566 1567 1568
			c->page = page;
			c->node = page_to_nid(page);
			stat(s, ALLOC_FROM_PARTIAL);
			object = t;
			available =  page->objects - page->inuse;
		} else {
			page->freelist = t;
			available = put_cpu_partial(s, page, 0);
1569
			stat(s, CPU_PARTIAL_NODE);
1570 1571 1572 1573
		}
		if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
			break;

1574
	}
C
Christoph Lameter 已提交
1575
	spin_unlock(&n->list_lock);
1576
	return object;
C
Christoph Lameter 已提交
1577 1578 1579
}

/*
C
Christoph Lameter 已提交
1580
 * Get a page from somewhere. Search in increasing NUMA distances.
C
Christoph Lameter 已提交
1581
 */
1582 1583
static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1584 1585 1586
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
1587
	struct zoneref *z;
1588 1589
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
1590
	void *object;
1591
	unsigned int cpuset_mems_cookie;
C
Christoph Lameter 已提交
1592 1593

	/*
C
Christoph Lameter 已提交
1594 1595 1596 1597
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
C
Christoph Lameter 已提交
1598
	 *
C
Christoph Lameter 已提交
1599 1600 1601 1602
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
C
Christoph Lameter 已提交
1603
	 *
C
Christoph Lameter 已提交
1604
	 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
C
Christoph Lameter 已提交
1605 1606 1607 1608 1609
	 * defrag_ratio = 1000) then every (well almost) allocation will
	 * first attempt to defrag slab caches on other nodes. This means
	 * scanning over all nodes to look for partial slabs which may be
	 * expensive if we do it every time we are trying to find a slab
	 * with available objects.
C
Christoph Lameter 已提交
1610
	 */
1611 1612
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
C
Christoph Lameter 已提交
1613 1614
		return NULL;

1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	do {
		cpuset_mems_cookie = get_mems_allowed();
		zonelist = node_zonelist(slab_node(current->mempolicy), flags);
		for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
			struct kmem_cache_node *n;

			n = get_node(s, zone_to_nid(zone));

			if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
					n->nr_partial > s->min_partial) {
				object = get_partial_node(s, n, c);
				if (object) {
					/*
					 * Return the object even if
					 * put_mems_allowed indicated that
					 * the cpuset mems_allowed was
					 * updated in parallel. It's a
					 * harmless race between the alloc
					 * and the cpuset update.
					 */
					put_mems_allowed(cpuset_mems_cookie);
					return object;
				}
1638
			}
C
Christoph Lameter 已提交
1639
		}
1640
	} while (!put_mems_allowed(cpuset_mems_cookie));
C
Christoph Lameter 已提交
1641 1642 1643 1644 1645 1646 1647
#endif
	return NULL;
}

/*
 * Get a partial page, lock it and return it.
 */
1648
static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
1649
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1650
{
1651
	void *object;
1652
	int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
C
Christoph Lameter 已提交
1653

1654 1655 1656
	object = get_partial_node(s, get_node(s, searchnode), c);
	if (object || node != NUMA_NO_NODE)
		return object;
C
Christoph Lameter 已提交
1657

1658
	return get_any_partial(s, flags, c);
C
Christoph Lameter 已提交
1659 1660
}

1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
#ifdef CONFIG_PREEMPT
/*
 * Calculate the next globally unique transaction for disambiguiation
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
#endif

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

	printk(KERN_INFO "%s %s: cmpxchg redo ", n, s->name);

#ifdef CONFIG_PREEMPT
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
		printk("due to cpu change %d -> %d\n",
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
		printk("due to cpu running other code. Event %ld->%ld\n",
			tid_to_event(tid), tid_to_event(actual_tid));
	else
		printk("for unknown reason: actual=%lx was=%lx target=%lx\n",
			actual_tid, tid, next_tid(tid));
#endif
1717
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1718 1719 1720 1721 1722 1723 1724 1725 1726
}

void init_kmem_cache_cpus(struct kmem_cache *s)
{
	int cpu;

	for_each_possible_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
}
1727

C
Christoph Lameter 已提交
1728 1729 1730
/*
 * Remove the cpu slab
 */
1731
static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1732
{
1733
	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
1734
	struct page *page = c->page;
1735 1736 1737 1738 1739
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	int lock = 0;
	enum slab_modes l = M_NONE, m = M_NONE;
	void *freelist;
	void *nextfree;
1740
	int tail = DEACTIVATE_TO_HEAD;
1741 1742 1743 1744
	struct page new;
	struct page old;

	if (page->freelist) {
1745
		stat(s, DEACTIVATE_REMOTE_FREES);
1746
		tail = DEACTIVATE_TO_TAIL;
1747 1748 1749 1750 1751 1752 1753
	}

	c->tid = next_tid(c->tid);
	c->page = NULL;
	freelist = c->freelist;
	c->freelist = NULL;

1754
	/*
1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
	 * Stage one: Free all available per cpu objects back
	 * to the page freelist while it is still frozen. Leave the
	 * last one.
	 *
	 * There is no need to take the list->lock because the page
	 * is still frozen.
	 */
	while (freelist && (nextfree = get_freepointer(s, freelist))) {
		void *prior;
		unsigned long counters;

		do {
			prior = page->freelist;
			counters = page->counters;
			set_freepointer(s, freelist, prior);
			new.counters = counters;
			new.inuse--;
			VM_BUG_ON(!new.frozen);

1774
		} while (!__cmpxchg_double_slab(s, page,
1775 1776 1777 1778 1779 1780 1781
			prior, counters,
			freelist, new.counters,
			"drain percpu freelist"));

		freelist = nextfree;
	}

1782
	/*
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
	 * Stage two: Ensure that the page is unfrozen while the
	 * list presence reflects the actual number of objects
	 * during unfreeze.
	 *
	 * We setup the list membership and then perform a cmpxchg
	 * with the count. If there is a mismatch then the page
	 * is not unfrozen but the page is on the wrong list.
	 *
	 * Then we restart the process which may have to remove
	 * the page from the list that we just put it on again
	 * because the number of objects in the slab may have
	 * changed.
1795
	 */
1796
redo:
1797

1798 1799 1800
	old.freelist = page->freelist;
	old.counters = page->counters;
	VM_BUG_ON(!old.frozen);
1801

1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
	/* Determine target state of the slab */
	new.counters = old.counters;
	if (freelist) {
		new.inuse--;
		set_freepointer(s, freelist, old.freelist);
		new.freelist = freelist;
	} else
		new.freelist = old.freelist;

	new.frozen = 0;

1813
	if (!new.inuse && n->nr_partial > s->min_partial)
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845
		m = M_FREE;
	else if (new.freelist) {
		m = M_PARTIAL;
		if (!lock) {
			lock = 1;
			/*
			 * Taking the spinlock removes the possiblity
			 * that acquire_slab() will see a slab page that
			 * is frozen
			 */
			spin_lock(&n->list_lock);
		}
	} else {
		m = M_FULL;
		if (kmem_cache_debug(s) && !lock) {
			lock = 1;
			/*
			 * This also ensures that the scanning of full
			 * slabs from diagnostic functions will not see
			 * any frozen slabs.
			 */
			spin_lock(&n->list_lock);
		}
	}

	if (l != m) {

		if (l == M_PARTIAL)

			remove_partial(n, page);

		else if (l == M_FULL)
1846

1847 1848 1849 1850 1851
			remove_full(s, page);

		if (m == M_PARTIAL) {

			add_partial(n, page, tail);
1852
			stat(s, tail);
1853 1854

		} else if (m == M_FULL) {
1855

1856 1857 1858 1859 1860 1861 1862
			stat(s, DEACTIVATE_FULL);
			add_full(s, n, page);

		}
	}

	l = m;
1863
	if (!__cmpxchg_double_slab(s, page,
1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"))
		goto redo;

	if (lock)
		spin_unlock(&n->list_lock);

	if (m == M_FREE) {
		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
1876
	}
C
Christoph Lameter 已提交
1877 1878
}

1879 1880 1881 1882 1883
/* Unfreeze all the cpu partial slabs */
static void unfreeze_partials(struct kmem_cache *s)
{
	struct kmem_cache_node *n = NULL;
	struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
1884
	struct page *page, *discard_page = NULL;
1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905

	while ((page = c->partial)) {
		enum slab_modes { M_PARTIAL, M_FREE };
		enum slab_modes l, m;
		struct page new;
		struct page old;

		c->partial = page->next;
		l = M_FREE;

		do {

			old.freelist = page->freelist;
			old.counters = page->counters;
			VM_BUG_ON(!old.frozen);

			new.counters = old.counters;
			new.freelist = old.freelist;

			new.frozen = 0;

1906
			if (!new.inuse && (!n || n->nr_partial > s->min_partial))
1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
				m = M_FREE;
			else {
				struct kmem_cache_node *n2 = get_node(s,
							page_to_nid(page));

				m = M_PARTIAL;
				if (n != n2) {
					if (n)
						spin_unlock(&n->list_lock);

					n = n2;
					spin_lock(&n->list_lock);
				}
			}

			if (l != m) {
S
Shaohua Li 已提交
1923
				if (l == M_PARTIAL) {
1924
					remove_partial(n, page);
S
Shaohua Li 已提交
1925 1926
					stat(s, FREE_REMOVE_PARTIAL);
				} else {
1927 1928
					add_partial(n, page,
						DEACTIVATE_TO_TAIL);
S
Shaohua Li 已提交
1929 1930
					stat(s, FREE_ADD_PARTIAL);
				}
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940

				l = m;
			}

		} while (!cmpxchg_double_slab(s, page,
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"));

		if (m == M_FREE) {
1941 1942
			page->next = discard_page;
			discard_page = page;
1943 1944 1945 1946 1947
		}
	}

	if (n)
		spin_unlock(&n->list_lock);
1948 1949 1950 1951 1952 1953 1954 1955 1956

	while (discard_page) {
		page = discard_page;
		discard_page = discard_page->next;

		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
	}
1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992
}

/*
 * Put a page that was just frozen (in __slab_free) into a partial page
 * slot if available. This is done without interrupts disabled and without
 * preemption disabled. The cmpxchg is racy and may put the partial page
 * onto a random cpus partial slot.
 *
 * If we did not find a slot then simply move all the partials to the
 * per node partial list.
 */
int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
{
	struct page *oldpage;
	int pages;
	int pobjects;

	do {
		pages = 0;
		pobjects = 0;
		oldpage = this_cpu_read(s->cpu_slab->partial);

		if (oldpage) {
			pobjects = oldpage->pobjects;
			pages = oldpage->pages;
			if (drain && pobjects > s->cpu_partial) {
				unsigned long flags;
				/*
				 * partial array is full. Move the existing
				 * set to the per node partial list.
				 */
				local_irq_save(flags);
				unfreeze_partials(s);
				local_irq_restore(flags);
				pobjects = 0;
				pages = 0;
1993
				stat(s, CPU_PARTIAL_DRAIN);
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
			}
		}

		pages++;
		pobjects += page->objects - page->inuse;

		page->pages = pages;
		page->pobjects = pobjects;
		page->next = oldpage;

2004
	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
2005 2006 2007
	return pobjects;
}

2008
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2009
{
2010
	stat(s, CPUSLAB_FLUSH);
2011
	deactivate_slab(s, c);
C
Christoph Lameter 已提交
2012 2013 2014 2015
}

/*
 * Flush cpu slab.
C
Christoph Lameter 已提交
2016
 *
C
Christoph Lameter 已提交
2017 2018
 * Called from IPI handler with interrupts disabled.
 */
2019
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
C
Christoph Lameter 已提交
2020
{
2021
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
C
Christoph Lameter 已提交
2022

2023 2024 2025 2026 2027 2028
	if (likely(c)) {
		if (c->page)
			flush_slab(s, c);

		unfreeze_partials(s);
	}
C
Christoph Lameter 已提交
2029 2030 2031 2032 2033 2034
}

static void flush_cpu_slab(void *d)
{
	struct kmem_cache *s = d;

2035
	__flush_cpu_slab(s, smp_processor_id());
C
Christoph Lameter 已提交
2036 2037
}

2038 2039 2040 2041 2042
static bool has_cpu_slab(int cpu, void *info)
{
	struct kmem_cache *s = info;
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

2043
	return c->page || c->partial;
2044 2045
}

C
Christoph Lameter 已提交
2046 2047
static void flush_all(struct kmem_cache *s)
{
2048
	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
C
Christoph Lameter 已提交
2049 2050
}

2051 2052 2053 2054 2055 2056 2057
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
static inline int node_match(struct kmem_cache_cpu *c, int node)
{
#ifdef CONFIG_NUMA
2058
	if (node != NUMA_NO_NODE && c->node != node)
2059 2060 2061 2062 2063
		return 0;
#endif
	return 1;
}

P
Pekka Enberg 已提交
2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
static int count_free(struct page *page)
{
	return page->objects - page->inuse;
}

static unsigned long count_partial(struct kmem_cache_node *n,
					int (*get_count)(struct page *))
{
	unsigned long flags;
	unsigned long x = 0;
	struct page *page;

	spin_lock_irqsave(&n->list_lock, flags);
	list_for_each_entry(page, &n->partial, lru)
		x += get_count(page);
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}

2083 2084 2085 2086 2087 2088 2089 2090 2091
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
#ifdef CONFIG_SLUB_DEBUG
	return atomic_long_read(&n->total_objects);
#else
	return 0;
#endif
}

P
Pekka Enberg 已提交
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
	int node;

	printk(KERN_WARNING
		"SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
		nid, gfpflags);
	printk(KERN_WARNING "  cache: %s, object size: %d, buffer size: %d, "
		"default order: %d, min order: %d\n", s->name, s->objsize,
		s->size, oo_order(s->oo), oo_order(s->min));

2104 2105 2106 2107
	if (oo_order(s->min) > get_order(s->objsize))
		printk(KERN_WARNING "  %s debugging increased min order, use "
		       "slub_debug=O to disable.\n", s->name);

P
Pekka Enberg 已提交
2108 2109 2110 2111 2112 2113 2114 2115 2116
	for_each_online_node(node) {
		struct kmem_cache_node *n = get_node(s, node);
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

		if (!n)
			continue;

2117 2118 2119
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
P
Pekka Enberg 已提交
2120 2121 2122 2123 2124 2125 2126

		printk(KERN_WARNING
			"  node %d: slabs: %ld, objs: %ld, free: %ld\n",
			node, nr_slabs, nr_objs, nr_free);
	}
}

2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155
static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
			int node, struct kmem_cache_cpu **pc)
{
	void *object;
	struct kmem_cache_cpu *c;
	struct page *page = new_slab(s, flags, node);

	if (page) {
		c = __this_cpu_ptr(s->cpu_slab);
		if (c->page)
			flush_slab(s, c);

		/*
		 * No other reference to the page yet so we can
		 * muck around with it freely without cmpxchg
		 */
		object = page->freelist;
		page->freelist = NULL;

		stat(s, ALLOC_SLAB);
		c->node = page_to_nid(page);
		c->page = page;
		*pc = c;
	} else
		object = NULL;

	return object;
}

2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
/*
 * Check the page->freelist of a page and either transfer the freelist to the per cpu freelist
 * or deactivate the page.
 *
 * The page is still frozen if the return value is not NULL.
 *
 * If this function returns NULL then the page has been unfrozen.
 */
static inline void *get_freelist(struct kmem_cache *s, struct page *page)
{
	struct page new;
	unsigned long counters;
	void *freelist;

	do {
		freelist = page->freelist;
		counters = page->counters;
		new.counters = counters;
		VM_BUG_ON(!new.frozen);

		new.inuse = page->objects;
		new.frozen = freelist != NULL;

	} while (!cmpxchg_double_slab(s, page,
		freelist, counters,
		NULL, new.counters,
		"get_freelist"));

	return freelist;
}

C
Christoph Lameter 已提交
2187
/*
2188 2189 2190 2191 2192 2193
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
C
Christoph Lameter 已提交
2194
 *
2195 2196 2197
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
C
Christoph Lameter 已提交
2198
 *
2199
 * And if we were unable to get a new slab from the partial slab lists then
C
Christoph Lameter 已提交
2200 2201
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
C
Christoph Lameter 已提交
2202
 */
2203 2204
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2205 2206
{
	void **object;
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPT
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif
C
Christoph Lameter 已提交
2218

2219
	if (!c->page)
C
Christoph Lameter 已提交
2220
		goto new_slab;
2221
redo:
2222
	if (unlikely(!node_match(c, node))) {
2223
		stat(s, ALLOC_NODE_MISMATCH);
2224 2225 2226
		deactivate_slab(s, c);
		goto new_slab;
	}
C
Christoph Lameter 已提交
2227

2228 2229 2230 2231
	/* must check again c->freelist in case of cpu migration or IRQ */
	object = c->freelist;
	if (object)
		goto load_freelist;
2232

2233
	stat(s, ALLOC_SLOWPATH);
2234

2235
	object = get_freelist(s, c->page);
C
Christoph Lameter 已提交
2236

2237
	if (!object) {
2238 2239
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
2240
		goto new_slab;
2241
	}
C
Christoph Lameter 已提交
2242

2243
	stat(s, ALLOC_REFILL);
C
Christoph Lameter 已提交
2244

2245
load_freelist:
2246
	c->freelist = get_freepointer(s, object);
2247 2248
	c->tid = next_tid(c->tid);
	local_irq_restore(flags);
C
Christoph Lameter 已提交
2249 2250 2251
	return object;

new_slab:
2252

2253 2254 2255 2256 2257 2258 2259
	if (c->partial) {
		c->page = c->partial;
		c->partial = c->page->next;
		c->node = page_to_nid(c->page);
		stat(s, CPU_PARTIAL_ALLOC);
		c->freelist = NULL;
		goto redo;
C
Christoph Lameter 已提交
2260 2261
	}

2262
	/* Then do expensive stuff like retrieving pages from the partial lists */
2263
	object = get_partial(s, gfpflags, node, c);
2264

2265
	if (unlikely(!object)) {
2266

2267
		object = new_slab_objects(s, gfpflags, node, &c);
2268

2269 2270 2271
		if (unlikely(!object)) {
			if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
				slab_out_of_memory(s, gfpflags, node);
2272

2273 2274 2275
			local_irq_restore(flags);
			return NULL;
		}
C
Christoph Lameter 已提交
2276
	}
2277

2278
	if (likely(!kmem_cache_debug(s)))
2279
		goto load_freelist;
2280

2281 2282 2283
	/* Only entered in the debug case */
	if (!alloc_debug_processing(s, c->page, object, addr))
		goto new_slab;	/* Slab failed checks. Next slab needed */
2284

2285
	c->freelist = get_freepointer(s, object);
2286
	deactivate_slab(s, c);
2287
	c->node = NUMA_NO_NODE;
2288 2289
	local_irq_restore(flags);
	return object;
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301
}

/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
P
Pekka Enberg 已提交
2302
static __always_inline void *slab_alloc(struct kmem_cache *s,
2303
		gfp_t gfpflags, int node, unsigned long addr)
2304 2305
{
	void **object;
2306
	struct kmem_cache_cpu *c;
2307
	unsigned long tid;
2308

2309
	if (slab_pre_alloc_hook(s, gfpflags))
A
Akinobu Mita 已提交
2310
		return NULL;
2311

2312 2313 2314 2315 2316 2317 2318 2319
redo:

	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
	 */
2320
	c = __this_cpu_ptr(s->cpu_slab);
2321 2322 2323 2324 2325 2326 2327 2328 2329 2330

	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */
	tid = c->tid;
	barrier();

2331 2332
	object = c->freelist;
	if (unlikely(!object || !node_match(c, node)))
2333

2334
		object = __slab_alloc(s, gfpflags, node, addr, c);
2335 2336

	else {
2337 2338
		void *next_object = get_freepointer_safe(s, object);

2339
		/*
L
Lucas De Marchi 已提交
2340
		 * The cmpxchg will only match if there was no additional
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350
		 * operation and if we are on the right processor.
		 *
		 * The cmpxchg does the following atomically (without lock semantics!)
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
		 * Since this is without lock semantics the protection is only against
		 * code executing on this cpu *not* from access by other cpus.
		 */
2351
		if (unlikely(!this_cpu_cmpxchg_double(
2352 2353
				s->cpu_slab->freelist, s->cpu_slab->tid,
				object, tid,
2354
				next_object, next_tid(tid)))) {
2355 2356 2357 2358

			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
2359
		prefetch_freepointer(s, next_object);
2360
		stat(s, ALLOC_FASTPATH);
2361
	}
2362

2363
	if (unlikely(gfpflags & __GFP_ZERO) && object)
2364
		memset(object, 0, s->objsize);
2365

2366
	slab_post_alloc_hook(s, gfpflags, object);
V
Vegard Nossum 已提交
2367

2368
	return object;
C
Christoph Lameter 已提交
2369 2370 2371 2372
}

void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
2373
	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2374

2375
	trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
E
Eduard - Gabriel Munteanu 已提交
2376 2377

	return ret;
C
Christoph Lameter 已提交
2378 2379 2380
}
EXPORT_SYMBOL(kmem_cache_alloc);

2381
#ifdef CONFIG_TRACING
2382 2383 2384 2385 2386 2387 2388 2389 2390
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
	return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);

void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
E
Eduard - Gabriel Munteanu 已提交
2391
{
2392 2393 2394
	void *ret = kmalloc_order(size, flags, order);
	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
2395
}
2396
EXPORT_SYMBOL(kmalloc_order_trace);
E
Eduard - Gabriel Munteanu 已提交
2397 2398
#endif

C
Christoph Lameter 已提交
2399 2400 2401
#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
E
Eduard - Gabriel Munteanu 已提交
2402 2403
	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);

2404 2405
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
				    s->objsize, s->size, gfpflags, node);
E
Eduard - Gabriel Munteanu 已提交
2406 2407

	return ret;
C
Christoph Lameter 已提交
2408 2409 2410
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

2411
#ifdef CONFIG_TRACING
2412
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
E
Eduard - Gabriel Munteanu 已提交
2413
				    gfp_t gfpflags,
2414
				    int node, size_t size)
E
Eduard - Gabriel Munteanu 已提交
2415
{
2416 2417 2418 2419 2420
	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);

	trace_kmalloc_node(_RET_IP_, ret,
			   size, s->size, gfpflags, node);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
2421
}
2422
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
E
Eduard - Gabriel Munteanu 已提交
2423
#endif
2424
#endif
E
Eduard - Gabriel Munteanu 已提交
2425

C
Christoph Lameter 已提交
2426
/*
2427 2428
 * Slow patch handling. This may still be called frequently since objects
 * have a longer lifetime than the cpu slabs in most processing loads.
C
Christoph Lameter 已提交
2429
 *
2430 2431 2432
 * So we still attempt to reduce cache line usage. Just take the slab
 * lock and free the item. If there is no additional partial page
 * handling required then we can return immediately.
C
Christoph Lameter 已提交
2433
 */
2434
static void __slab_free(struct kmem_cache *s, struct page *page,
2435
			void *x, unsigned long addr)
C
Christoph Lameter 已提交
2436 2437 2438
{
	void *prior;
	void **object = (void *)x;
2439 2440 2441 2442 2443
	int was_frozen;
	int inuse;
	struct page new;
	unsigned long counters;
	struct kmem_cache_node *n = NULL;
2444
	unsigned long uninitialized_var(flags);
C
Christoph Lameter 已提交
2445

2446
	stat(s, FREE_SLOWPATH);
C
Christoph Lameter 已提交
2447

2448
	if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
2449
		return;
C
Christoph Lameter 已提交
2450

2451 2452 2453 2454 2455 2456 2457 2458
	do {
		prior = page->freelist;
		counters = page->counters;
		set_freepointer(s, object, prior);
		new.counters = counters;
		was_frozen = new.frozen;
		new.inuse--;
		if ((!new.inuse || !prior) && !was_frozen && !n) {
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481

			if (!kmem_cache_debug(s) && !prior)

				/*
				 * Slab was on no list before and will be partially empty
				 * We can defer the list move and instead freeze it.
				 */
				new.frozen = 1;

			else { /* Needs to be taken off a list */

	                        n = get_node(s, page_to_nid(page));
				/*
				 * Speculatively acquire the list_lock.
				 * If the cmpxchg does not succeed then we may
				 * drop the list_lock without any processing.
				 *
				 * Otherwise the list_lock will synchronize with
				 * other processors updating the list of slabs.
				 */
				spin_lock_irqsave(&n->list_lock, flags);

			}
2482 2483
		}
		inuse = new.inuse;
C
Christoph Lameter 已提交
2484

2485 2486 2487 2488
	} while (!cmpxchg_double_slab(s, page,
		prior, counters,
		object, new.counters,
		"__slab_free"));
C
Christoph Lameter 已提交
2489

2490
	if (likely(!n)) {
2491 2492 2493 2494 2495

		/*
		 * If we just froze the page then put it onto the
		 * per cpu partial list.
		 */
2496
		if (new.frozen && !was_frozen) {
2497
			put_cpu_partial(s, page, 1);
2498 2499
			stat(s, CPU_PARTIAL_FREE);
		}
2500
		/*
2501 2502 2503 2504 2505
		 * The list lock was not taken therefore no list
		 * activity can be necessary.
		 */
                if (was_frozen)
                        stat(s, FREE_FROZEN);
2506
                return;
2507
        }
C
Christoph Lameter 已提交
2508 2509

	/*
2510 2511
	 * was_frozen may have been set after we acquired the list_lock in
	 * an earlier loop. So we need to check it here again.
C
Christoph Lameter 已提交
2512
	 */
2513 2514 2515 2516 2517
	if (was_frozen)
		stat(s, FREE_FROZEN);
	else {
		if (unlikely(!inuse && n->nr_partial > s->min_partial))
                        goto slab_empty;
C
Christoph Lameter 已提交
2518

2519 2520 2521 2522 2523 2524
		/*
		 * Objects left in the slab. If it was not on the partial list before
		 * then add it.
		 */
		if (unlikely(!prior)) {
			remove_full(s, page);
2525
			add_partial(n, page, DEACTIVATE_TO_TAIL);
2526 2527
			stat(s, FREE_ADD_PARTIAL);
		}
2528
	}
2529
	spin_unlock_irqrestore(&n->list_lock, flags);
C
Christoph Lameter 已提交
2530 2531 2532
	return;

slab_empty:
2533
	if (prior) {
C
Christoph Lameter 已提交
2534
		/*
2535
		 * Slab on the partial list.
C
Christoph Lameter 已提交
2536
		 */
2537
		remove_partial(n, page);
2538
		stat(s, FREE_REMOVE_PARTIAL);
2539 2540 2541
	} else
		/* Slab must be on the full list */
		remove_full(s, page);
2542

2543
	spin_unlock_irqrestore(&n->list_lock, flags);
2544
	stat(s, FREE_SLAB);
C
Christoph Lameter 已提交
2545 2546 2547
	discard_slab(s, page);
}

2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
 */
P
Pekka Enberg 已提交
2559
static __always_inline void slab_free(struct kmem_cache *s,
2560
			struct page *page, void *x, unsigned long addr)
2561 2562
{
	void **object = (void *)x;
2563
	struct kmem_cache_cpu *c;
2564
	unsigned long tid;
2565

2566 2567
	slab_free_hook(s, x);

2568 2569 2570 2571 2572 2573 2574
redo:
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
	 * during the cmpxchg then the free will succedd.
	 */
2575
	c = __this_cpu_ptr(s->cpu_slab);
2576

2577 2578
	tid = c->tid;
	barrier();
2579

2580
	if (likely(page == c->page)) {
2581
		set_freepointer(s, object, c->freelist);
2582

2583
		if (unlikely(!this_cpu_cmpxchg_double(
2584 2585 2586 2587 2588 2589 2590
				s->cpu_slab->freelist, s->cpu_slab->tid,
				c->freelist, tid,
				object, next_tid(tid)))) {

			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
2591
		stat(s, FREE_FASTPATH);
2592
	} else
2593
		__slab_free(s, page, x, addr);
2594 2595 2596

}

C
Christoph Lameter 已提交
2597 2598
void kmem_cache_free(struct kmem_cache *s, void *x)
{
C
Christoph Lameter 已提交
2599
	struct page *page;
C
Christoph Lameter 已提交
2600

2601
	page = virt_to_head_page(x);
C
Christoph Lameter 已提交
2602

2603
	slab_free(s, page, x, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2604

2605
	trace_kmem_cache_free(_RET_IP_, x);
C
Christoph Lameter 已提交
2606 2607 2608 2609
}
EXPORT_SYMBOL(kmem_cache_free);

/*
C
Christoph Lameter 已提交
2610 2611 2612 2613
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
C
Christoph Lameter 已提交
2614 2615 2616 2617
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
C
Christoph Lameter 已提交
2618
 * must be moved on and off the partial lists and is therefore a factor in
C
Christoph Lameter 已提交
2619 2620 2621 2622 2623 2624 2625 2626 2627 2628
 * locking overhead.
 */

/*
 * Mininum / Maximum order of slab pages. This influences locking overhead
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
static int slub_min_order;
2629
static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
2630
static int slub_min_objects;
C
Christoph Lameter 已提交
2631 2632 2633

/*
 * Merge control. If this is set then no merging of slab caches will occur.
C
Christoph Lameter 已提交
2634
 * (Could be removed. This was introduced to pacify the merge skeptics.)
C
Christoph Lameter 已提交
2635 2636 2637 2638 2639 2640
 */
static int slub_nomerge;

/*
 * Calculate the order of allocation given an slab object size.
 *
C
Christoph Lameter 已提交
2641 2642 2643 2644
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
C
Christoph Lameter 已提交
2645
 * unused space left. We go to a higher order if more than 1/16th of the slab
C
Christoph Lameter 已提交
2646 2647 2648 2649 2650 2651
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
C
Christoph Lameter 已提交
2652
 *
C
Christoph Lameter 已提交
2653 2654 2655 2656
 * slub_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slub_max_order then
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
C
Christoph Lameter 已提交
2657
 *
C
Christoph Lameter 已提交
2658 2659 2660 2661
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
 * requested a higher mininum order then we start with that one instead of
 * the smallest order which will fit the object.
C
Christoph Lameter 已提交
2662
 */
2663
static inline int slab_order(int size, int min_objects,
2664
				int max_order, int fract_leftover, int reserved)
C
Christoph Lameter 已提交
2665 2666 2667
{
	int order;
	int rem;
2668
	int min_order = slub_min_order;
C
Christoph Lameter 已提交
2669

2670
	if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
2671
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
2672

2673
	for (order = max(min_order,
2674 2675
				fls(min_objects * size - 1) - PAGE_SHIFT);
			order <= max_order; order++) {
C
Christoph Lameter 已提交
2676

2677
		unsigned long slab_size = PAGE_SIZE << order;
C
Christoph Lameter 已提交
2678

2679
		if (slab_size < min_objects * size + reserved)
C
Christoph Lameter 已提交
2680 2681
			continue;

2682
		rem = (slab_size - reserved) % size;
C
Christoph Lameter 已提交
2683

2684
		if (rem <= slab_size / fract_leftover)
C
Christoph Lameter 已提交
2685 2686 2687
			break;

	}
C
Christoph Lameter 已提交
2688

C
Christoph Lameter 已提交
2689 2690 2691
	return order;
}

2692
static inline int calculate_order(int size, int reserved)
2693 2694 2695 2696
{
	int order;
	int min_objects;
	int fraction;
2697
	int max_objects;
2698 2699 2700 2701 2702 2703 2704 2705 2706 2707

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 *
	 * First we reduce the acceptable waste in a slab. Then
	 * we reduce the minimum objects required in a slab.
	 */
	min_objects = slub_min_objects;
2708 2709
	if (!min_objects)
		min_objects = 4 * (fls(nr_cpu_ids) + 1);
2710
	max_objects = order_objects(slub_max_order, size, reserved);
2711 2712
	min_objects = min(min_objects, max_objects);

2713
	while (min_objects > 1) {
C
Christoph Lameter 已提交
2714
		fraction = 16;
2715 2716
		while (fraction >= 4) {
			order = slab_order(size, min_objects,
2717
					slub_max_order, fraction, reserved);
2718 2719 2720 2721
			if (order <= slub_max_order)
				return order;
			fraction /= 2;
		}
2722
		min_objects--;
2723 2724 2725 2726 2727 2728
	}

	/*
	 * We were unable to place multiple objects in a slab. Now
	 * lets see if we can place a single object there.
	 */
2729
	order = slab_order(size, 1, slub_max_order, 1, reserved);
2730 2731 2732 2733 2734 2735
	if (order <= slub_max_order)
		return order;

	/*
	 * Doh this slab cannot be placed using slub_max_order.
	 */
2736
	order = slab_order(size, 1, MAX_ORDER, 1, reserved);
D
David Rientjes 已提交
2737
	if (order < MAX_ORDER)
2738 2739 2740 2741
		return order;
	return -ENOSYS;
}

C
Christoph Lameter 已提交
2742
/*
C
Christoph Lameter 已提交
2743
 * Figure out what the alignment of the objects will be.
C
Christoph Lameter 已提交
2744 2745 2746 2747 2748
 */
static unsigned long calculate_alignment(unsigned long flags,
		unsigned long align, unsigned long size)
{
	/*
C
Christoph Lameter 已提交
2749 2750
	 * If the user wants hardware cache aligned objects then follow that
	 * suggestion if the object is sufficiently large.
C
Christoph Lameter 已提交
2751
	 *
C
Christoph Lameter 已提交
2752 2753
	 * The hardware cache alignment cannot override the specified
	 * alignment though. If that is greater then use it.
C
Christoph Lameter 已提交
2754
	 */
2755 2756 2757 2758 2759 2760
	if (flags & SLAB_HWCACHE_ALIGN) {
		unsigned long ralign = cache_line_size();
		while (size <= ralign / 2)
			ralign /= 2;
		align = max(align, ralign);
	}
C
Christoph Lameter 已提交
2761 2762

	if (align < ARCH_SLAB_MINALIGN)
2763
		align = ARCH_SLAB_MINALIGN;
C
Christoph Lameter 已提交
2764 2765 2766 2767

	return ALIGN(align, sizeof(void *));
}

2768 2769
static void
init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
C
Christoph Lameter 已提交
2770 2771 2772 2773
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
2774
#ifdef CONFIG_SLUB_DEBUG
2775
	atomic_long_set(&n->nr_slabs, 0);
2776
	atomic_long_set(&n->total_objects, 0);
2777
	INIT_LIST_HEAD(&n->full);
2778
#endif
C
Christoph Lameter 已提交
2779 2780
}

2781
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
2782
{
2783 2784
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
			SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
2785

2786
	/*
2787 2788
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
2789
	 */
2790 2791
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));
2792 2793 2794 2795 2796

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
2797

2798
	return 1;
2799 2800
}

2801 2802
static struct kmem_cache *kmem_cache_node;

C
Christoph Lameter 已提交
2803 2804 2805 2806 2807 2808
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
 * Note that this function only works on the kmalloc_node_cache
2809 2810
 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
 * memory on a fresh node that has no slab structures yet.
C
Christoph Lameter 已提交
2811
 */
2812
static void early_kmem_cache_node_alloc(int node)
C
Christoph Lameter 已提交
2813 2814 2815 2816
{
	struct page *page;
	struct kmem_cache_node *n;

2817
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
C
Christoph Lameter 已提交
2818

2819
	page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
C
Christoph Lameter 已提交
2820 2821

	BUG_ON(!page);
2822 2823 2824 2825 2826 2827 2828
	if (page_to_nid(page) != node) {
		printk(KERN_ERR "SLUB: Unable to allocate memory from "
				"node %d\n", node);
		printk(KERN_ERR "SLUB: Allocating a useless per node structure "
				"in order to be able to continue\n");
	}

C
Christoph Lameter 已提交
2829 2830
	n = page->freelist;
	BUG_ON(!n);
2831
	page->freelist = get_freepointer(kmem_cache_node, n);
2832
	page->inuse = 1;
2833
	page->frozen = 0;
2834
	kmem_cache_node->node[node] = n;
2835
#ifdef CONFIG_SLUB_DEBUG
2836
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
2837
	init_tracking(kmem_cache_node, n);
2838
#endif
2839 2840
	init_kmem_cache_node(n, kmem_cache_node);
	inc_slabs_node(kmem_cache_node, node, page->objects);
C
Christoph Lameter 已提交
2841

2842
	add_partial(n, page, DEACTIVATE_TO_HEAD);
C
Christoph Lameter 已提交
2843 2844 2845 2846 2847 2848
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;

C
Christoph Lameter 已提交
2849
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
2850
		struct kmem_cache_node *n = s->node[node];
2851

2852
		if (n)
2853 2854
			kmem_cache_free(kmem_cache_node, n);

C
Christoph Lameter 已提交
2855 2856 2857 2858
		s->node[node] = NULL;
	}
}

2859
static int init_kmem_cache_nodes(struct kmem_cache *s)
C
Christoph Lameter 已提交
2860 2861 2862
{
	int node;

C
Christoph Lameter 已提交
2863
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
2864 2865
		struct kmem_cache_node *n;

2866
		if (slab_state == DOWN) {
2867
			early_kmem_cache_node_alloc(node);
2868 2869
			continue;
		}
2870
		n = kmem_cache_alloc_node(kmem_cache_node,
2871
						GFP_KERNEL, node);
C
Christoph Lameter 已提交
2872

2873 2874 2875
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
C
Christoph Lameter 已提交
2876
		}
2877

C
Christoph Lameter 已提交
2878
		s->node[node] = n;
2879
		init_kmem_cache_node(n, s);
C
Christoph Lameter 已提交
2880 2881 2882 2883
	}
	return 1;
}

2884
static void set_min_partial(struct kmem_cache *s, unsigned long min)
2885 2886 2887 2888 2889 2890 2891 2892
{
	if (min < MIN_PARTIAL)
		min = MIN_PARTIAL;
	else if (min > MAX_PARTIAL)
		min = MAX_PARTIAL;
	s->min_partial = min;
}

C
Christoph Lameter 已提交
2893 2894 2895 2896
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
2897
static int calculate_sizes(struct kmem_cache *s, int forced_order)
C
Christoph Lameter 已提交
2898 2899 2900 2901
{
	unsigned long flags = s->flags;
	unsigned long size = s->objsize;
	unsigned long align = s->align;
2902
	int order;
C
Christoph Lameter 已提交
2903

2904 2905 2906 2907 2908 2909 2910 2911
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
2912 2913 2914 2915 2916 2917
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
	if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
2918
			!s->ctor)
C
Christoph Lameter 已提交
2919 2920 2921 2922 2923 2924
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
C
Christoph Lameter 已提交
2925
	 * If we are Redzoning then check if there is some space between the
C
Christoph Lameter 已提交
2926
	 * end of the object and the free pointer. If not then add an
C
Christoph Lameter 已提交
2927
	 * additional word to have some bytes to store Redzone information.
C
Christoph Lameter 已提交
2928 2929 2930
	 */
	if ((flags & SLAB_RED_ZONE) && size == s->objsize)
		size += sizeof(void *);
C
Christoph Lameter 已提交
2931
#endif
C
Christoph Lameter 已提交
2932 2933

	/*
C
Christoph Lameter 已提交
2934 2935
	 * With that we have determined the number of bytes in actual use
	 * by the object. This is the potential offset to the free pointer.
C
Christoph Lameter 已提交
2936 2937 2938 2939
	 */
	s->inuse = size;

	if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
2940
		s->ctor)) {
C
Christoph Lameter 已提交
2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
		 * destructor or are poisoning the objects.
		 */
		s->offset = size;
		size += sizeof(void *);
	}

2953
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
2954 2955 2956 2957 2958 2959 2960
	if (flags & SLAB_STORE_USER)
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);

2961
	if (flags & SLAB_RED_ZONE)
C
Christoph Lameter 已提交
2962 2963 2964 2965
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
2966
		 * corrupted if a user writes before the start
C
Christoph Lameter 已提交
2967 2968 2969
		 * of the object.
		 */
		size += sizeof(void *);
C
Christoph Lameter 已提交
2970
#endif
C
Christoph Lameter 已提交
2971

C
Christoph Lameter 已提交
2972 2973
	/*
	 * Determine the alignment based on various parameters that the
2974 2975
	 * user specified and the dynamic determination of cache line size
	 * on bootup.
C
Christoph Lameter 已提交
2976 2977
	 */
	align = calculate_alignment(flags, align, s->objsize);
2978
	s->align = align;
C
Christoph Lameter 已提交
2979 2980 2981 2982 2983 2984 2985 2986

	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
	size = ALIGN(size, align);
	s->size = size;
2987 2988 2989
	if (forced_order >= 0)
		order = forced_order;
	else
2990
		order = calculate_order(size, s->reserved);
C
Christoph Lameter 已提交
2991

2992
	if (order < 0)
C
Christoph Lameter 已提交
2993 2994
		return 0;

2995
	s->allocflags = 0;
2996
	if (order)
2997 2998 2999 3000 3001 3002 3003 3004
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
		s->allocflags |= SLUB_DMA;

	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

C
Christoph Lameter 已提交
3005 3006 3007
	/*
	 * Determine the number of objects per slab
	 */
3008 3009
	s->oo = oo_make(order, size, s->reserved);
	s->min = oo_make(get_order(size), size, s->reserved);
3010 3011
	if (oo_objects(s->oo) > oo_objects(s->max))
		s->max = s->oo;
C
Christoph Lameter 已提交
3012

3013
	return !!oo_objects(s->oo);
C
Christoph Lameter 已提交
3014 3015 3016

}

3017
static int kmem_cache_open(struct kmem_cache *s,
C
Christoph Lameter 已提交
3018 3019
		const char *name, size_t size,
		size_t align, unsigned long flags,
3020
		void (*ctor)(void *))
C
Christoph Lameter 已提交
3021 3022 3023 3024 3025 3026
{
	memset(s, 0, kmem_size);
	s->name = name;
	s->ctor = ctor;
	s->objsize = size;
	s->align = align;
3027
	s->flags = kmem_cache_flags(size, flags, name, ctor);
3028
	s->reserved = 0;
C
Christoph Lameter 已提交
3029

3030 3031
	if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
		s->reserved = sizeof(struct rcu_head);
C
Christoph Lameter 已提交
3032

3033
	if (!calculate_sizes(s, -1))
C
Christoph Lameter 已提交
3034
		goto error;
3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
		if (get_order(s->size) > get_order(s->objsize)) {
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
			if (!calculate_sizes(s, -1))
				goto error;
		}
	}
C
Christoph Lameter 已提交
3047

3048 3049
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3050 3051 3052 3053 3054
	if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
		/* Enable fast mode */
		s->flags |= __CMPXCHG_DOUBLE;
#endif

3055 3056 3057 3058
	/*
	 * The larger the object size is, the more pages we want on the partial
	 * list to avoid pounding the page allocator excessively.
	 */
3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073
	set_min_partial(s, ilog2(s->size) / 2);

	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
	 * This setting also determines
	 *
	 * A) The number of objects from per cpu partial slabs dumped to the
	 *    per node list when we reach the limit.
3074
	 * B) The number of objects in cpu partial slabs to extract from the
3075 3076 3077
	 *    per node list when we run out of per cpu objects. We only fetch 50%
	 *    to keep some capacity around for frees.
	 */
3078 3079 3080
	if (kmem_cache_debug(s))
		s->cpu_partial = 0;
	else if (s->size >= PAGE_SIZE)
3081 3082 3083 3084 3085 3086 3087 3088
		s->cpu_partial = 2;
	else if (s->size >= 1024)
		s->cpu_partial = 6;
	else if (s->size >= 256)
		s->cpu_partial = 13;
	else
		s->cpu_partial = 30;

C
Christoph Lameter 已提交
3089 3090
	s->refcount = 1;
#ifdef CONFIG_NUMA
3091
	s->remote_node_defrag_ratio = 1000;
C
Christoph Lameter 已提交
3092
#endif
3093
	if (!init_kmem_cache_nodes(s))
3094
		goto error;
C
Christoph Lameter 已提交
3095

3096
	if (alloc_kmem_cache_cpus(s))
C
Christoph Lameter 已提交
3097
		return 1;
3098

3099
	free_kmem_cache_nodes(s);
C
Christoph Lameter 已提交
3100 3101 3102 3103
error:
	if (flags & SLAB_PANIC)
		panic("Cannot create slab %s size=%lu realsize=%u "
			"order=%u offset=%u flags=%lx\n",
3104
			s->name, (unsigned long)size, s->size, oo_order(s->oo),
C
Christoph Lameter 已提交
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117
			s->offset, flags);
	return 0;
}

/*
 * Determine the size of a slab object
 */
unsigned int kmem_cache_size(struct kmem_cache *s)
{
	return s->objsize;
}
EXPORT_SYMBOL(kmem_cache_size);

3118 3119 3120 3121 3122 3123
static void list_slab_objects(struct kmem_cache *s, struct page *page,
							const char *text)
{
#ifdef CONFIG_SLUB_DEBUG
	void *addr = page_address(page);
	void *p;
N
Namhyung Kim 已提交
3124 3125
	unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
				     sizeof(long), GFP_ATOMIC);
E
Eric Dumazet 已提交
3126 3127
	if (!map)
		return;
3128 3129 3130
	slab_err(s, page, "%s", text);
	slab_lock(page);

3131
	get_map(s, page, map);
3132 3133 3134 3135 3136 3137 3138 3139 3140
	for_each_object(p, s, addr, page->objects) {

		if (!test_bit(slab_index(p, s, addr), map)) {
			printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
							p, p - addr);
			print_tracking(s, p);
		}
	}
	slab_unlock(page);
E
Eric Dumazet 已提交
3141
	kfree(map);
3142 3143 3144
#endif
}

C
Christoph Lameter 已提交
3145
/*
C
Christoph Lameter 已提交
3146
 * Attempt to free all partial slabs on a node.
3147 3148
 * This is called from kmem_cache_close(). We must be the last thread
 * using the cache and therefore we do not need to lock anymore.
C
Christoph Lameter 已提交
3149
 */
C
Christoph Lameter 已提交
3150
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3151 3152 3153
{
	struct page *page, *h;

3154
	list_for_each_entry_safe(page, h, &n->partial, lru) {
C
Christoph Lameter 已提交
3155
		if (!page->inuse) {
3156
			remove_partial(n, page);
C
Christoph Lameter 已提交
3157
			discard_slab(s, page);
3158 3159 3160
		} else {
			list_slab_objects(s, page,
				"Objects remaining on kmem_cache_close()");
C
Christoph Lameter 已提交
3161
		}
3162
	}
C
Christoph Lameter 已提交
3163 3164 3165
}

/*
C
Christoph Lameter 已提交
3166
 * Release all resources used by a slab cache.
C
Christoph Lameter 已提交
3167
 */
3168
static inline int kmem_cache_close(struct kmem_cache *s)
C
Christoph Lameter 已提交
3169 3170 3171 3172
{
	int node;

	flush_all(s);
3173
	free_percpu(s->cpu_slab);
C
Christoph Lameter 已提交
3174
	/* Attempt to free all objects */
C
Christoph Lameter 已提交
3175
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
3176 3177
		struct kmem_cache_node *n = get_node(s, node);

C
Christoph Lameter 已提交
3178 3179
		free_partial(s, n);
		if (n->nr_partial || slabs_node(s, node))
C
Christoph Lameter 已提交
3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195
			return 1;
	}
	free_kmem_cache_nodes(s);
	return 0;
}

/*
 * Close a cache and release the kmem_cache structure
 * (must be used for caches created using kmem_cache_create)
 */
void kmem_cache_destroy(struct kmem_cache *s)
{
	down_write(&slub_lock);
	s->refcount--;
	if (!s->refcount) {
		list_del(&s->list);
3196
		up_write(&slub_lock);
3197 3198 3199 3200 3201
		if (kmem_cache_close(s)) {
			printk(KERN_ERR "SLUB %s: %s called for cache that "
				"still has objects.\n", s->name, __func__);
			dump_stack();
		}
3202 3203
		if (s->flags & SLAB_DESTROY_BY_RCU)
			rcu_barrier();
C
Christoph Lameter 已提交
3204
		sysfs_slab_remove(s);
3205 3206
	} else
		up_write(&slub_lock);
C
Christoph Lameter 已提交
3207 3208 3209 3210 3211 3212 3213
}
EXPORT_SYMBOL(kmem_cache_destroy);

/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

3214
struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
C
Christoph Lameter 已提交
3215 3216
EXPORT_SYMBOL(kmalloc_caches);

3217 3218
static struct kmem_cache *kmem_cache;

3219
#ifdef CONFIG_ZONE_DMA
3220
static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
3221 3222
#endif

C
Christoph Lameter 已提交
3223 3224
static int __init setup_slub_min_order(char *str)
{
P
Pekka Enberg 已提交
3225
	get_option(&str, &slub_min_order);
C
Christoph Lameter 已提交
3226 3227 3228 3229 3230 3231 3232 3233

	return 1;
}

__setup("slub_min_order=", setup_slub_min_order);

static int __init setup_slub_max_order(char *str)
{
P
Pekka Enberg 已提交
3234
	get_option(&str, &slub_max_order);
D
David Rientjes 已提交
3235
	slub_max_order = min(slub_max_order, MAX_ORDER - 1);
C
Christoph Lameter 已提交
3236 3237 3238 3239 3240 3241 3242 3243

	return 1;
}

__setup("slub_max_order=", setup_slub_max_order);

static int __init setup_slub_min_objects(char *str)
{
P
Pekka Enberg 已提交
3244
	get_option(&str, &slub_min_objects);
C
Christoph Lameter 已提交
3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258

	return 1;
}

__setup("slub_min_objects=", setup_slub_min_objects);

static int __init setup_slub_nomerge(char *str)
{
	slub_nomerge = 1;
	return 1;
}

__setup("slub_nomerge", setup_slub_nomerge);

3259 3260
static struct kmem_cache *__init create_kmalloc_cache(const char *name,
						int size, unsigned int flags)
C
Christoph Lameter 已提交
3261
{
3262 3263 3264 3265
	struct kmem_cache *s;

	s = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);

3266 3267 3268 3269
	/*
	 * This function is called with IRQs disabled during early-boot on
	 * single CPU so there's no need to take slub_lock here.
	 */
3270
	if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
3271
								flags, NULL))
C
Christoph Lameter 已提交
3272 3273 3274
		goto panic;

	list_add(&s->list, &slab_caches);
3275
	return s;
C
Christoph Lameter 已提交
3276 3277 3278

panic:
	panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
3279
	return NULL;
C
Christoph Lameter 已提交
3280 3281
}

3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314
/*
 * Conversion table for small slabs sizes / 8 to the index in the
 * kmalloc array. This is necessary for slabs < 192 since we have non power
 * of two cache sizes there. The size of larger slabs can be determined using
 * fls.
 */
static s8 size_index[24] = {
	3,	/* 8 */
	4,	/* 16 */
	5,	/* 24 */
	5,	/* 32 */
	6,	/* 40 */
	6,	/* 48 */
	6,	/* 56 */
	6,	/* 64 */
	1,	/* 72 */
	1,	/* 80 */
	1,	/* 88 */
	1,	/* 96 */
	7,	/* 104 */
	7,	/* 112 */
	7,	/* 120 */
	7,	/* 128 */
	2,	/* 136 */
	2,	/* 144 */
	2,	/* 152 */
	2,	/* 160 */
	2,	/* 168 */
	2,	/* 176 */
	2,	/* 184 */
	2	/* 192 */
};

3315 3316 3317 3318 3319
static inline int size_index_elem(size_t bytes)
{
	return (bytes - 1) / 8;
}

C
Christoph Lameter 已提交
3320 3321
static struct kmem_cache *get_slab(size_t size, gfp_t flags)
{
3322
	int index;
C
Christoph Lameter 已提交
3323

3324 3325 3326
	if (size <= 192) {
		if (!size)
			return ZERO_SIZE_PTR;
C
Christoph Lameter 已提交
3327

3328
		index = size_index[size_index_elem(size)];
3329
	} else
3330
		index = fls(size - 1);
C
Christoph Lameter 已提交
3331 3332

#ifdef CONFIG_ZONE_DMA
3333
	if (unlikely((flags & SLUB_DMA)))
3334
		return kmalloc_dma_caches[index];
3335

C
Christoph Lameter 已提交
3336
#endif
3337
	return kmalloc_caches[index];
C
Christoph Lameter 已提交
3338 3339 3340 3341
}

void *__kmalloc(size_t size, gfp_t flags)
{
3342
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
3343
	void *ret;
C
Christoph Lameter 已提交
3344

3345
	if (unlikely(size > SLUB_MAX_SIZE))
3346
		return kmalloc_large(size, flags);
3347 3348 3349 3350

	s = get_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3351 3352
		return s;

3353
	ret = slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3354

3355
	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3356 3357

	return ret;
C
Christoph Lameter 已提交
3358 3359 3360
}
EXPORT_SYMBOL(__kmalloc);

3361
#ifdef CONFIG_NUMA
3362 3363
static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
3364
	struct page *page;
3365
	void *ptr = NULL;
3366

3367 3368
	flags |= __GFP_COMP | __GFP_NOTRACK;
	page = alloc_pages_node(node, flags, get_order(size));
3369
	if (page)
3370 3371 3372 3373
		ptr = page_address(page);

	kmemleak_alloc(ptr, size, 1, flags);
	return ptr;
3374 3375
}

C
Christoph Lameter 已提交
3376 3377
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
3378
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
3379
	void *ret;
C
Christoph Lameter 已提交
3380

I
Ingo Molnar 已提交
3381
	if (unlikely(size > SLUB_MAX_SIZE)) {
E
Eduard - Gabriel Munteanu 已提交
3382 3383
		ret = kmalloc_large_node(size, flags, node);

3384 3385 3386
		trace_kmalloc_node(_RET_IP_, ret,
				   size, PAGE_SIZE << get_order(size),
				   flags, node);
E
Eduard - Gabriel Munteanu 已提交
3387 3388 3389

		return ret;
	}
3390 3391 3392 3393

	s = get_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3394 3395
		return s;

E
Eduard - Gabriel Munteanu 已提交
3396 3397
	ret = slab_alloc(s, flags, node, _RET_IP_);

3398
	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
E
Eduard - Gabriel Munteanu 已提交
3399 3400

	return ret;
C
Christoph Lameter 已提交
3401 3402 3403 3404 3405 3406
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

size_t ksize(const void *object)
{
3407
	struct page *page;
C
Christoph Lameter 已提交
3408

3409
	if (unlikely(object == ZERO_SIZE_PTR))
3410 3411
		return 0;

3412 3413
	page = virt_to_head_page(object);

P
Pekka Enberg 已提交
3414 3415
	if (unlikely(!PageSlab(page))) {
		WARN_ON(!PageCompound(page));
3416
		return PAGE_SIZE << compound_order(page);
P
Pekka Enberg 已提交
3417
	}
C
Christoph Lameter 已提交
3418

3419
	return slab_ksize(page->slab);
C
Christoph Lameter 已提交
3420
}
K
Kirill A. Shutemov 已提交
3421
EXPORT_SYMBOL(ksize);
C
Christoph Lameter 已提交
3422

3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458
#ifdef CONFIG_SLUB_DEBUG
bool verify_mem_not_deleted(const void *x)
{
	struct page *page;
	void *object = (void *)x;
	unsigned long flags;
	bool rv;

	if (unlikely(ZERO_OR_NULL_PTR(x)))
		return false;

	local_irq_save(flags);

	page = virt_to_head_page(x);
	if (unlikely(!PageSlab(page))) {
		/* maybe it was from stack? */
		rv = true;
		goto out_unlock;
	}

	slab_lock(page);
	if (on_freelist(page->slab, page, object)) {
		object_err(page->slab, page, object, "Object is on free-list");
		rv = false;
	} else {
		rv = true;
	}
	slab_unlock(page);

out_unlock:
	local_irq_restore(flags);
	return rv;
}
EXPORT_SYMBOL(verify_mem_not_deleted);
#endif

C
Christoph Lameter 已提交
3459 3460 3461
void kfree(const void *x)
{
	struct page *page;
3462
	void *object = (void *)x;
C
Christoph Lameter 已提交
3463

3464 3465
	trace_kfree(_RET_IP_, x);

3466
	if (unlikely(ZERO_OR_NULL_PTR(x)))
C
Christoph Lameter 已提交
3467 3468
		return;

3469
	page = virt_to_head_page(x);
3470
	if (unlikely(!PageSlab(page))) {
3471
		BUG_ON(!PageCompound(page));
3472
		kmemleak_free(x);
3473 3474 3475
		put_page(page);
		return;
	}
3476
	slab_free(page->slab, page, object, _RET_IP_);
C
Christoph Lameter 已提交
3477 3478 3479
}
EXPORT_SYMBOL(kfree);

3480
/*
C
Christoph Lameter 已提交
3481 3482 3483 3484 3485 3486 3487 3488
 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
 * the remaining slabs by the number of items in use. The slabs with the
 * most items in use come first. New allocations will then fill those up
 * and thus they can be removed from the partial lists.
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
3489 3490 3491 3492 3493 3494 3495 3496
 */
int kmem_cache_shrink(struct kmem_cache *s)
{
	int node;
	int i;
	struct kmem_cache_node *n;
	struct page *page;
	struct page *t;
3497
	int objects = oo_objects(s->max);
3498
	struct list_head *slabs_by_inuse =
3499
		kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
3500 3501 3502 3503 3504 3505
	unsigned long flags;

	if (!slabs_by_inuse)
		return -ENOMEM;

	flush_all(s);
C
Christoph Lameter 已提交
3506
	for_each_node_state(node, N_NORMAL_MEMORY) {
3507 3508 3509 3510 3511
		n = get_node(s, node);

		if (!n->nr_partial)
			continue;

3512
		for (i = 0; i < objects; i++)
3513 3514 3515 3516 3517
			INIT_LIST_HEAD(slabs_by_inuse + i);

		spin_lock_irqsave(&n->list_lock, flags);

		/*
C
Christoph Lameter 已提交
3518
		 * Build lists indexed by the items in use in each slab.
3519
		 *
C
Christoph Lameter 已提交
3520 3521
		 * Note that concurrent frees may occur while we hold the
		 * list_lock. page->inuse here is the upper limit.
3522 3523
		 */
		list_for_each_entry_safe(page, t, &n->partial, lru) {
3524 3525 3526
			list_move(&page->lru, slabs_by_inuse + page->inuse);
			if (!page->inuse)
				n->nr_partial--;
3527 3528 3529
		}

		/*
C
Christoph Lameter 已提交
3530 3531
		 * Rebuild the partial list with the slabs filled up most
		 * first and the least used slabs at the end.
3532
		 */
3533
		for (i = objects - 1; i > 0; i--)
3534 3535 3536
			list_splice(slabs_by_inuse + i, n->partial.prev);

		spin_unlock_irqrestore(&n->list_lock, flags);
3537 3538 3539 3540

		/* Release empty slabs */
		list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
			discard_slab(s, page);
3541 3542 3543 3544 3545 3546 3547
	}

	kfree(slabs_by_inuse);
	return 0;
}
EXPORT_SYMBOL(kmem_cache_shrink);

P
Pekka Enberg 已提交
3548
#if defined(CONFIG_MEMORY_HOTPLUG)
3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list)
		kmem_cache_shrink(s);
	up_read(&slub_lock);

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int offline_node;

	offline_node = marg->status_change_nid;

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list) {
		n = get_node(s, offline_node);
		if (n) {
			/*
			 * if n->nr_slabs > 0, slabs still exist on the node
			 * that is going down. We were unable to free them,
3584
			 * and offline_pages() function shouldn't call this
3585 3586
			 * callback. So, we must fail.
			 */
3587
			BUG_ON(slabs_node(s, offline_node));
3588 3589

			s->node[offline_node] = NULL;
3590
			kmem_cache_free(kmem_cache_node, n);
3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611
		}
	}
	up_read(&slub_lock);
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int nid = marg->status_change_nid;
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
3612
	 * We are bringing a node online. No memory is available yet. We must
3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list) {
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
3623
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
3624 3625 3626 3627
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
3628
		init_kmem_cache_node(n, s);
3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655
		s->node[nid] = n;
	}
out:
	up_read(&slub_lock);
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
3656 3657 3658 3659
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
3660 3661 3662 3663 3664
	return ret;
}

#endif /* CONFIG_MEMORY_HOTPLUG */

C
Christoph Lameter 已提交
3665 3666 3667 3668
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688
/*
 * Used for early kmem_cache structures that were allocated using
 * the page allocator
 */

static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
{
	int node;

	list_add(&s->list, &slab_caches);
	s->refcount = -1;

	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n = get_node(s, node);
		struct page *p;

		if (n) {
			list_for_each_entry(p, &n->partial, lru)
				p->slab = s;

L
Li Zefan 已提交
3689
#ifdef CONFIG_SLUB_DEBUG
3690 3691 3692 3693 3694 3695 3696
			list_for_each_entry(p, &n->full, lru)
				p->slab = s;
#endif
		}
	}
}

C
Christoph Lameter 已提交
3697 3698 3699
void __init kmem_cache_init(void)
{
	int i;
3700
	int caches = 0;
3701 3702 3703 3704 3705
	struct kmem_cache *temp_kmem_cache;
	int order;
	struct kmem_cache *temp_kmem_cache_node;
	unsigned long kmalloc_size;

3706 3707 3708
	if (debug_guardpage_minorder())
		slub_max_order = 0;

3709 3710 3711 3712 3713 3714 3715 3716
	kmem_size = offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *);

	/* Allocate two kmem_caches from the page allocator */
	kmalloc_size = ALIGN(kmem_size, cache_line_size());
	order = get_order(2 * kmalloc_size);
	kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order);

C
Christoph Lameter 已提交
3717 3718
	/*
	 * Must first have the slab cache available for the allocations of the
C
Christoph Lameter 已提交
3719
	 * struct kmem_cache_node's. There is special bootstrap code in
C
Christoph Lameter 已提交
3720 3721
	 * kmem_cache_open for slab_state == DOWN.
	 */
3722 3723 3724 3725 3726
	kmem_cache_node = (void *)kmem_cache + kmalloc_size;

	kmem_cache_open(kmem_cache_node, "kmem_cache_node",
		sizeof(struct kmem_cache_node),
		0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
3727

3728
	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
C
Christoph Lameter 已提交
3729 3730 3731 3732

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

3733 3734 3735 3736 3737
	temp_kmem_cache = kmem_cache;
	kmem_cache_open(kmem_cache, "kmem_cache", kmem_size,
		0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
	kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
	memcpy(kmem_cache, temp_kmem_cache, kmem_size);
C
Christoph Lameter 已提交
3738

3739 3740 3741 3742 3743 3744
	/*
	 * Allocate kmem_cache_node properly from the kmem_cache slab.
	 * kmem_cache_node is separately allocated so no need to
	 * update any list pointers.
	 */
	temp_kmem_cache_node = kmem_cache_node;
C
Christoph Lameter 已提交
3745

3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757
	kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
	memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);

	kmem_cache_bootstrap_fixup(kmem_cache_node);

	caches++;
	kmem_cache_bootstrap_fixup(kmem_cache);
	caches++;
	/* Free temporary boot structure */
	free_pages((unsigned long)temp_kmem_cache, order);

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
3758 3759 3760 3761

	/*
	 * Patch up the size_index table if we have strange large alignment
	 * requirements for the kmalloc array. This is only the case for
C
Christoph Lameter 已提交
3762
	 * MIPS it seems. The standard arches will not generate any code here.
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772
	 *
	 * Largest permitted alignment is 256 bytes due to the way we
	 * handle the index determination for the smaller caches.
	 *
	 * Make sure that nothing crazy happens if someone starts tinkering
	 * around with ARCH_KMALLOC_MINALIGN
	 */
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
		(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

3773 3774 3775 3776 3777 3778
	for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
		int elem = size_index_elem(i);
		if (elem >= ARRAY_SIZE(size_index))
			break;
		size_index[elem] = KMALLOC_SHIFT_LOW;
	}
3779

3780 3781 3782 3783 3784 3785 3786 3787
	if (KMALLOC_MIN_SIZE == 64) {
		/*
		 * The 96 byte size cache is not used if the alignment
		 * is 64 byte.
		 */
		for (i = 64 + 8; i <= 96; i += 8)
			size_index[size_index_elem(i)] = 7;
	} else if (KMALLOC_MIN_SIZE == 128) {
3788 3789 3790 3791 3792 3793
		/*
		 * The 192 byte sized cache is not used if the alignment
		 * is 128 byte. Redirect kmalloc to use the 256 byte cache
		 * instead.
		 */
		for (i = 128 + 8; i <= 192; i += 8)
3794
			size_index[size_index_elem(i)] = 8;
3795 3796
	}

3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812
	/* Caches that are not of the two-to-the-power-of size */
	if (KMALLOC_MIN_SIZE <= 32) {
		kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
		caches++;
	}

	if (KMALLOC_MIN_SIZE <= 64) {
		kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
		caches++;
	}

	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
		kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
		caches++;
	}

C
Christoph Lameter 已提交
3813 3814 3815
	slab_state = UP;

	/* Provide the correct kmalloc names now that the caches are up */
P
Pekka Enberg 已提交
3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
	if (KMALLOC_MIN_SIZE <= 32) {
		kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
		BUG_ON(!kmalloc_caches[1]->name);
	}

	if (KMALLOC_MIN_SIZE <= 64) {
		kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
		BUG_ON(!kmalloc_caches[2]->name);
	}

3826 3827 3828 3829
	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
		char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);

		BUG_ON(!s);
3830
		kmalloc_caches[i]->name = s;
3831
	}
C
Christoph Lameter 已提交
3832 3833 3834

#ifdef CONFIG_SMP
	register_cpu_notifier(&slab_notifier);
3835
#endif
C
Christoph Lameter 已提交
3836

3837
#ifdef CONFIG_ZONE_DMA
3838 3839
	for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
		struct kmem_cache *s = kmalloc_caches[i];
3840

3841
		if (s && s->size) {
3842 3843 3844 3845
			char *name = kasprintf(GFP_NOWAIT,
				 "dma-kmalloc-%d", s->objsize);

			BUG_ON(!name);
3846 3847
			kmalloc_dma_caches[i] = create_kmalloc_cache(name,
				s->objsize, SLAB_CACHE_DMA);
3848 3849 3850
		}
	}
#endif
I
Ingo Molnar 已提交
3851 3852
	printk(KERN_INFO
		"SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
3853 3854
		" CPUs=%d, Nodes=%d\n",
		caches, cache_line_size(),
C
Christoph Lameter 已提交
3855 3856 3857 3858
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

3859 3860 3861 3862
void __init kmem_cache_init_late(void)
{
}

C
Christoph Lameter 已提交
3863 3864 3865 3866 3867 3868 3869 3870
/*
 * Find a mergeable slab cache
 */
static int slab_unmergeable(struct kmem_cache *s)
{
	if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
		return 1;

3871
	if (s->ctor)
C
Christoph Lameter 已提交
3872 3873
		return 1;

3874 3875 3876 3877 3878 3879
	/*
	 * We may have set a slab to be unmergeable during bootstrap.
	 */
	if (s->refcount < 0)
		return 1;

C
Christoph Lameter 已提交
3880 3881 3882 3883
	return 0;
}

static struct kmem_cache *find_mergeable(size_t size,
3884
		size_t align, unsigned long flags, const char *name,
3885
		void (*ctor)(void *))
C
Christoph Lameter 已提交
3886
{
3887
	struct kmem_cache *s;
C
Christoph Lameter 已提交
3888 3889 3890 3891

	if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
		return NULL;

3892
	if (ctor)
C
Christoph Lameter 已提交
3893 3894 3895 3896 3897
		return NULL;

	size = ALIGN(size, sizeof(void *));
	align = calculate_alignment(flags, align, size);
	size = ALIGN(size, align);
3898
	flags = kmem_cache_flags(size, flags, name, NULL);
C
Christoph Lameter 已提交
3899

3900
	list_for_each_entry(s, &slab_caches, list) {
C
Christoph Lameter 已提交
3901 3902 3903 3904 3905 3906
		if (slab_unmergeable(s))
			continue;

		if (size > s->size)
			continue;

3907
		if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
C
Christoph Lameter 已提交
3908 3909 3910 3911 3912
				continue;
		/*
		 * Check if alignment is compatible.
		 * Courtesy of Adrian Drzewiecki
		 */
P
Pekka Enberg 已提交
3913
		if ((s->size & ~(align - 1)) != s->size)
C
Christoph Lameter 已提交
3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924
			continue;

		if (s->size - size >= sizeof(void *))
			continue;

		return s;
	}
	return NULL;
}

struct kmem_cache *kmem_cache_create(const char *name, size_t size,
3925
		size_t align, unsigned long flags, void (*ctor)(void *))
C
Christoph Lameter 已提交
3926 3927
{
	struct kmem_cache *s;
P
Pekka Enberg 已提交
3928
	char *n;
C
Christoph Lameter 已提交
3929

3930 3931 3932
	if (WARN_ON(!name))
		return NULL;

C
Christoph Lameter 已提交
3933
	down_write(&slub_lock);
3934
	s = find_mergeable(size, align, flags, name, ctor);
C
Christoph Lameter 已提交
3935 3936 3937 3938 3939 3940 3941 3942
	if (s) {
		s->refcount++;
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
		s->objsize = max(s->objsize, (int)size);
		s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
C
Christoph Lameter 已提交
3943

3944 3945
		if (sysfs_slab_alias(s, name)) {
			s->refcount--;
C
Christoph Lameter 已提交
3946
			goto err;
3947
		}
3948
		up_write(&slub_lock);
3949 3950
		return s;
	}
C
Christoph Lameter 已提交
3951

P
Pekka Enberg 已提交
3952 3953 3954 3955
	n = kstrdup(name, GFP_KERNEL);
	if (!n)
		goto err;

3956 3957
	s = kmalloc(kmem_size, GFP_KERNEL);
	if (s) {
P
Pekka Enberg 已提交
3958
		if (kmem_cache_open(s, n,
3959
				size, align, flags, ctor)) {
C
Christoph Lameter 已提交
3960
			list_add(&s->list, &slab_caches);
3961
			up_write(&slub_lock);
3962
			if (sysfs_slab_add(s)) {
3963
				down_write(&slub_lock);
3964
				list_del(&s->list);
P
Pekka Enberg 已提交
3965
				kfree(n);
3966
				kfree(s);
3967
				goto err;
3968
			}
3969 3970
			return s;
		}
P
Pekka Enberg 已提交
3971
		kfree(n);
3972
		kfree(s);
C
Christoph Lameter 已提交
3973
	}
3974
err:
C
Christoph Lameter 已提交
3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986
	up_write(&slub_lock);

	if (flags & SLAB_PANIC)
		panic("Cannot create slabcache %s\n", name);
	else
		s = NULL;
	return s;
}
EXPORT_SYMBOL(kmem_cache_create);

#ifdef CONFIG_SMP
/*
C
Christoph Lameter 已提交
3987 3988
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
C
Christoph Lameter 已提交
3989 3990 3991 3992 3993
 */
static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
		unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
3994 3995
	struct kmem_cache *s;
	unsigned long flags;
C
Christoph Lameter 已提交
3996 3997 3998

	switch (action) {
	case CPU_UP_CANCELED:
3999
	case CPU_UP_CANCELED_FROZEN:
C
Christoph Lameter 已提交
4000
	case CPU_DEAD:
4001
	case CPU_DEAD_FROZEN:
4002 4003 4004 4005 4006 4007 4008
		down_read(&slub_lock);
		list_for_each_entry(s, &slab_caches, list) {
			local_irq_save(flags);
			__flush_cpu_slab(s, cpu);
			local_irq_restore(flags);
		}
		up_read(&slub_lock);
C
Christoph Lameter 已提交
4009 4010 4011 4012 4013 4014 4015
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

P
Pekka Enberg 已提交
4016
static struct notifier_block __cpuinitdata slab_notifier = {
I
Ingo Molnar 已提交
4017
	.notifier_call = slab_cpuup_callback
P
Pekka Enberg 已提交
4018
};
C
Christoph Lameter 已提交
4019 4020 4021

#endif

4022
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
C
Christoph Lameter 已提交
4023
{
4024
	struct kmem_cache *s;
4025
	void *ret;
4026

4027
	if (unlikely(size > SLUB_MAX_SIZE))
4028 4029
		return kmalloc_large(size, gfpflags);

4030
	s = get_slab(size, gfpflags);
C
Christoph Lameter 已提交
4031

4032
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4033
		return s;
C
Christoph Lameter 已提交
4034

4035
	ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller);
4036

L
Lucas De Marchi 已提交
4037
	/* Honor the call site pointer we received. */
4038
	trace_kmalloc(caller, ret, size, s->size, gfpflags);
4039 4040

	return ret;
C
Christoph Lameter 已提交
4041 4042
}

4043
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4044
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4045
					int node, unsigned long caller)
C
Christoph Lameter 已提交
4046
{
4047
	struct kmem_cache *s;
4048
	void *ret;
4049

4050 4051 4052 4053 4054 4055 4056 4057 4058
	if (unlikely(size > SLUB_MAX_SIZE)) {
		ret = kmalloc_large_node(size, gfpflags, node);

		trace_kmalloc_node(caller, ret,
				   size, PAGE_SIZE << get_order(size),
				   gfpflags, node);

		return ret;
	}
4059

4060
	s = get_slab(size, gfpflags);
C
Christoph Lameter 已提交
4061

4062
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4063
		return s;
C
Christoph Lameter 已提交
4064

4065 4066
	ret = slab_alloc(s, gfpflags, node, caller);

L
Lucas De Marchi 已提交
4067
	/* Honor the call site pointer we received. */
4068
	trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4069 4070

	return ret;
C
Christoph Lameter 已提交
4071
}
4072
#endif
C
Christoph Lameter 已提交
4073

4074
#ifdef CONFIG_SYSFS
4075 4076 4077 4078 4079 4080 4081 4082 4083
static int count_inuse(struct page *page)
{
	return page->inuse;
}

static int count_total(struct page *page)
{
	return page->objects;
}
4084
#endif
4085

4086
#ifdef CONFIG_SLUB_DEBUG
4087 4088
static int validate_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4089 4090
{
	void *p;
4091
	void *addr = page_address(page);
4092 4093 4094 4095 4096 4097

	if (!check_slab(s, page) ||
			!on_freelist(s, page, NULL))
		return 0;

	/* Now we know that a valid freelist exists */
4098
	bitmap_zero(map, page->objects);
4099

4100 4101 4102 4103 4104
	get_map(s, page, map);
	for_each_object(p, s, addr, page->objects) {
		if (test_bit(slab_index(p, s, addr), map))
			if (!check_object(s, page, p, SLUB_RED_INACTIVE))
				return 0;
4105 4106
	}

4107
	for_each_object(p, s, addr, page->objects)
4108
		if (!test_bit(slab_index(p, s, addr), map))
4109
			if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4110 4111 4112 4113
				return 0;
	return 1;
}

4114 4115
static void validate_slab_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4116
{
4117 4118 4119
	slab_lock(page);
	validate_slab(s, page, map);
	slab_unlock(page);
4120 4121
}

4122 4123
static int validate_slab_node(struct kmem_cache *s,
		struct kmem_cache_node *n, unsigned long *map)
4124 4125 4126 4127 4128 4129 4130 4131
{
	unsigned long count = 0;
	struct page *page;
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

	list_for_each_entry(page, &n->partial, lru) {
4132
		validate_slab_slab(s, page, map);
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142
		count++;
	}
	if (count != n->nr_partial)
		printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
			"counter=%ld\n", s->name, count, n->nr_partial);

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

	list_for_each_entry(page, &n->full, lru) {
4143
		validate_slab_slab(s, page, map);
4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155
		count++;
	}
	if (count != atomic_long_read(&n->nr_slabs))
		printk(KERN_ERR "SLUB: %s %ld slabs counted but "
			"counter=%ld\n", s->name, count,
			atomic_long_read(&n->nr_slabs));

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

4156
static long validate_slab_cache(struct kmem_cache *s)
4157 4158 4159
{
	int node;
	unsigned long count = 0;
4160
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4161 4162 4163 4164
				sizeof(unsigned long), GFP_KERNEL);

	if (!map)
		return -ENOMEM;
4165 4166

	flush_all(s);
C
Christoph Lameter 已提交
4167
	for_each_node_state(node, N_NORMAL_MEMORY) {
4168 4169
		struct kmem_cache_node *n = get_node(s, node);

4170
		count += validate_slab_node(s, n, map);
4171
	}
4172
	kfree(map);
4173 4174
	return count;
}
4175
/*
C
Christoph Lameter 已提交
4176
 * Generate lists of code addresses where slabcache objects are allocated
4177 4178 4179 4180 4181
 * and freed.
 */

struct location {
	unsigned long count;
4182
	unsigned long addr;
4183 4184 4185 4186 4187
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
R
Rusty Russell 已提交
4188
	DECLARE_BITMAP(cpus, NR_CPUS);
4189
	nodemask_t nodes;
4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
};

static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

4205
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4206 4207 4208 4209 4210 4211
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

4212
	l = (void *)__get_free_pages(flags, order);
4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
4226
				const struct track *track)
4227 4228 4229
{
	long start, end, pos;
	struct location *l;
4230
	unsigned long caddr;
4231
	unsigned long age = jiffies - track->when;
4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246

	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

		caddr = t->loc[pos].addr;
4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262
		if (track->addr == caddr) {

			l = &t->loc[pos];
			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

R
Rusty Russell 已提交
4263 4264
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
4265 4266
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
4267 4268 4269
			return 1;
		}

4270
		if (track->addr < caddr)
4271 4272 4273 4274 4275 4276
			end = pos;
		else
			start = pos;
	}

	/*
C
Christoph Lameter 已提交
4277
	 * Not found. Insert new tracking element.
4278
	 */
4279
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4280 4281 4282 4283 4284 4285 4286 4287
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
4288 4289 4290 4291 4292 4293
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
R
Rusty Russell 已提交
4294 4295
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4296 4297
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
4298 4299 4300 4301
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
E
Eric Dumazet 已提交
4302
		struct page *page, enum track_item alloc,
N
Namhyung Kim 已提交
4303
		unsigned long *map)
4304
{
4305
	void *addr = page_address(page);
4306 4307
	void *p;

4308
	bitmap_zero(map, page->objects);
4309
	get_map(s, page, map);
4310

4311
	for_each_object(p, s, addr, page->objects)
4312 4313
		if (!test_bit(slab_index(p, s, addr), map))
			add_location(t, s, get_track(s, p, alloc));
4314 4315 4316 4317 4318
}

static int list_locations(struct kmem_cache *s, char *buf,
					enum track_item alloc)
{
4319
	int len = 0;
4320
	unsigned long i;
4321
	struct loc_track t = { 0, 0, NULL };
4322
	int node;
E
Eric Dumazet 已提交
4323 4324
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
				     sizeof(unsigned long), GFP_KERNEL);
4325

E
Eric Dumazet 已提交
4326 4327 4328
	if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
				     GFP_TEMPORARY)) {
		kfree(map);
4329
		return sprintf(buf, "Out of memory\n");
E
Eric Dumazet 已提交
4330
	}
4331 4332 4333
	/* Push back cpu slabs */
	flush_all(s);

C
Christoph Lameter 已提交
4334
	for_each_node_state(node, N_NORMAL_MEMORY) {
4335 4336 4337 4338
		struct kmem_cache_node *n = get_node(s, node);
		unsigned long flags;
		struct page *page;

4339
		if (!atomic_long_read(&n->nr_slabs))
4340 4341 4342 4343
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry(page, &n->partial, lru)
E
Eric Dumazet 已提交
4344
			process_slab(&t, s, page, alloc, map);
4345
		list_for_each_entry(page, &n->full, lru)
E
Eric Dumazet 已提交
4346
			process_slab(&t, s, page, alloc, map);
4347 4348 4349 4350
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	for (i = 0; i < t.count; i++) {
4351
		struct location *l = &t.loc[i];
4352

H
Hugh Dickins 已提交
4353
		if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4354
			break;
4355
		len += sprintf(buf + len, "%7ld ", l->count);
4356 4357

		if (l->addr)
J
Joe Perches 已提交
4358
			len += sprintf(buf + len, "%pS", (void *)l->addr);
4359
		else
4360
			len += sprintf(buf + len, "<not-available>");
4361 4362

		if (l->sum_time != l->min_time) {
4363
			len += sprintf(buf + len, " age=%ld/%ld/%ld",
R
Roman Zippel 已提交
4364 4365 4366
				l->min_time,
				(long)div_u64(l->sum_time, l->count),
				l->max_time);
4367
		} else
4368
			len += sprintf(buf + len, " age=%ld",
4369 4370 4371
				l->min_time);

		if (l->min_pid != l->max_pid)
4372
			len += sprintf(buf + len, " pid=%ld-%ld",
4373 4374
				l->min_pid, l->max_pid);
		else
4375
			len += sprintf(buf + len, " pid=%ld",
4376 4377
				l->min_pid);

R
Rusty Russell 已提交
4378 4379
		if (num_online_cpus() > 1 &&
				!cpumask_empty(to_cpumask(l->cpus)) &&
4380 4381 4382
				len < PAGE_SIZE - 60) {
			len += sprintf(buf + len, " cpus=");
			len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
R
Rusty Russell 已提交
4383
						 to_cpumask(l->cpus));
4384 4385
		}

4386
		if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4387 4388 4389
				len < PAGE_SIZE - 60) {
			len += sprintf(buf + len, " nodes=");
			len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
4390 4391 4392
					l->nodes);
		}

4393
		len += sprintf(buf + len, "\n");
4394 4395 4396
	}

	free_loc_track(&t);
E
Eric Dumazet 已提交
4397
	kfree(map);
4398
	if (!t.count)
4399 4400
		len += sprintf(buf, "No data\n");
	return len;
4401
}
4402
#endif
4403

4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465
#ifdef SLUB_RESILIENCY_TEST
static void resiliency_test(void)
{
	u8 *p;

	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || SLUB_PAGE_SHIFT < 10);

	printk(KERN_ERR "SLUB resiliency testing\n");
	printk(KERN_ERR "-----------------------\n");
	printk(KERN_ERR "A. Corruption after allocation\n");

	p = kzalloc(16, GFP_KERNEL);
	p[16] = 0x12;
	printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
			" 0x12->0x%p\n\n", p + 16);

	validate_slab_cache(kmalloc_caches[4]);

	/* Hmmm... The next two are dangerous */
	p = kzalloc(32, GFP_KERNEL);
	p[32 + sizeof(void *)] = 0x34;
	printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
			" 0x34 -> -0x%p\n", p);
	printk(KERN_ERR
		"If allocated object is overwritten then not detectable\n\n");

	validate_slab_cache(kmalloc_caches[5]);
	p = kzalloc(64, GFP_KERNEL);
	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
	*p = 0x56;
	printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
									p);
	printk(KERN_ERR
		"If allocated object is overwritten then not detectable\n\n");
	validate_slab_cache(kmalloc_caches[6]);

	printk(KERN_ERR "\nB. Corruption after free\n");
	p = kzalloc(128, GFP_KERNEL);
	kfree(p);
	*p = 0x78;
	printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[7]);

	p = kzalloc(256, GFP_KERNEL);
	kfree(p);
	p[50] = 0x9a;
	printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
			p);
	validate_slab_cache(kmalloc_caches[8]);

	p = kzalloc(512, GFP_KERNEL);
	kfree(p);
	p[512] = 0xab;
	printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[9]);
}
#else
#ifdef CONFIG_SYSFS
static void resiliency_test(void) {};
#endif
#endif

4466
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
4467
enum slab_stat_type {
4468 4469 4470 4471 4472
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
C
Christoph Lameter 已提交
4473 4474
};

4475
#define SO_ALL		(1 << SL_ALL)
C
Christoph Lameter 已提交
4476 4477 4478
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
4479
#define SO_TOTAL	(1 << SL_TOTAL)
C
Christoph Lameter 已提交
4480

4481 4482
static ssize_t show_slab_objects(struct kmem_cache *s,
			    char *buf, unsigned long flags)
C
Christoph Lameter 已提交
4483 4484 4485 4486 4487 4488 4489 4490
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;
	unsigned long *per_cpu;

	nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4491 4492
	if (!nodes)
		return -ENOMEM;
C
Christoph Lameter 已提交
4493 4494
	per_cpu = nodes + nr_node_ids;

4495 4496
	if (flags & SO_CPU) {
		int cpu;
C
Christoph Lameter 已提交
4497

4498
		for_each_possible_cpu(cpu) {
4499
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4500
			int node = ACCESS_ONCE(c->node);
4501
			struct page *page;
4502

4503
			if (node < 0)
4504
				continue;
4505 4506 4507 4508
			page = ACCESS_ONCE(c->page);
			if (page) {
				if (flags & SO_TOTAL)
					x = page->objects;
4509
				else if (flags & SO_OBJECTS)
4510
					x = page->inuse;
C
Christoph Lameter 已提交
4511 4512
				else
					x = 1;
4513

C
Christoph Lameter 已提交
4514
				total += x;
4515
				nodes[node] += x;
C
Christoph Lameter 已提交
4516
			}
4517 4518 4519 4520
			page = c->partial;

			if (page) {
				x = page->pobjects;
4521 4522
				total += x;
				nodes[node] += x;
4523
			}
4524
			per_cpu[node]++;
C
Christoph Lameter 已提交
4525 4526 4527
		}
	}

4528
	lock_memory_hotplug();
4529
#ifdef CONFIG_SLUB_DEBUG
4530 4531 4532 4533 4534 4535 4536 4537 4538
	if (flags & SO_ALL) {
		for_each_node_state(node, N_NORMAL_MEMORY) {
			struct kmem_cache_node *n = get_node(s, node);

		if (flags & SO_TOTAL)
			x = atomic_long_read(&n->total_objects);
		else if (flags & SO_OBJECTS)
			x = atomic_long_read(&n->total_objects) -
				count_partial(n, count_free);
C
Christoph Lameter 已提交
4539 4540

			else
4541
				x = atomic_long_read(&n->nr_slabs);
C
Christoph Lameter 已提交
4542 4543 4544 4545
			total += x;
			nodes[node] += x;
		}

4546 4547 4548
	} else
#endif
	if (flags & SO_PARTIAL) {
4549 4550
		for_each_node_state(node, N_NORMAL_MEMORY) {
			struct kmem_cache_node *n = get_node(s, node);
C
Christoph Lameter 已提交
4551

4552 4553 4554 4555
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
C
Christoph Lameter 已提交
4556
			else
4557
				x = n->nr_partial;
C
Christoph Lameter 已提交
4558 4559 4560 4561 4562 4563
			total += x;
			nodes[node] += x;
		}
	}
	x = sprintf(buf, "%lu", total);
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4564
	for_each_node_state(node, N_NORMAL_MEMORY)
C
Christoph Lameter 已提交
4565 4566 4567 4568
		if (nodes[node])
			x += sprintf(buf + x, " N%d=%lu",
					node, nodes[node]);
#endif
4569
	unlock_memory_hotplug();
C
Christoph Lameter 已提交
4570 4571 4572 4573
	kfree(nodes);
	return x + sprintf(buf + x, "\n");
}

4574
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4575 4576 4577 4578
static int any_slab_objects(struct kmem_cache *s)
{
	int node;

4579
	for_each_online_node(node) {
C
Christoph Lameter 已提交
4580 4581
		struct kmem_cache_node *n = get_node(s, node);

4582 4583 4584
		if (!n)
			continue;

4585
		if (atomic_long_read(&n->total_objects))
C
Christoph Lameter 已提交
4586 4587 4588 4589
			return 1;
	}
	return 0;
}
4590
#endif
C
Christoph Lameter 已提交
4591 4592

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4593
#define to_slab(n) container_of(n, struct kmem_cache, kobj)
C
Christoph Lameter 已提交
4594 4595 4596 4597 4598 4599 4600 4601

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
4602 4603
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)
C
Christoph Lameter 已提交
4604 4605 4606

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
4607
	__ATTR(_name, 0600, _name##_show, _name##_store)
C
Christoph Lameter 已提交
4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->size);
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->align);
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->objsize);
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
4629
	return sprintf(buf, "%d\n", oo_objects(s->oo));
C
Christoph Lameter 已提交
4630 4631 4632
}
SLAB_ATTR_RO(objs_per_slab);

4633 4634 4635
static ssize_t order_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
4636 4637 4638 4639 4640 4641
	unsigned long order;
	int err;

	err = strict_strtoul(buf, 10, &order);
	if (err)
		return err;
4642 4643 4644 4645 4646 4647 4648 4649

	if (order > slub_max_order || order < slub_min_order)
		return -EINVAL;

	calculate_sizes(s, order);
	return length;
}

C
Christoph Lameter 已提交
4650 4651
static ssize_t order_show(struct kmem_cache *s, char *buf)
{
4652
	return sprintf(buf, "%d\n", oo_order(s->oo));
C
Christoph Lameter 已提交
4653
}
4654
SLAB_ATTR(order);
C
Christoph Lameter 已提交
4655

4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%lu\n", s->min_partial);
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

	err = strict_strtoul(buf, 10, &min);
	if (err)
		return err;

4671
	set_min_partial(s, min);
4672 4673 4674 4675
	return length;
}
SLAB_ATTR(min_partial);

4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->cpu_partial);
}

static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long objects;
	int err;

	err = strict_strtoul(buf, 10, &objects);
	if (err)
		return err;
4690 4691
	if (objects && kmem_cache_debug(s))
		return -EINVAL;
4692 4693 4694 4695 4696 4697 4698

	s->cpu_partial = objects;
	flush_all(s);
	return length;
}
SLAB_ATTR(cpu_partial);

C
Christoph Lameter 已提交
4699 4700
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
J
Joe Perches 已提交
4701 4702 4703
	if (!s->ctor)
		return 0;
	return sprintf(buf, "%pS\n", s->ctor);
C
Christoph Lameter 已提交
4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->refcount - 1);
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
4715
	return show_slab_objects(s, buf, SO_PARTIAL);
C
Christoph Lameter 已提交
4716 4717 4718 4719 4720
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
4721
	return show_slab_objects(s, buf, SO_CPU);
C
Christoph Lameter 已提交
4722 4723 4724 4725 4726
}
SLAB_ATTR_RO(cpu_slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
4727
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
C
Christoph Lameter 已提交
4728 4729 4730
}
SLAB_ATTR_RO(objects);

4731 4732 4733 4734 4735 4736
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
	int pages = 0;
	int cpu;
	int len;

	for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;

		if (page) {
			pages += page->pages;
			objects += page->pobjects;
		}
	}

	len = sprintf(buf, "%d(%d)", objects, pages);

#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;

		if (page && len < PAGE_SIZE - 20)
			len += sprintf(buf + len, " C%d=%d(%d)", cpu,
				page->pobjects, page->pages);
	}
#endif
	return len + sprintf(buf + len, "\n");
}
SLAB_ATTR_RO(slabs_cpu_partial);

4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}

static ssize_t reclaim_account_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
	if (buf[0] == '1')
		s->flags |= SLAB_RECLAIM_ACCOUNT;
	return length;
}
SLAB_ATTR(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
}
SLAB_ATTR_RO(destroy_by_rcu);

4803 4804 4805 4806 4807 4808
static ssize_t reserved_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->reserved);
}
SLAB_ATTR_RO(reserved);

4809
#ifdef CONFIG_SLUB_DEBUG
4810 4811 4812 4813 4814 4815
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

4816 4817 4818 4819 4820 4821
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

C
Christoph Lameter 已提交
4822 4823 4824 4825 4826 4827 4828 4829 4830
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
}

static ssize_t sanity_checks_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_DEBUG_FREE;
4831 4832
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4833
		s->flags |= SLAB_DEBUG_FREE;
4834
	}
C
Christoph Lameter 已提交
4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847
	return length;
}
SLAB_ATTR(sanity_checks);

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
}

static ssize_t trace_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
	s->flags &= ~SLAB_TRACE;
4848 4849
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4850
		s->flags |= SLAB_TRACE;
4851
	}
C
Christoph Lameter 已提交
4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867
	return length;
}
SLAB_ATTR(trace);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}

static ssize_t red_zone_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_RED_ZONE;
4868 4869
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4870
		s->flags |= SLAB_RED_ZONE;
4871
	}
4872
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888
	return length;
}
SLAB_ATTR(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}

static ssize_t poison_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_POISON;
4889 4890
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4891
		s->flags |= SLAB_POISON;
4892
	}
4893
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909
	return length;
}
SLAB_ATTR(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}

static ssize_t store_user_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_STORE_USER;
4910 4911
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4912
		s->flags |= SLAB_STORE_USER;
4913
	}
4914
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4915 4916 4917 4918
	return length;
}
SLAB_ATTR(store_user);

4919 4920 4921 4922 4923 4924 4925 4926
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
4927 4928 4929 4930 4931 4932 4933 4934
	int ret = -EINVAL;

	if (buf[0] == '1') {
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
4935 4936
}
SLAB_ATTR(validate);
4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969

static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_ALLOC);
}
SLAB_ATTR_RO(alloc_calls);

static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_FREE);
}
SLAB_ATTR_RO(free_calls);
#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
}

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
	s->flags &= ~SLAB_FAILSLAB;
	if (buf[0] == '1')
		s->flags |= SLAB_FAILSLAB;
	return length;
}
SLAB_ATTR(failslab);
4970
#endif
4971

4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
	if (buf[0] == '1') {
		int rc = kmem_cache_shrink(s);

		if (rc)
			return rc;
	} else
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

C
Christoph Lameter 已提交
4991
#ifdef CONFIG_NUMA
4992
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
C
Christoph Lameter 已提交
4993
{
4994
	return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
C
Christoph Lameter 已提交
4995 4996
}

4997
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
C
Christoph Lameter 已提交
4998 4999
				const char *buf, size_t length)
{
5000 5001 5002 5003 5004 5005 5006
	unsigned long ratio;
	int err;

	err = strict_strtoul(buf, 10, &ratio);
	if (err)
		return err;

5007
	if (ratio <= 100)
5008
		s->remote_node_defrag_ratio = ratio * 10;
C
Christoph Lameter 已提交
5009 5010 5011

	return length;
}
5012
SLAB_ATTR(remote_node_defrag_ratio);
C
Christoph Lameter 已提交
5013 5014
#endif

5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
	int len;
	int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
5027
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5028 5029 5030 5031 5032 5033 5034

		data[cpu] = x;
		sum += x;
	}

	len = sprintf(buf, "%lu", sum);

5035
#ifdef CONFIG_SMP
5036 5037
	for_each_online_cpu(cpu) {
		if (data[cpu] && len < PAGE_SIZE - 20)
5038
			len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5039
	}
5040
#endif
5041 5042 5043 5044
	kfree(data);
	return len + sprintf(buf + len, "\n");
}

D
David Rientjes 已提交
5045 5046 5047 5048 5049
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
5050
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
D
David Rientjes 已提交
5051 5052
}

5053 5054 5055 5056 5057
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
D
David Rientjes 已提交
5058 5059 5060 5061 5062 5063 5064 5065 5066
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
5078
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5079 5080 5081 5082 5083 5084 5085
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5086
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5087
STAT_ATTR(ORDER_FALLBACK, order_fallback);
5088 5089
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5090 5091
STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5092 5093
STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5094 5095
#endif

P
Pekka Enberg 已提交
5096
static struct attribute *slab_attrs[] = {
C
Christoph Lameter 已提交
5097 5098 5099 5100
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
5101
	&min_partial_attr.attr,
5102
	&cpu_partial_attr.attr,
C
Christoph Lameter 已提交
5103
	&objects_attr.attr,
5104
	&objects_partial_attr.attr,
C
Christoph Lameter 已提交
5105 5106 5107 5108 5109 5110 5111 5112
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
5113
	&shrink_attr.attr,
5114
	&reserved_attr.attr,
5115
	&slabs_cpu_partial_attr.attr,
5116
#ifdef CONFIG_SLUB_DEBUG
5117 5118 5119 5120
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
C
Christoph Lameter 已提交
5121 5122 5123
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
5124
	&validate_attr.attr,
5125 5126
	&alloc_calls_attr.attr,
	&free_calls_attr.attr,
5127
#endif
C
Christoph Lameter 已提交
5128 5129 5130 5131
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
5132
	&remote_node_defrag_ratio_attr.attr,
5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
5145
	&alloc_node_mismatch_attr.attr,
5146 5147 5148 5149 5150 5151 5152
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
5153
	&deactivate_bypass_attr.attr,
5154
	&order_fallback_attr.attr,
5155 5156
	&cmpxchg_double_fail_attr.attr,
	&cmpxchg_double_cpu_fail_attr.attr,
5157 5158
	&cpu_partial_alloc_attr.attr,
	&cpu_partial_free_attr.attr,
5159 5160
	&cpu_partial_node_attr.attr,
	&cpu_partial_drain_attr.attr,
C
Christoph Lameter 已提交
5161
#endif
5162 5163 5164 5165
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif

C
Christoph Lameter 已提交
5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210
	NULL
};

static struct attribute_group slab_attr_group = {
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);

	return err;
}

C
Christoph Lameter 已提交
5211 5212 5213 5214
static void kmem_cache_release(struct kobject *kobj)
{
	struct kmem_cache *s = to_slab(kobj);

P
Pekka Enberg 已提交
5215
	kfree(s->name);
C
Christoph Lameter 已提交
5216 5217 5218
	kfree(s);
}

5219
static const struct sysfs_ops slab_sysfs_ops = {
C
Christoph Lameter 已提交
5220 5221 5222 5223 5224 5225
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
C
Christoph Lameter 已提交
5226
	.release = kmem_cache_release
C
Christoph Lameter 已提交
5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;
}

5238
static const struct kset_uevent_ops slab_uevent_ops = {
C
Christoph Lameter 已提交
5239 5240 5241
	.filter = uevent_filter,
};

5242
static struct kset *slab_kset;
C
Christoph Lameter 已提交
5243 5244 5245 5246

#define ID_STR_LENGTH 64

/* Create a unique string id for a slab cache:
C
Christoph Lameter 已提交
5247 5248
 *
 * Format	:[flags-]size
C
Christoph Lameter 已提交
5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

	BUG_ON(!name);

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
	if (s->flags & SLAB_DEBUG_FREE)
		*p++ = 'F';
V
Vegard Nossum 已提交
5271 5272
	if (!(s->flags & SLAB_NOTRACK))
		*p++ = 't';
C
Christoph Lameter 已提交
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296
	if (p != name + 1)
		*p++ = '-';
	p += sprintf(p, "%07d", s->size);
	BUG_ON(p > name + ID_STR_LENGTH - 1);
	return name;
}

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
	int unmergeable;

	if (slab_state < SYSFS)
		/* Defer until later */
		return 0;

	unmergeable = slab_unmergeable(s);
	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
5297
		sysfs_remove_link(&slab_kset->kobj, s->name);
C
Christoph Lameter 已提交
5298 5299 5300 5301 5302 5303 5304 5305 5306
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
	}

5307
	s->kobj.kset = slab_kset;
5308 5309 5310
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
	if (err) {
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5311
		return err;
5312
	}
C
Christoph Lameter 已提交
5313 5314

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
5315 5316 5317
	if (err) {
		kobject_del(&s->kobj);
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5318
		return err;
5319
	}
C
Christoph Lameter 已提交
5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330
	kobject_uevent(&s->kobj, KOBJ_ADD);
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
		kfree(name);
	}
	return 0;
}

static void sysfs_slab_remove(struct kmem_cache *s)
{
5331 5332 5333 5334 5335 5336 5337
	if (slab_state < SYSFS)
		/*
		 * Sysfs has not been setup yet so no need to remove the
		 * cache from sysfs.
		 */
		return;

C
Christoph Lameter 已提交
5338 5339
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
	kobject_del(&s->kobj);
C
Christoph Lameter 已提交
5340
	kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5341 5342 5343 5344
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
N
Nick Andrew 已提交
5345
 * available lest we lose that information.
C
Christoph Lameter 已提交
5346 5347 5348 5349 5350 5351 5352
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

A
Adrian Bunk 已提交
5353
static struct saved_alias *alias_list;
C
Christoph Lameter 已提交
5354 5355 5356 5357 5358 5359 5360 5361 5362

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

	if (slab_state == SYSFS) {
		/*
		 * If we have a leftover link then remove it.
		 */
5363 5364
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
C
Christoph Lameter 已提交
5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
	return 0;
}

static int __init slab_sysfs_init(void)
{
5380
	struct kmem_cache *s;
C
Christoph Lameter 已提交
5381 5382
	int err;

5383 5384
	down_write(&slub_lock);

5385
	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5386
	if (!slab_kset) {
5387
		up_write(&slub_lock);
C
Christoph Lameter 已提交
5388 5389 5390 5391
		printk(KERN_ERR "Cannot register slab subsystem.\n");
		return -ENOSYS;
	}

5392 5393
	slab_state = SYSFS;

5394
	list_for_each_entry(s, &slab_caches, list) {
5395
		err = sysfs_slab_add(s);
5396 5397 5398
		if (err)
			printk(KERN_ERR "SLUB: Unable to add boot slab %s"
						" to sysfs\n", s->name);
5399
	}
C
Christoph Lameter 已提交
5400 5401 5402 5403 5404 5405

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
5406 5407 5408
		if (err)
			printk(KERN_ERR "SLUB: Unable to add boot slab alias"
					" %s to sysfs\n", s->name);
C
Christoph Lameter 已提交
5409 5410 5411
		kfree(al);
	}

5412
	up_write(&slub_lock);
C
Christoph Lameter 已提交
5413 5414 5415 5416 5417
	resiliency_test();
	return 0;
}

__initcall(slab_sysfs_init);
5418
#endif /* CONFIG_SYSFS */
P
Pekka J Enberg 已提交
5419 5420 5421 5422

/*
 * The /proc/slabinfo ABI
 */
5423
#ifdef CONFIG_SLABINFO
P
Pekka J Enberg 已提交
5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459
static void print_slabinfo_header(struct seq_file *m)
{
	seq_puts(m, "slabinfo - version: 2.1\n");
	seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
		 "<objperslab> <pagesperslab>");
	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
	seq_putc(m, '\n');
}

static void *s_start(struct seq_file *m, loff_t *pos)
{
	loff_t n = *pos;

	down_read(&slub_lock);
	if (!n)
		print_slabinfo_header(m);

	return seq_list_start(&slab_caches, *pos);
}

static void *s_next(struct seq_file *m, void *p, loff_t *pos)
{
	return seq_list_next(p, &slab_caches, pos);
}

static void s_stop(struct seq_file *m, void *p)
{
	up_read(&slub_lock);
}

static int s_show(struct seq_file *m, void *p)
{
	unsigned long nr_partials = 0;
	unsigned long nr_slabs = 0;
	unsigned long nr_inuse = 0;
5460 5461
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
P
Pekka J Enberg 已提交
5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474
	struct kmem_cache *s;
	int node;

	s = list_entry(p, struct kmem_cache, list);

	for_each_online_node(node) {
		struct kmem_cache_node *n = get_node(s, node);

		if (!n)
			continue;

		nr_partials += n->nr_partial;
		nr_slabs += atomic_long_read(&n->nr_slabs);
5475 5476
		nr_objs += atomic_long_read(&n->total_objects);
		nr_free += count_partial(n, count_free);
P
Pekka J Enberg 已提交
5477 5478
	}

5479
	nr_inuse = nr_objs - nr_free;
P
Pekka J Enberg 已提交
5480 5481

	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
5482 5483
		   nr_objs, s->size, oo_objects(s->oo),
		   (1 << oo_order(s->oo)));
P
Pekka J Enberg 已提交
5484 5485 5486 5487 5488 5489 5490
	seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
	seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
		   0UL);
	seq_putc(m, '\n');
	return 0;
}

5491
static const struct seq_operations slabinfo_op = {
P
Pekka J Enberg 已提交
5492 5493 5494 5495 5496 5497
	.start = s_start,
	.next = s_next,
	.stop = s_stop,
	.show = s_show,
};

5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511
static int slabinfo_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &slabinfo_op);
}

static const struct file_operations proc_slabinfo_operations = {
	.open		= slabinfo_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init slab_proc_init(void)
{
5512
	proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
5513 5514 5515
	return 0;
}
module_init(slab_proc_init);
5516
#endif /* CONFIG_SLABINFO */