slub.c 116.1 KB
Newer Older
C
Christoph Lameter 已提交
1 2 3 4 5 6 7
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
 * The allocator synchronizes using per slab locks and only
 * uses a centralized lock to manage a pool of partial slabs.
 *
C
Christoph Lameter 已提交
8
 * (C) 2007 SGI, Christoph Lameter
C
Christoph Lameter 已提交
9 10 11
 */

#include <linux/mm.h>
N
Nick Piggin 已提交
12
#include <linux/swap.h> /* struct reclaim_state */
C
Christoph Lameter 已提交
13 14 15 16 17
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/slab.h>
18
#include <linux/proc_fs.h>
C
Christoph Lameter 已提交
19
#include <linux/seq_file.h>
V
Vegard Nossum 已提交
20
#include <linux/kmemcheck.h>
C
Christoph Lameter 已提交
21 22 23 24
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
25
#include <linux/debugobjects.h>
C
Christoph Lameter 已提交
26
#include <linux/kallsyms.h>
27
#include <linux/memory.h>
R
Roman Zippel 已提交
28
#include <linux/math64.h>
A
Akinobu Mita 已提交
29
#include <linux/fault-inject.h>
C
Christoph Lameter 已提交
30

31 32
#include <trace/events/kmem.h>

C
Christoph Lameter 已提交
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
/*
 * Lock order:
 *   1. slab_lock(page)
 *   2. slab->list_lock
 *
 *   The slab_lock protects operations on the object of a particular
 *   slab and its metadata in the page struct. If the slab lock
 *   has been taken then no allocations nor frees can be performed
 *   on the objects in the slab nor can the slab be added or removed
 *   from the partial or full lists since this would mean modifying
 *   the page_struct of the slab.
 *
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
 *
 *   The lock order is sometimes inverted when we are trying to get a slab
 *   off a list. We take the list_lock and then look for a page on the list
 *   to use. While we do that objects in the slabs may be freed. We can
 *   only operate on the slab if we have also taken the slab_lock. So we use
 *   a slab_trylock() on the slab. If trylock was successful then no frees
 *   can occur anymore and we can use the slab for allocations etc. If the
 *   slab_trylock() does not succeed then frees are in progress in the slab and
 *   we must stay away from it for a while since we may cause a bouncing
 *   cacheline if we try to acquire the lock. So go onto the next slab.
 *   If all pages are busy then we may allocate a new slab instead of reusing
 *   a partial slab. A new slab has noone operating on it and thus there is
 *   no danger of cacheline contention.
 *
 *   Interrupts are disabled during allocation and deallocation in order to
 *   make the slab allocator safe to use in the context of an irq. In addition
 *   interrupts are disabled to ensure that the processor does not change
 *   while handling per_cpu slabs, due to kernel preemption.
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
C
Christoph Lameter 已提交
78 79
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
C
Christoph Lameter 已提交
80
 * freed then the slab will show up again on the partial lists.
C
Christoph Lameter 已提交
81 82
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
C
Christoph Lameter 已提交
83 84 85 86 87 88 89
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
 * Overloading of page flags that are otherwise used for LRU management.
 *
90 91 92 93 94 95 96 97 98 99 100 101
 * PageActive 		The slab is frozen and exempt from list processing.
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
102
 * 			freelist that allows lockless access to
103 104
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
C
Christoph Lameter 已提交
105 106 107
 *
 * PageError		Slab requires special handling due to debug
 * 			options set. This moves	slab handling out of
108
 * 			the fast path and disables lockless freelists.
C
Christoph Lameter 已提交
109 110
 */

111 112 113 114 115
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
		SLAB_TRACE | SLAB_DEBUG_FREE)

static inline int kmem_cache_debug(struct kmem_cache *s)
{
116
#ifdef CONFIG_SLUB_DEBUG
117
	return unlikely(s->flags & SLAB_DEBUG_FLAGS);
118
#else
119
	return 0;
120
#endif
121
}
122

C
Christoph Lameter 已提交
123 124 125 126 127 128 129 130 131 132 133
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST

134 135 136 137
/*
 * Mininum number of partial slabs. These will be left on the partial
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
C
Christoph Lameter 已提交
138
#define MIN_PARTIAL 5
C
Christoph Lameter 已提交
139

140 141 142 143 144 145 146
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
 * sort the partial list by the number of objects in the.
 */
#define MAX_PARTIAL 10

C
Christoph Lameter 已提交
147 148
#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)
C
Christoph Lameter 已提交
149

150
/*
151 152 153
 * Debugging flags that require metadata to be stored in the slab.  These get
 * disabled when slub_debug=O is used and a cache's min order increases with
 * metadata.
154
 */
155
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
156

C
Christoph Lameter 已提交
157 158 159 160
/*
 * Set of flags that will prevent slab merging
 */
#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
161 162
		SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
		SLAB_FAILSLAB)
C
Christoph Lameter 已提交
163 164

#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
V
Vegard Nossum 已提交
165
		SLAB_CACHE_DMA | SLAB_NOTRACK)
C
Christoph Lameter 已提交
166

167 168 169 170
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
#define MAX_OBJS_PER_PAGE	65535 /* since page.objects is u16 */

C
Christoph Lameter 已提交
171
/* Internal SLUB flags */
C
Christoph Lameter 已提交
172
#define __OBJECT_POISON		0x80000000UL /* Poison object */
C
Christoph Lameter 已提交
173 174 175 176 177 178 179 180 181

static int kmem_size = sizeof(struct kmem_cache);

#ifdef CONFIG_SMP
static struct notifier_block slab_notifier;
#endif

static enum {
	DOWN,		/* No slab functionality available */
182
	PARTIAL,	/* Kmem_cache_node works */
C
Christoph Lameter 已提交
183
	UP,		/* Everything works but does not show up in sysfs */
C
Christoph Lameter 已提交
184 185 186 187 188
	SYSFS		/* Sysfs up */
} slab_state = DOWN;

/* A list of all slab caches on the system */
static DECLARE_RWSEM(slub_lock);
A
Adrian Bunk 已提交
189
static LIST_HEAD(slab_caches);
C
Christoph Lameter 已提交
190

191 192 193 194
/*
 * Tracking user of a slab.
 */
struct track {
195
	unsigned long addr;	/* Called from address */
196 197 198 199 200 201 202
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

203
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
204 205 206
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
static void sysfs_slab_remove(struct kmem_cache *);
207

C
Christoph Lameter 已提交
208
#else
209 210 211
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
C
Christoph Lameter 已提交
212 213
static inline void sysfs_slab_remove(struct kmem_cache *s)
{
P
Pekka Enberg 已提交
214
	kfree(s->name);
C
Christoph Lameter 已提交
215 216
	kfree(s);
}
217

C
Christoph Lameter 已提交
218 219
#endif

220
static inline void stat(const struct kmem_cache *s, enum stat_item si)
221 222
{
#ifdef CONFIG_SLUB_STATS
223
	__this_cpu_inc(s->cpu_slab->stat[si]);
224 225 226
#endif
}

C
Christoph Lameter 已提交
227 228 229 230 231 232 233 234 235 236 237 238 239 240
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

int slab_is_available(void)
{
	return slab_state >= UP;
}

static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
	return s->node[node];
}

C
Christoph Lameter 已提交
241
/* Verify that a pointer has an address that is valid within a slab page */
242 243 244 245 246
static inline int check_valid_pointer(struct kmem_cache *s,
				struct page *page, const void *object)
{
	void *base;

247
	if (!object)
248 249
		return 1;

250
	base = page_address(page);
251
	if (object < base || object >= base + page->objects * s->size ||
252 253 254 255 256 257 258
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

259 260 261 262 263 264 265 266 267 268 269
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
	return *(void **)(object + s->offset);
}

static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
	*(void **)(object + s->offset) = fp;
}

/* Loop over all objects in a slab */
270 271
#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
272 273 274 275 276 277 278 279
			__p += (__s)->size)

/* Determine object index from a given position */
static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
{
	return (p - addr) / s->size;
}

280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
static inline size_t slab_ksize(const struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_DEBUG
	/*
	 * Debugging requires use of the padding between object
	 * and whatever may come after it.
	 */
	if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
		return s->objsize;

#endif
	/*
	 * If we have the need to store the freelist pointer
	 * back there or track user information then we can
	 * only use the space before that information.
	 */
	if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
		return s->inuse;
	/*
	 * Else we can use all the padding etc for the allocation
	 */
	return s->size;
}

304 305 306 307 308
static inline int order_objects(int order, unsigned long size, int reserved)
{
	return ((PAGE_SIZE << order) - reserved) / size;
}

309
static inline struct kmem_cache_order_objects oo_make(int order,
310
		unsigned long size, int reserved)
311 312
{
	struct kmem_cache_order_objects x = {
313
		(order << OO_SHIFT) + order_objects(order, size, reserved)
314 315 316 317 318 319 320
	};

	return x;
}

static inline int oo_order(struct kmem_cache_order_objects x)
{
321
	return x.x >> OO_SHIFT;
322 323 324 325
}

static inline int oo_objects(struct kmem_cache_order_objects x)
{
326
	return x.x & OO_MASK;
327 328
}

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
/*
 * Determine a map of object in use on a page.
 *
 * Slab lock or node listlock must be held to guarantee that the page does
 * not vanish from under us.
 */
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
{
	void *p;
	void *addr = page_address(page);

	for (p = page->freelist; p; p = get_freepointer(s, p))
		set_bit(slab_index(p, s, addr), map);
}

C
Christoph Lameter 已提交
344 345 346 347
#ifdef CONFIG_SLUB_DEBUG
/*
 * Debug settings:
 */
348 349 350
#ifdef CONFIG_SLUB_DEBUG_ON
static int slub_debug = DEBUG_DEFAULT_FLAGS;
#else
C
Christoph Lameter 已提交
351
static int slub_debug;
352
#endif
C
Christoph Lameter 已提交
353 354

static char *slub_debug_slabs;
355
static int disable_higher_order_debug;
C
Christoph Lameter 已提交
356

C
Christoph Lameter 已提交
357 358 359 360 361 362 363 364 365 366 367 368 369
/*
 * Object debugging
 */
static void print_section(char *text, u8 *addr, unsigned int length)
{
	int i, offset;
	int newline = 1;
	char ascii[17];

	ascii[16] = 0;

	for (i = 0; i < length; i++) {
		if (newline) {
370
			printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
C
Christoph Lameter 已提交
371 372
			newline = 0;
		}
P
Pekka Enberg 已提交
373
		printk(KERN_CONT " %02x", addr[i]);
C
Christoph Lameter 已提交
374 375 376
		offset = i % 16;
		ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
		if (offset == 15) {
P
Pekka Enberg 已提交
377
			printk(KERN_CONT " %s\n", ascii);
C
Christoph Lameter 已提交
378 379 380 381 382 383
			newline = 1;
		}
	}
	if (!newline) {
		i %= 16;
		while (i < 16) {
P
Pekka Enberg 已提交
384
			printk(KERN_CONT "   ");
C
Christoph Lameter 已提交
385 386 387
			ascii[i] = ' ';
			i++;
		}
P
Pekka Enberg 已提交
388
		printk(KERN_CONT " %s\n", ascii);
C
Christoph Lameter 已提交
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
	}
}

static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
406
			enum track_item alloc, unsigned long addr)
C
Christoph Lameter 已提交
407
{
A
Akinobu Mita 已提交
408
	struct track *p = get_track(s, object, alloc);
C
Christoph Lameter 已提交
409 410 411 412

	if (addr) {
		p->addr = addr;
		p->cpu = smp_processor_id();
A
Alexey Dobriyan 已提交
413
		p->pid = current->pid;
C
Christoph Lameter 已提交
414 415 416 417 418 419 420
		p->when = jiffies;
	} else
		memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
421 422 423
	if (!(s->flags & SLAB_STORE_USER))
		return;

424 425
	set_track(s, object, TRACK_FREE, 0UL);
	set_track(s, object, TRACK_ALLOC, 0UL);
C
Christoph Lameter 已提交
426 427 428 429 430 431 432
}

static void print_track(const char *s, struct track *t)
{
	if (!t->addr)
		return;

433
	printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
434
		s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
435 436 437 438 439 440 441 442 443 444 445 446 447
}

static void print_tracking(struct kmem_cache *s, void *object)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	print_track("Allocated", get_track(s, object, TRACK_ALLOC));
	print_track("Freed", get_track(s, object, TRACK_FREE));
}

static void print_page_info(struct page *page)
{
448 449
	printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
		page, page->objects, page->inuse, page->freelist, page->flags);
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465

}

static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[100];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "========================================"
			"=====================================\n");
	printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
	printk(KERN_ERR "----------------------------------------"
			"-------------------------------------\n\n");
C
Christoph Lameter 已提交
466 467
}

468 469 470 471 472 473 474 475 476 477 478 479
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
	va_list args;
	char buf[100];

	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
	va_end(args);
	printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
}

static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
C
Christoph Lameter 已提交
480 481
{
	unsigned int off;	/* Offset of last byte */
482
	u8 *addr = page_address(page);
483 484 485 486 487 488 489 490 491 492 493

	print_tracking(s, p);

	print_page_info(page);

	printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
			p, p - addr, get_freepointer(s, p));

	if (p > addr + 16)
		print_section("Bytes b4", p - 16, 16);

494
	print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE));
C
Christoph Lameter 已提交
495 496 497 498 499 500 501 502 503 504

	if (s->flags & SLAB_RED_ZONE)
		print_section("Redzone", p + s->objsize,
			s->inuse - s->objsize);

	if (s->offset)
		off = s->offset + sizeof(void *);
	else
		off = s->inuse;

505
	if (s->flags & SLAB_STORE_USER)
C
Christoph Lameter 已提交
506 507 508 509
		off += 2 * sizeof(struct track);

	if (off != s->size)
		/* Beginning of the filler is the free pointer */
510 511 512
		print_section("Padding", p + off, s->size - off);

	dump_stack();
C
Christoph Lameter 已提交
513 514 515 516 517
}

static void object_err(struct kmem_cache *s, struct page *page,
			u8 *object, char *reason)
{
518
	slab_bug(s, "%s", reason);
519
	print_trailer(s, page, object);
C
Christoph Lameter 已提交
520 521
}

522
static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
C
Christoph Lameter 已提交
523 524 525 526
{
	va_list args;
	char buf[100];

527 528
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
C
Christoph Lameter 已提交
529
	va_end(args);
530
	slab_bug(s, "%s", buf);
531
	print_page_info(page);
C
Christoph Lameter 已提交
532 533 534
	dump_stack();
}

535
static void init_object(struct kmem_cache *s, void *object, u8 val)
C
Christoph Lameter 已提交
536 537 538 539 540
{
	u8 *p = object;

	if (s->flags & __OBJECT_POISON) {
		memset(p, POISON_FREE, s->objsize - 1);
P
Pekka Enberg 已提交
541
		p[s->objsize - 1] = POISON_END;
C
Christoph Lameter 已提交
542 543 544
	}

	if (s->flags & SLAB_RED_ZONE)
545
		memset(p + s->objsize, val, s->inuse - s->objsize);
C
Christoph Lameter 已提交
546 547
}

548
static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
C
Christoph Lameter 已提交
549 550 551
{
	while (bytes) {
		if (*start != (u8)value)
552
			return start;
C
Christoph Lameter 已提交
553 554 555
		start++;
		bytes--;
	}
556 557 558 559 560 561 562 563 564 565 566 567
	return NULL;
}

static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
	memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
			u8 *object, char *what,
P
Pekka Enberg 已提交
568
			u8 *start, unsigned int value, unsigned int bytes)
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
{
	u8 *fault;
	u8 *end;

	fault = check_bytes(start, value, bytes);
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
	printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
					fault, end - 1, fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
C
Christoph Lameter 已提交
588 589 590 591 592 593 594 595 596
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
 * 	pointer is the first word of the object.
C
Christoph Lameter 已提交
597
 *
C
Christoph Lameter 已提交
598 599 600 601 602
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
 * object + s->objsize
 * 	Padding to reach word boundary. This is also used for Redzoning.
C
Christoph Lameter 已提交
603 604 605
 * 	Padding is extended by another word if Redzoning is enabled and
 * 	objsize == inuse.
 *
C
Christoph Lameter 已提交
606 607 608 609
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
C
Christoph Lameter 已提交
610 611
 * 	Meta data starts here.
 *
C
Christoph Lameter 已提交
612 613
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
C
Christoph Lameter 已提交
614
 * 	C. Padding to reach required alignment boundary or at mininum
C
Christoph Lameter 已提交
615
 * 		one word if debugging is on to be able to detect writes
C
Christoph Lameter 已提交
616 617 618
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
C
Christoph Lameter 已提交
619 620
 *
 * object + s->size
C
Christoph Lameter 已提交
621
 * 	Nothing is used beyond s->size.
C
Christoph Lameter 已提交
622
 *
C
Christoph Lameter 已提交
623 624
 * If slabcaches are merged then the objsize and inuse boundaries are mostly
 * ignored. And therefore no slab options that rely on these boundaries
C
Christoph Lameter 已提交
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
 * may be used with merged slabcaches.
 */

static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
{
	unsigned long off = s->inuse;	/* The end of info */

	if (s->offset)
		/* Freepointer is placed after the object. */
		off += sizeof(void *);

	if (s->flags & SLAB_STORE_USER)
		/* We also have user information there */
		off += 2 * sizeof(struct track);

	if (s->size == off)
		return 1;

643 644
	return check_bytes_and_report(s, page, p, "Object padding",
				p + off, POISON_INUSE, s->size - off);
C
Christoph Lameter 已提交
645 646
}

647
/* Check the pad bytes at the end of a slab page */
C
Christoph Lameter 已提交
648 649
static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
650 651 652 653 654
	u8 *start;
	u8 *fault;
	u8 *end;
	int length;
	int remainder;
C
Christoph Lameter 已提交
655 656 657 658

	if (!(s->flags & SLAB_POISON))
		return 1;

659
	start = page_address(page);
660
	length = (PAGE_SIZE << compound_order(page)) - s->reserved;
661 662
	end = start + length;
	remainder = length % s->size;
C
Christoph Lameter 已提交
663 664 665
	if (!remainder)
		return 1;

666
	fault = check_bytes(end - remainder, POISON_INUSE, remainder);
667 668 669 670 671 672
	if (!fault)
		return 1;
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
673
	print_section("Padding", end - remainder, remainder);
674

E
Eric Dumazet 已提交
675
	restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
676
	return 0;
C
Christoph Lameter 已提交
677 678 679
}

static int check_object(struct kmem_cache *s, struct page *page,
680
					void *object, u8 val)
C
Christoph Lameter 已提交
681 682 683 684 685
{
	u8 *p = object;
	u8 *endobject = object + s->objsize;

	if (s->flags & SLAB_RED_ZONE) {
686
		if (!check_bytes_and_report(s, page, object, "Redzone",
687
			endobject, val, s->inuse - s->objsize))
C
Christoph Lameter 已提交
688 689
			return 0;
	} else {
I
Ingo Molnar 已提交
690 691 692 693
		if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
			check_bytes_and_report(s, page, p, "Alignment padding",
				endobject, POISON_INUSE, s->inuse - s->objsize);
		}
C
Christoph Lameter 已提交
694 695 696
	}

	if (s->flags & SLAB_POISON) {
697
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
698 699 700
			(!check_bytes_and_report(s, page, p, "Poison", p,
					POISON_FREE, s->objsize - 1) ||
			 !check_bytes_and_report(s, page, p, "Poison",
P
Pekka Enberg 已提交
701
				p + s->objsize - 1, POISON_END, 1)))
C
Christoph Lameter 已提交
702 703 704 705 706 707 708
			return 0;
		/*
		 * check_pad_bytes cleans up on its own.
		 */
		check_pad_bytes(s, page, p);
	}

709
	if (!s->offset && val == SLUB_RED_ACTIVE)
C
Christoph Lameter 已提交
710 711 712 713 714 715 716 717 718 719
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
	if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
		object_err(s, page, p, "Freepointer corrupt");
		/*
N
Nick Andrew 已提交
720
		 * No choice but to zap it and thus lose the remainder
C
Christoph Lameter 已提交
721
		 * of the free objects in this slab. May cause
C
Christoph Lameter 已提交
722
		 * another error because the object count is now wrong.
C
Christoph Lameter 已提交
723
		 */
724
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
725 726 727 728 729 730 731
		return 0;
	}
	return 1;
}

static int check_slab(struct kmem_cache *s, struct page *page)
{
732 733
	int maxobj;

C
Christoph Lameter 已提交
734 735 736
	VM_BUG_ON(!irqs_disabled());

	if (!PageSlab(page)) {
737
		slab_err(s, page, "Not a valid slab page");
C
Christoph Lameter 已提交
738 739
		return 0;
	}
740

741
	maxobj = order_objects(compound_order(page), s->size, s->reserved);
742 743 744 745 746 747
	if (page->objects > maxobj) {
		slab_err(s, page, "objects %u > max %u",
			s->name, page->objects, maxobj);
		return 0;
	}
	if (page->inuse > page->objects) {
748
		slab_err(s, page, "inuse %u > max %u",
749
			s->name, page->inuse, page->objects);
C
Christoph Lameter 已提交
750 751 752 753 754 755 756 757
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
	slab_pad_check(s, page);
	return 1;
}

/*
C
Christoph Lameter 已提交
758 759
 * Determine if a certain object on a page is on the freelist. Must hold the
 * slab lock to guarantee that the chains are in a consistent state.
C
Christoph Lameter 已提交
760 761 762 763 764 765
 */
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
	int nr = 0;
	void *fp = page->freelist;
	void *object = NULL;
766
	unsigned long max_objects;
C
Christoph Lameter 已提交
767

768
	while (fp && nr <= page->objects) {
C
Christoph Lameter 已提交
769 770 771 772 773 774
		if (fp == search)
			return 1;
		if (!check_valid_pointer(s, page, fp)) {
			if (object) {
				object_err(s, page, object,
					"Freechain corrupt");
775
				set_freepointer(s, object, NULL);
C
Christoph Lameter 已提交
776 777
				break;
			} else {
778
				slab_err(s, page, "Freepointer corrupt");
779
				page->freelist = NULL;
780
				page->inuse = page->objects;
781
				slab_fix(s, "Freelist cleared");
C
Christoph Lameter 已提交
782 783 784 785 786 787 788 789 790
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

791
	max_objects = order_objects(compound_order(page), s->size, s->reserved);
792 793
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
794 795 796 797 798 799 800

	if (page->objects != max_objects) {
		slab_err(s, page, "Wrong number of objects. Found %d but "
			"should be %d", page->objects, max_objects);
		page->objects = max_objects;
		slab_fix(s, "Number of objects adjusted.");
	}
801
	if (page->inuse != page->objects - nr) {
802
		slab_err(s, page, "Wrong object count. Counter is %d but "
803 804
			"counted were %d", page->inuse, page->objects - nr);
		page->inuse = page->objects - nr;
805
		slab_fix(s, "Object count adjusted.");
C
Christoph Lameter 已提交
806 807 808 809
	}
	return search == NULL;
}

810 811
static void trace(struct kmem_cache *s, struct page *page, void *object,
								int alloc)
C
Christoph Lameter 已提交
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
{
	if (s->flags & SLAB_TRACE) {
		printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
			print_section("Object", (void *)object, s->objsize);

		dump_stack();
	}
}

827 828 829 830 831 832
/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
 */
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
833
	flags &= gfp_allowed_mask;
834 835 836 837 838 839 840 841
	lockdep_trace_alloc(flags);
	might_sleep_if(flags & __GFP_WAIT);

	return should_failslab(s->objsize, flags, s->flags);
}

static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{
842
	flags &= gfp_allowed_mask;
843
	kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
844 845 846 847 848 849 850
	kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
}

static inline void slab_free_hook(struct kmem_cache *s, void *x)
{
	kmemleak_free_recursive(x, s->flags);

851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
	/*
	 * Trouble is that we may no longer disable interupts in the fast path
	 * So in order to make the debug calls that expect irqs to be
	 * disabled we need to disable interrupts temporarily.
	 */
#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
	{
		unsigned long flags;

		local_irq_save(flags);
		kmemcheck_slab_free(s, x, s->objsize);
		debug_check_no_locks_freed(x, s->objsize);
		local_irq_restore(flags);
	}
#endif
866 867
	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->objsize);
868 869
}

870
/*
C
Christoph Lameter 已提交
871
 * Tracking of fully allocated slabs for debugging purposes.
872
 */
C
Christoph Lameter 已提交
873
static void add_full(struct kmem_cache_node *n, struct page *page)
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
{
	spin_lock(&n->list_lock);
	list_add(&page->lru, &n->full);
	spin_unlock(&n->list_lock);
}

static void remove_full(struct kmem_cache *s, struct page *page)
{
	struct kmem_cache_node *n;

	if (!(s->flags & SLAB_STORE_USER))
		return;

	n = get_node(s, page_to_nid(page));

	spin_lock(&n->list_lock);
	list_del(&page->lru);
	spin_unlock(&n->list_lock);
}

894 895 896 897 898 899 900 901
/* Tracking of the number of slabs for debugging purposes */
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
{
	struct kmem_cache_node *n = get_node(s, node);

	return atomic_long_read(&n->nr_slabs);
}

902 903 904 905 906
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

907
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
908 909 910 911 912 913 914 915 916
{
	struct kmem_cache_node *n = get_node(s, node);

	/*
	 * May be called early in order to allocate a slab for the
	 * kmem_cache_node structure. Solve the chicken-egg
	 * dilemma by deferring the increment of the count during
	 * bootstrap (see early_kmem_cache_node_alloc).
	 */
917
	if (n) {
918
		atomic_long_inc(&n->nr_slabs);
919 920
		atomic_long_add(objects, &n->total_objects);
	}
921
}
922
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
923 924 925 926
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
927
	atomic_long_sub(objects, &n->total_objects);
928 929 930
}

/* Object debug checks for alloc/free paths */
C
Christoph Lameter 已提交
931 932 933 934 935 936
static void setup_object_debug(struct kmem_cache *s, struct page *page,
								void *object)
{
	if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
		return;

937
	init_object(s, object, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
938 939 940
	init_tracking(s, object);
}

941
static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *page,
942
					void *object, unsigned long addr)
C
Christoph Lameter 已提交
943 944 945 946
{
	if (!check_slab(s, page))
		goto bad;

947
	if (!on_freelist(s, page, object)) {
948
		object_err(s, page, object, "Object already allocated");
949
		goto bad;
C
Christoph Lameter 已提交
950 951 952 953
	}

	if (!check_valid_pointer(s, page, object)) {
		object_err(s, page, object, "Freelist Pointer check fails");
954
		goto bad;
C
Christoph Lameter 已提交
955 956
	}

957
	if (!check_object(s, page, object, SLUB_RED_INACTIVE))
C
Christoph Lameter 已提交
958 959
		goto bad;

C
Christoph Lameter 已提交
960 961 962 963
	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
	trace(s, page, object, 1);
964
	init_object(s, object, SLUB_RED_ACTIVE);
C
Christoph Lameter 已提交
965
	return 1;
C
Christoph Lameter 已提交
966

C
Christoph Lameter 已提交
967 968 969 970 971
bad:
	if (PageSlab(page)) {
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
C
Christoph Lameter 已提交
972
		 * as used avoids touching the remaining objects.
C
Christoph Lameter 已提交
973
		 */
974
		slab_fix(s, "Marking all objects used");
975
		page->inuse = page->objects;
976
		page->freelist = NULL;
C
Christoph Lameter 已提交
977 978 979 980
	}
	return 0;
}

981 982
static noinline int free_debug_processing(struct kmem_cache *s,
		 struct page *page, void *object, unsigned long addr)
C
Christoph Lameter 已提交
983 984 985 986 987
{
	if (!check_slab(s, page))
		goto fail;

	if (!check_valid_pointer(s, page, object)) {
988
		slab_err(s, page, "Invalid object pointer 0x%p", object);
C
Christoph Lameter 已提交
989 990 991 992
		goto fail;
	}

	if (on_freelist(s, page, object)) {
993
		object_err(s, page, object, "Object already free");
C
Christoph Lameter 已提交
994 995 996
		goto fail;
	}

997
	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
C
Christoph Lameter 已提交
998 999 1000
		return 0;

	if (unlikely(s != page->slab)) {
I
Ingo Molnar 已提交
1001
		if (!PageSlab(page)) {
1002 1003
			slab_err(s, page, "Attempt to free object(0x%p) "
				"outside of slab", object);
I
Ingo Molnar 已提交
1004
		} else if (!page->slab) {
C
Christoph Lameter 已提交
1005
			printk(KERN_ERR
1006
				"SLUB <none>: no slab for object 0x%p.\n",
C
Christoph Lameter 已提交
1007
						object);
1008
			dump_stack();
P
Pekka Enberg 已提交
1009
		} else
1010 1011
			object_err(s, page, object,
					"page slab pointer corrupt.");
C
Christoph Lameter 已提交
1012 1013
		goto fail;
	}
C
Christoph Lameter 已提交
1014 1015

	/* Special debug activities for freeing objects */
1016
	if (!PageSlubFrozen(page) && !page->freelist)
C
Christoph Lameter 已提交
1017 1018 1019 1020
		remove_full(s, page);
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
1021
	init_object(s, object, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1022
	return 1;
C
Christoph Lameter 已提交
1023

C
Christoph Lameter 已提交
1024
fail:
1025
	slab_fix(s, "Object at 0x%p not freed", object);
C
Christoph Lameter 已提交
1026 1027 1028
	return 0;
}

C
Christoph Lameter 已提交
1029 1030
static int __init setup_slub_debug(char *str)
{
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
	slub_debug = DEBUG_DEFAULT_FLAGS;
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	if (*str == ',')
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
		goto check_slabs;

1045 1046 1047 1048 1049 1050 1051 1052 1053
	if (tolower(*str) == 'o') {
		/*
		 * Avoid enabling debugging on caches if its minimum order
		 * would increase as a result.
		 */
		disable_higher_order_debug = 1;
		goto out;
	}

1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
	slub_debug = 0;
	if (*str == '-')
		/*
		 * Switch off all debugging measures.
		 */
		goto out;

	/*
	 * Determine which debug features should be switched on
	 */
P
Pekka Enberg 已提交
1064
	for (; *str && *str != ','; str++) {
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
		switch (tolower(*str)) {
		case 'f':
			slub_debug |= SLAB_DEBUG_FREE;
			break;
		case 'z':
			slub_debug |= SLAB_RED_ZONE;
			break;
		case 'p':
			slub_debug |= SLAB_POISON;
			break;
		case 'u':
			slub_debug |= SLAB_STORE_USER;
			break;
		case 't':
			slub_debug |= SLAB_TRACE;
			break;
1081 1082 1083
		case 'a':
			slub_debug |= SLAB_FAILSLAB;
			break;
1084 1085
		default:
			printk(KERN_ERR "slub_debug option '%c' "
P
Pekka Enberg 已提交
1086
				"unknown. skipped\n", *str);
1087
		}
C
Christoph Lameter 已提交
1088 1089
	}

1090
check_slabs:
C
Christoph Lameter 已提交
1091 1092
	if (*str == ',')
		slub_debug_slabs = str + 1;
1093
out:
C
Christoph Lameter 已提交
1094 1095 1096 1097 1098
	return 1;
}

__setup("slub_debug", setup_slub_debug);

1099 1100
static unsigned long kmem_cache_flags(unsigned long objsize,
	unsigned long flags, const char *name,
1101
	void (*ctor)(void *))
C
Christoph Lameter 已提交
1102 1103
{
	/*
1104
	 * Enable debugging if selected on the kernel commandline.
C
Christoph Lameter 已提交
1105
	 */
1106
	if (slub_debug && (!slub_debug_slabs ||
1107 1108
		!strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
		flags |= slub_debug;
1109 1110

	return flags;
C
Christoph Lameter 已提交
1111 1112
}
#else
C
Christoph Lameter 已提交
1113 1114
static inline void setup_object_debug(struct kmem_cache *s,
			struct page *page, void *object) {}
C
Christoph Lameter 已提交
1115

C
Christoph Lameter 已提交
1116
static inline int alloc_debug_processing(struct kmem_cache *s,
1117
	struct page *page, void *object, unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1118

C
Christoph Lameter 已提交
1119
static inline int free_debug_processing(struct kmem_cache *s,
1120
	struct page *page, void *object, unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1121 1122 1123 1124

static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
			{ return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page,
1125
			void *object, u8 val) { return 1; }
C
Christoph Lameter 已提交
1126
static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
1127 1128
static inline unsigned long kmem_cache_flags(unsigned long objsize,
	unsigned long flags, const char *name,
1129
	void (*ctor)(void *))
1130 1131 1132
{
	return flags;
}
C
Christoph Lameter 已提交
1133
#define slub_debug 0
1134

1135 1136
#define disable_higher_order_debug 0

1137 1138
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }
1139 1140
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1141 1142 1143 1144
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1145 1146 1147 1148 1149 1150 1151 1152 1153

static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
							{ return 0; }

static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
		void *object) {}

static inline void slab_free_hook(struct kmem_cache *s, void *x) {}

1154
#endif /* CONFIG_SLUB_DEBUG */
1155

C
Christoph Lameter 已提交
1156 1157 1158
/*
 * Slab allocation and freeing
 */
1159 1160 1161 1162 1163
static inline struct page *alloc_slab_page(gfp_t flags, int node,
					struct kmem_cache_order_objects oo)
{
	int order = oo_order(oo);

1164 1165
	flags |= __GFP_NOTRACK;

1166
	if (node == NUMA_NO_NODE)
1167 1168
		return alloc_pages(flags, order);
	else
1169
		return alloc_pages_exact_node(node, flags, order);
1170 1171
}

C
Christoph Lameter 已提交
1172 1173
static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
{
P
Pekka Enberg 已提交
1174
	struct page *page;
1175
	struct kmem_cache_order_objects oo = s->oo;
1176
	gfp_t alloc_gfp;
C
Christoph Lameter 已提交
1177

1178
	flags |= s->allocflags;
1179

1180 1181 1182 1183 1184 1185 1186
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;

	page = alloc_slab_page(alloc_gfp, node, oo);
1187 1188 1189 1190 1191 1192 1193 1194 1195
	if (unlikely(!page)) {
		oo = s->min;
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
		page = alloc_slab_page(flags, node, oo);
		if (!page)
			return NULL;
C
Christoph Lameter 已提交
1196

1197
		stat(s, ORDER_FALLBACK);
1198
	}
V
Vegard Nossum 已提交
1199 1200

	if (kmemcheck_enabled
1201
		&& !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
		int pages = 1 << oo_order(oo);

		kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);

		/*
		 * Objects from caches that have a constructor don't get
		 * cleared when they're allocated, so we need to do it here.
		 */
		if (s->ctor)
			kmemcheck_mark_uninitialized_pages(page, pages);
		else
			kmemcheck_mark_unallocated_pages(page, pages);
V
Vegard Nossum 已提交
1214 1215
	}

1216
	page->objects = oo_objects(oo);
C
Christoph Lameter 已提交
1217 1218 1219
	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1220
		1 << oo_order(oo));
C
Christoph Lameter 已提交
1221 1222 1223 1224 1225 1226 1227

	return page;
}

static void setup_object(struct kmem_cache *s, struct page *page,
				void *object)
{
C
Christoph Lameter 已提交
1228
	setup_object_debug(s, page, object);
1229
	if (unlikely(s->ctor))
1230
		s->ctor(object);
C
Christoph Lameter 已提交
1231 1232 1233 1234 1235 1236 1237 1238 1239
}

static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
	struct page *page;
	void *start;
	void *last;
	void *p;

C
Christoph Lameter 已提交
1240
	BUG_ON(flags & GFP_SLAB_BUG_MASK);
C
Christoph Lameter 已提交
1241

C
Christoph Lameter 已提交
1242 1243
	page = allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
C
Christoph Lameter 已提交
1244 1245 1246
	if (!page)
		goto out;

1247
	inc_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1248 1249 1250 1251 1252 1253
	page->slab = s;
	page->flags |= 1 << PG_slab;

	start = page_address(page);

	if (unlikely(s->flags & SLAB_POISON))
1254
		memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
C
Christoph Lameter 已提交
1255 1256

	last = start;
1257
	for_each_object(p, s, start, page->objects) {
C
Christoph Lameter 已提交
1258 1259 1260 1261 1262
		setup_object(s, page, last);
		set_freepointer(s, last, p);
		last = p;
	}
	setup_object(s, page, last);
1263
	set_freepointer(s, last, NULL);
C
Christoph Lameter 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272

	page->freelist = start;
	page->inuse = 0;
out:
	return page;
}

static void __free_slab(struct kmem_cache *s, struct page *page)
{
1273 1274
	int order = compound_order(page);
	int pages = 1 << order;
C
Christoph Lameter 已提交
1275

1276
	if (kmem_cache_debug(s)) {
C
Christoph Lameter 已提交
1277 1278 1279
		void *p;

		slab_pad_check(s, page);
1280 1281
		for_each_object(p, s, page_address(page),
						page->objects)
1282
			check_object(s, page, p, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1283 1284
	}

1285
	kmemcheck_free_shadow(page, compound_order(page));
V
Vegard Nossum 已提交
1286

C
Christoph Lameter 已提交
1287 1288 1289
	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
P
Pekka Enberg 已提交
1290
		-pages);
C
Christoph Lameter 已提交
1291

1292 1293
	__ClearPageSlab(page);
	reset_page_mapcount(page);
N
Nick Piggin 已提交
1294 1295
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
1296
	__free_pages(page, order);
C
Christoph Lameter 已提交
1297 1298
}

1299 1300 1301
#define need_reserve_slab_rcu						\
	(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))

C
Christoph Lameter 已提交
1302 1303 1304 1305
static void rcu_free_slab(struct rcu_head *h)
{
	struct page *page;

1306 1307 1308 1309 1310
	if (need_reserve_slab_rcu)
		page = virt_to_head_page(h);
	else
		page = container_of((struct list_head *)h, struct page, lru);

C
Christoph Lameter 已提交
1311 1312 1313 1314 1315 1316
	__free_slab(page->slab, page);
}

static void free_slab(struct kmem_cache *s, struct page *page)
{
	if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
		struct rcu_head *head;

		if (need_reserve_slab_rcu) {
			int order = compound_order(page);
			int offset = (PAGE_SIZE << order) - s->reserved;

			VM_BUG_ON(s->reserved != sizeof(*head));
			head = page_address(page) + offset;
		} else {
			/*
			 * RCU free overloads the RCU head over the LRU
			 */
			head = (void *)&page->lru;
		}
C
Christoph Lameter 已提交
1331 1332 1333 1334 1335 1336 1337 1338

		call_rcu(head, rcu_free_slab);
	} else
		__free_slab(s, page);
}

static void discard_slab(struct kmem_cache *s, struct page *page)
{
1339
	dec_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
	free_slab(s, page);
}

/*
 * Per slab locking using the pagelock
 */
static __always_inline void slab_lock(struct page *page)
{
	bit_spin_lock(PG_locked, &page->flags);
}

static __always_inline void slab_unlock(struct page *page)
{
N
Nick Piggin 已提交
1353
	__bit_spin_unlock(PG_locked, &page->flags);
C
Christoph Lameter 已提交
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
}

static __always_inline int slab_trylock(struct page *page)
{
	int rc = 1;

	rc = bit_spin_trylock(PG_locked, &page->flags);
	return rc;
}

/*
 * Management of partially allocated slabs
 */
1367 1368
static void add_partial(struct kmem_cache_node *n,
				struct page *page, int tail)
C
Christoph Lameter 已提交
1369
{
C
Christoph Lameter 已提交
1370 1371
	spin_lock(&n->list_lock);
	n->nr_partial++;
1372 1373 1374 1375
	if (tail)
		list_add_tail(&page->lru, &n->partial);
	else
		list_add(&page->lru, &n->partial);
C
Christoph Lameter 已提交
1376 1377 1378
	spin_unlock(&n->list_lock);
}

1379 1380 1381 1382 1383 1384 1385
static inline void __remove_partial(struct kmem_cache_node *n,
					struct page *page)
{
	list_del(&page->lru);
	n->nr_partial--;
}

1386
static void remove_partial(struct kmem_cache *s, struct page *page)
C
Christoph Lameter 已提交
1387 1388 1389 1390
{
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));

	spin_lock(&n->list_lock);
1391
	__remove_partial(n, page);
C
Christoph Lameter 已提交
1392 1393 1394 1395
	spin_unlock(&n->list_lock);
}

/*
C
Christoph Lameter 已提交
1396
 * Lock slab and remove from the partial list.
C
Christoph Lameter 已提交
1397
 *
C
Christoph Lameter 已提交
1398
 * Must hold list_lock.
C
Christoph Lameter 已提交
1399
 */
1400 1401
static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
							struct page *page)
C
Christoph Lameter 已提交
1402 1403
{
	if (slab_trylock(page)) {
1404
		__remove_partial(n, page);
1405
		__SetPageSlubFrozen(page);
C
Christoph Lameter 已提交
1406 1407 1408 1409 1410 1411
		return 1;
	}
	return 0;
}

/*
C
Christoph Lameter 已提交
1412
 * Try to allocate a partial slab from a specific node.
C
Christoph Lameter 已提交
1413 1414 1415 1416 1417 1418 1419 1420
 */
static struct page *get_partial_node(struct kmem_cache_node *n)
{
	struct page *page;

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
C
Christoph Lameter 已提交
1421 1422
	 * partial slab and there is none available then get_partials()
	 * will return NULL.
C
Christoph Lameter 已提交
1423 1424 1425 1426 1427 1428
	 */
	if (!n || !n->nr_partial)
		return NULL;

	spin_lock(&n->list_lock);
	list_for_each_entry(page, &n->partial, lru)
1429
		if (lock_and_freeze_slab(n, page))
C
Christoph Lameter 已提交
1430 1431 1432 1433 1434 1435 1436 1437
			goto out;
	page = NULL;
out:
	spin_unlock(&n->list_lock);
	return page;
}

/*
C
Christoph Lameter 已提交
1438
 * Get a page from somewhere. Search in increasing NUMA distances.
C
Christoph Lameter 已提交
1439 1440 1441 1442 1443
 */
static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
1444
	struct zoneref *z;
1445 1446
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
C
Christoph Lameter 已提交
1447 1448 1449
	struct page *page;

	/*
C
Christoph Lameter 已提交
1450 1451 1452 1453
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
C
Christoph Lameter 已提交
1454
	 *
C
Christoph Lameter 已提交
1455 1456 1457 1458
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
C
Christoph Lameter 已提交
1459
	 *
C
Christoph Lameter 已提交
1460
	 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
C
Christoph Lameter 已提交
1461 1462 1463 1464 1465
	 * defrag_ratio = 1000) then every (well almost) allocation will
	 * first attempt to defrag slab caches on other nodes. This means
	 * scanning over all nodes to look for partial slabs which may be
	 * expensive if we do it every time we are trying to find a slab
	 * with available objects.
C
Christoph Lameter 已提交
1466
	 */
1467 1468
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
C
Christoph Lameter 已提交
1469 1470
		return NULL;

1471
	get_mems_allowed();
1472
	zonelist = node_zonelist(slab_node(current->mempolicy), flags);
1473
	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
C
Christoph Lameter 已提交
1474 1475
		struct kmem_cache_node *n;

1476
		n = get_node(s, zone_to_nid(zone));
C
Christoph Lameter 已提交
1477

1478
		if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
1479
				n->nr_partial > s->min_partial) {
C
Christoph Lameter 已提交
1480
			page = get_partial_node(n);
1481 1482
			if (page) {
				put_mems_allowed();
C
Christoph Lameter 已提交
1483
				return page;
1484
			}
C
Christoph Lameter 已提交
1485 1486
		}
	}
1487
	put_mems_allowed();
C
Christoph Lameter 已提交
1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
#endif
	return NULL;
}

/*
 * Get a partial page, lock it and return it.
 */
static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
{
	struct page *page;
1498
	int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
C
Christoph Lameter 已提交
1499 1500

	page = get_partial_node(get_node(s, searchnode));
1501
	if (page || node != NUMA_NO_NODE)
C
Christoph Lameter 已提交
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
		return page;

	return get_any_partial(s, flags);
}

/*
 * Move a page back to the lists.
 *
 * Must be called with the slab lock held.
 *
 * On exit the slab lock will have been dropped.
 */
1514
static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
N
Namhyung Kim 已提交
1515
	__releases(bitlock)
C
Christoph Lameter 已提交
1516
{
C
Christoph Lameter 已提交
1517 1518
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));

1519
	__ClearPageSlubFrozen(page);
C
Christoph Lameter 已提交
1520
	if (page->inuse) {
C
Christoph Lameter 已提交
1521

1522
		if (page->freelist) {
1523
			add_partial(n, page, tail);
1524
			stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
1525
		} else {
1526
			stat(s, DEACTIVATE_FULL);
1527
			if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
1528 1529
				add_full(n, page);
		}
C
Christoph Lameter 已提交
1530 1531
		slab_unlock(page);
	} else {
1532
		stat(s, DEACTIVATE_EMPTY);
1533
		if (n->nr_partial < s->min_partial) {
C
Christoph Lameter 已提交
1534
			/*
C
Christoph Lameter 已提交
1535 1536 1537
			 * Adding an empty slab to the partial slabs in order
			 * to avoid page allocator overhead. This slab needs
			 * to come after the other slabs with objects in
C
Christoph Lameter 已提交
1538 1539 1540
			 * so that the others get filled first. That way the
			 * size of the partial list stays small.
			 *
1541 1542
			 * kmem_cache_shrink can reclaim any empty slabs from
			 * the partial list.
C
Christoph Lameter 已提交
1543
			 */
1544
			add_partial(n, page, 1);
C
Christoph Lameter 已提交
1545 1546 1547
			slab_unlock(page);
		} else {
			slab_unlock(page);
1548
			stat(s, FREE_SLAB);
C
Christoph Lameter 已提交
1549 1550
			discard_slab(s, page);
		}
C
Christoph Lameter 已提交
1551 1552 1553
	}
}

1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
#ifdef CONFIG_CMPXCHG_LOCAL
#ifdef CONFIG_PREEMPT
/*
 * Calculate the next globally unique transaction for disambiguiation
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
#endif

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

	printk(KERN_INFO "%s %s: cmpxchg redo ", n, s->name);

#ifdef CONFIG_PREEMPT
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
		printk("due to cpu change %d -> %d\n",
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
		printk("due to cpu running other code. Event %ld->%ld\n",
			tid_to_event(tid), tid_to_event(actual_tid));
	else
		printk("for unknown reason: actual=%lx was=%lx target=%lx\n",
			actual_tid, tid, next_tid(tid));
#endif
1611
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1612 1613 1614 1615 1616 1617
}

#endif

void init_kmem_cache_cpus(struct kmem_cache *s)
{
1618
#ifdef CONFIG_CMPXCHG_LOCAL
1619 1620 1621 1622 1623 1624 1625
	int cpu;

	for_each_possible_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
#endif

}
C
Christoph Lameter 已提交
1626 1627 1628
/*
 * Remove the cpu slab
 */
1629
static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
N
Namhyung Kim 已提交
1630
	__releases(bitlock)
C
Christoph Lameter 已提交
1631
{
1632
	struct page *page = c->page;
1633
	int tail = 1;
1634

1635
	if (page->freelist)
1636
		stat(s, DEACTIVATE_REMOTE_FREES);
1637
	/*
C
Christoph Lameter 已提交
1638
	 * Merge cpu freelist into slab freelist. Typically we get here
1639 1640 1641
	 * because both freelists are empty. So this is unlikely
	 * to occur.
	 */
1642
	while (unlikely(c->freelist)) {
1643 1644
		void **object;

1645 1646
		tail = 0;	/* Hot objects. Put the slab first */

1647
		/* Retrieve object from cpu_freelist */
1648
		object = c->freelist;
1649
		c->freelist = get_freepointer(s, c->freelist);
1650 1651

		/* And put onto the regular freelist */
1652
		set_freepointer(s, object, page->freelist);
1653 1654 1655
		page->freelist = object;
		page->inuse--;
	}
1656
	c->page = NULL;
1657 1658 1659
#ifdef CONFIG_CMPXCHG_LOCAL
	c->tid = next_tid(c->tid);
#endif
1660
	unfreeze_slab(s, page, tail);
C
Christoph Lameter 已提交
1661 1662
}

1663
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1664
{
1665
	stat(s, CPUSLAB_FLUSH);
1666 1667
	slab_lock(c->page);
	deactivate_slab(s, c);
C
Christoph Lameter 已提交
1668 1669 1670 1671
}

/*
 * Flush cpu slab.
C
Christoph Lameter 已提交
1672
 *
C
Christoph Lameter 已提交
1673 1674
 * Called from IPI handler with interrupts disabled.
 */
1675
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
C
Christoph Lameter 已提交
1676
{
1677
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
C
Christoph Lameter 已提交
1678

1679 1680
	if (likely(c && c->page))
		flush_slab(s, c);
C
Christoph Lameter 已提交
1681 1682 1683 1684 1685 1686
}

static void flush_cpu_slab(void *d)
{
	struct kmem_cache *s = d;

1687
	__flush_cpu_slab(s, smp_processor_id());
C
Christoph Lameter 已提交
1688 1689 1690 1691
}

static void flush_all(struct kmem_cache *s)
{
1692
	on_each_cpu(flush_cpu_slab, s, 1);
C
Christoph Lameter 已提交
1693 1694
}

1695 1696 1697 1698 1699 1700 1701
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
static inline int node_match(struct kmem_cache_cpu *c, int node)
{
#ifdef CONFIG_NUMA
1702
	if (node != NUMA_NO_NODE && c->node != node)
1703 1704 1705 1706 1707
		return 0;
#endif
	return 1;
}

P
Pekka Enberg 已提交
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726
static int count_free(struct page *page)
{
	return page->objects - page->inuse;
}

static unsigned long count_partial(struct kmem_cache_node *n,
					int (*get_count)(struct page *))
{
	unsigned long flags;
	unsigned long x = 0;
	struct page *page;

	spin_lock_irqsave(&n->list_lock, flags);
	list_for_each_entry(page, &n->partial, lru)
		x += get_count(page);
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}

1727 1728 1729 1730 1731 1732 1733 1734 1735
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
#ifdef CONFIG_SLUB_DEBUG
	return atomic_long_read(&n->total_objects);
#else
	return 0;
#endif
}

P
Pekka Enberg 已提交
1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
	int node;

	printk(KERN_WARNING
		"SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
		nid, gfpflags);
	printk(KERN_WARNING "  cache: %s, object size: %d, buffer size: %d, "
		"default order: %d, min order: %d\n", s->name, s->objsize,
		s->size, oo_order(s->oo), oo_order(s->min));

1748 1749 1750 1751
	if (oo_order(s->min) > get_order(s->objsize))
		printk(KERN_WARNING "  %s debugging increased min order, use "
		       "slub_debug=O to disable.\n", s->name);

P
Pekka Enberg 已提交
1752 1753 1754 1755 1756 1757 1758 1759 1760
	for_each_online_node(node) {
		struct kmem_cache_node *n = get_node(s, node);
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

		if (!n)
			continue;

1761 1762 1763
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
P
Pekka Enberg 已提交
1764 1765 1766 1767 1768 1769 1770

		printk(KERN_WARNING
			"  node %d: slabs: %ld, objs: %ld, free: %ld\n",
			node, nr_slabs, nr_objs, nr_free);
	}
}

C
Christoph Lameter 已提交
1771
/*
1772 1773 1774 1775
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Interrupts are disabled.
C
Christoph Lameter 已提交
1776
 *
1777 1778 1779
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
C
Christoph Lameter 已提交
1780
 *
1781 1782 1783
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
C
Christoph Lameter 已提交
1784
 *
1785
 * And if we were unable to get a new slab from the partial slab lists then
C
Christoph Lameter 已提交
1786 1787
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
C
Christoph Lameter 已提交
1788
 */
1789 1790
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1791 1792
{
	void **object;
1793
	struct page *page;
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPT
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif
#endif
C
Christoph Lameter 已提交
1807

1808 1809 1810
	/* We handle __GFP_ZERO in the caller */
	gfpflags &= ~__GFP_ZERO;

1811 1812
	page = c->page;
	if (!page)
C
Christoph Lameter 已提交
1813 1814
		goto new_slab;

1815
	slab_lock(page);
1816
	if (unlikely(!node_match(c, node)))
C
Christoph Lameter 已提交
1817
		goto another_slab;
C
Christoph Lameter 已提交
1818

1819
	stat(s, ALLOC_REFILL);
C
Christoph Lameter 已提交
1820

1821
load_freelist:
1822
	object = page->freelist;
1823
	if (unlikely(!object))
C
Christoph Lameter 已提交
1824
		goto another_slab;
1825
	if (kmem_cache_debug(s))
C
Christoph Lameter 已提交
1826 1827
		goto debug;

1828
	c->freelist = get_freepointer(s, object);
1829 1830 1831
	page->inuse = page->objects;
	page->freelist = NULL;

1832
unlock_out:
1833
	slab_unlock(page);
1834 1835 1836 1837
#ifdef CONFIG_CMPXCHG_LOCAL
	c->tid = next_tid(c->tid);
	local_irq_restore(flags);
#endif
1838
	stat(s, ALLOC_SLOWPATH);
C
Christoph Lameter 已提交
1839 1840 1841
	return object;

another_slab:
1842
	deactivate_slab(s, c);
C
Christoph Lameter 已提交
1843 1844

new_slab:
1845 1846
	page = get_partial(s, gfpflags, node);
	if (page) {
1847
		stat(s, ALLOC_FROM_PARTIAL);
1848 1849 1850
load_from_page:
		c->node = page_to_nid(page);
		c->page = page;
1851
		goto load_freelist;
C
Christoph Lameter 已提交
1852 1853
	}

1854
	gfpflags &= gfp_allowed_mask;
1855 1856 1857
	if (gfpflags & __GFP_WAIT)
		local_irq_enable();

1858
	page = new_slab(s, gfpflags, node);
1859 1860 1861 1862

	if (gfpflags & __GFP_WAIT)
		local_irq_disable();

1863
	if (page) {
1864
		c = __this_cpu_ptr(s->cpu_slab);
1865
		stat(s, ALLOC_SLAB);
1866
		if (c->page)
1867
			flush_slab(s, c);
1868 1869 1870

		slab_lock(page);
		__SetPageSlubFrozen(page);
1871 1872

		goto load_from_page;
C
Christoph Lameter 已提交
1873
	}
1874 1875
	if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
		slab_out_of_memory(s, gfpflags, node);
1876 1877 1878
#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);
#endif
1879
	return NULL;
C
Christoph Lameter 已提交
1880
debug:
1881
	if (!alloc_debug_processing(s, page, object, addr))
C
Christoph Lameter 已提交
1882
		goto another_slab;
1883

1884 1885
	page->inuse++;
	page->freelist = get_freepointer(s, object);
1886
	c->node = NUMA_NO_NODE;
1887
	goto unlock_out;
1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
}

/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
P
Pekka Enberg 已提交
1900
static __always_inline void *slab_alloc(struct kmem_cache *s,
1901
		gfp_t gfpflags, int node, unsigned long addr)
1902 1903
{
	void **object;
1904
	struct kmem_cache_cpu *c;
1905 1906 1907
#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long tid;
#else
1908
	unsigned long flags;
1909
#endif
1910

1911
	if (slab_pre_alloc_hook(s, gfpflags))
A
Akinobu Mita 已提交
1912
		return NULL;
1913

1914
#ifndef CONFIG_CMPXCHG_LOCAL
1915
	local_irq_save(flags);
1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
#else
redo:
#endif

	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
	 */
1926
	c = __this_cpu_ptr(s->cpu_slab);
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938

#ifdef CONFIG_CMPXCHG_LOCAL
	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */
	tid = c->tid;
	barrier();
#endif

1939 1940
	object = c->freelist;
	if (unlikely(!object || !node_match(c, node)))
1941

1942
		object = __slab_alloc(s, gfpflags, node, addr, c);
1943 1944

	else {
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
#ifdef CONFIG_CMPXCHG_LOCAL
		/*
		 * The cmpxchg will only match if there was no additonal
		 * operation and if we are on the right processor.
		 *
		 * The cmpxchg does the following atomically (without lock semantics!)
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
		 * Since this is without lock semantics the protection is only against
		 * code executing on this cpu *not* from access by other cpus.
		 */
		if (unlikely(!this_cpu_cmpxchg_double(
				s->cpu_slab->freelist, s->cpu_slab->tid,
				object, tid,
				get_freepointer(s, object), next_tid(tid)))) {

			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
#else
1967
		c->freelist = get_freepointer(s, object);
1968
#endif
1969
		stat(s, ALLOC_FASTPATH);
1970
	}
1971 1972

#ifndef CONFIG_CMPXCHG_LOCAL
1973
	local_irq_restore(flags);
1974
#endif
1975

1976
	if (unlikely(gfpflags & __GFP_ZERO) && object)
1977
		memset(object, 0, s->objsize);
1978

1979
	slab_post_alloc_hook(s, gfpflags, object);
V
Vegard Nossum 已提交
1980

1981
	return object;
C
Christoph Lameter 已提交
1982 1983 1984 1985
}

void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
1986
	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
1987

1988
	trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
E
Eduard - Gabriel Munteanu 已提交
1989 1990

	return ret;
C
Christoph Lameter 已提交
1991 1992 1993
}
EXPORT_SYMBOL(kmem_cache_alloc);

1994
#ifdef CONFIG_TRACING
1995 1996 1997 1998 1999 2000 2001 2002 2003
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
	return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);

void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
E
Eduard - Gabriel Munteanu 已提交
2004
{
2005 2006 2007
	void *ret = kmalloc_order(size, flags, order);
	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
2008
}
2009
EXPORT_SYMBOL(kmalloc_order_trace);
E
Eduard - Gabriel Munteanu 已提交
2010 2011
#endif

C
Christoph Lameter 已提交
2012 2013 2014
#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
E
Eduard - Gabriel Munteanu 已提交
2015 2016
	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);

2017 2018
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
				    s->objsize, s->size, gfpflags, node);
E
Eduard - Gabriel Munteanu 已提交
2019 2020

	return ret;
C
Christoph Lameter 已提交
2021 2022 2023
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

2024
#ifdef CONFIG_TRACING
2025
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
E
Eduard - Gabriel Munteanu 已提交
2026
				    gfp_t gfpflags,
2027
				    int node, size_t size)
E
Eduard - Gabriel Munteanu 已提交
2028
{
2029 2030 2031 2032 2033
	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);

	trace_kmalloc_node(_RET_IP_, ret,
			   size, s->size, gfpflags, node);
	return ret;
E
Eduard - Gabriel Munteanu 已提交
2034
}
2035
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
E
Eduard - Gabriel Munteanu 已提交
2036
#endif
2037
#endif
E
Eduard - Gabriel Munteanu 已提交
2038

C
Christoph Lameter 已提交
2039
/*
2040 2041
 * Slow patch handling. This may still be called frequently since objects
 * have a longer lifetime than the cpu slabs in most processing loads.
C
Christoph Lameter 已提交
2042
 *
2043 2044 2045
 * So we still attempt to reduce cache line usage. Just take the slab
 * lock and free the item. If there is no additional partial page
 * handling required then we can return immediately.
C
Christoph Lameter 已提交
2046
 */
2047
static void __slab_free(struct kmem_cache *s, struct page *page,
2048
			void *x, unsigned long addr)
C
Christoph Lameter 已提交
2049 2050 2051
{
	void *prior;
	void **object = (void *)x;
2052 2053
#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long flags;
C
Christoph Lameter 已提交
2054

2055 2056
	local_irq_save(flags);
#endif
C
Christoph Lameter 已提交
2057
	slab_lock(page);
2058
	stat(s, FREE_SLOWPATH);
C
Christoph Lameter 已提交
2059

2060 2061
	if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
		goto out_unlock;
C
Christoph Lameter 已提交
2062

2063 2064
	prior = page->freelist;
	set_freepointer(s, object, prior);
C
Christoph Lameter 已提交
2065 2066 2067
	page->freelist = object;
	page->inuse--;

2068
	if (unlikely(PageSlubFrozen(page))) {
2069
		stat(s, FREE_FROZEN);
C
Christoph Lameter 已提交
2070
		goto out_unlock;
2071
	}
C
Christoph Lameter 已提交
2072 2073 2074 2075 2076

	if (unlikely(!page->inuse))
		goto slab_empty;

	/*
C
Christoph Lameter 已提交
2077
	 * Objects left in the slab. If it was not on the partial list before
C
Christoph Lameter 已提交
2078 2079
	 * then add it.
	 */
2080
	if (unlikely(!prior)) {
2081
		add_partial(get_node(s, page_to_nid(page)), page, 1);
2082
		stat(s, FREE_ADD_PARTIAL);
2083
	}
C
Christoph Lameter 已提交
2084 2085 2086

out_unlock:
	slab_unlock(page);
2087 2088 2089
#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);
#endif
C
Christoph Lameter 已提交
2090 2091 2092
	return;

slab_empty:
2093
	if (prior) {
C
Christoph Lameter 已提交
2094
		/*
C
Christoph Lameter 已提交
2095
		 * Slab still on the partial list.
C
Christoph Lameter 已提交
2096 2097
		 */
		remove_partial(s, page);
2098
		stat(s, FREE_REMOVE_PARTIAL);
2099
	}
C
Christoph Lameter 已提交
2100
	slab_unlock(page);
2101 2102 2103
#ifdef CONFIG_CMPXCHG_LOCAL
	local_irq_restore(flags);
#endif
2104
	stat(s, FREE_SLAB);
C
Christoph Lameter 已提交
2105 2106 2107
	discard_slab(s, page);
}

2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
 */
P
Pekka Enberg 已提交
2119
static __always_inline void slab_free(struct kmem_cache *s,
2120
			struct page *page, void *x, unsigned long addr)
2121 2122
{
	void **object = (void *)x;
2123
	struct kmem_cache_cpu *c;
2124 2125 2126
#ifdef CONFIG_CMPXCHG_LOCAL
	unsigned long tid;
#else
2127
	unsigned long flags;
2128
#endif
2129

2130 2131
	slab_free_hook(s, x);

2132
#ifndef CONFIG_CMPXCHG_LOCAL
2133
	local_irq_save(flags);
2134

2135
#else
2136
redo:
2137 2138
#endif

2139 2140 2141 2142 2143 2144
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
	 * during the cmpxchg then the free will succedd.
	 */
2145
	c = __this_cpu_ptr(s->cpu_slab);
2146

2147 2148 2149 2150
#ifdef CONFIG_CMPXCHG_LOCAL
	tid = c->tid;
	barrier();
#endif
2151

2152
	if (likely(page == c->page && c->node != NUMA_NO_NODE)) {
2153
		set_freepointer(s, object, c->freelist);
2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164

#ifdef CONFIG_CMPXCHG_LOCAL
		if (unlikely(!this_cpu_cmpxchg_double(
				s->cpu_slab->freelist, s->cpu_slab->tid,
				c->freelist, tid,
				object, next_tid(tid)))) {

			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
#else
2165
		c->freelist = object;
2166
#endif
2167
		stat(s, FREE_FASTPATH);
2168
	} else
2169
		__slab_free(s, page, x, addr);
2170

2171
#ifndef CONFIG_CMPXCHG_LOCAL
2172
	local_irq_restore(flags);
2173
#endif
2174 2175
}

C
Christoph Lameter 已提交
2176 2177
void kmem_cache_free(struct kmem_cache *s, void *x)
{
C
Christoph Lameter 已提交
2178
	struct page *page;
C
Christoph Lameter 已提交
2179

2180
	page = virt_to_head_page(x);
C
Christoph Lameter 已提交
2181

2182
	slab_free(s, page, x, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2183

2184
	trace_kmem_cache_free(_RET_IP_, x);
C
Christoph Lameter 已提交
2185 2186 2187 2188
}
EXPORT_SYMBOL(kmem_cache_free);

/*
C
Christoph Lameter 已提交
2189 2190 2191 2192
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
C
Christoph Lameter 已提交
2193 2194 2195 2196
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
C
Christoph Lameter 已提交
2197
 * must be moved on and off the partial lists and is therefore a factor in
C
Christoph Lameter 已提交
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
 * locking overhead.
 */

/*
 * Mininum / Maximum order of slab pages. This influences locking overhead
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
static int slub_min_order;
2208
static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
2209
static int slub_min_objects;
C
Christoph Lameter 已提交
2210 2211 2212

/*
 * Merge control. If this is set then no merging of slab caches will occur.
C
Christoph Lameter 已提交
2213
 * (Could be removed. This was introduced to pacify the merge skeptics.)
C
Christoph Lameter 已提交
2214 2215 2216 2217 2218 2219
 */
static int slub_nomerge;

/*
 * Calculate the order of allocation given an slab object size.
 *
C
Christoph Lameter 已提交
2220 2221 2222 2223
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
C
Christoph Lameter 已提交
2224
 * unused space left. We go to a higher order if more than 1/16th of the slab
C
Christoph Lameter 已提交
2225 2226 2227 2228 2229 2230
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
C
Christoph Lameter 已提交
2231
 *
C
Christoph Lameter 已提交
2232 2233 2234 2235
 * slub_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slub_max_order then
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
C
Christoph Lameter 已提交
2236
 *
C
Christoph Lameter 已提交
2237 2238 2239 2240
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
 * requested a higher mininum order then we start with that one instead of
 * the smallest order which will fit the object.
C
Christoph Lameter 已提交
2241
 */
2242
static inline int slab_order(int size, int min_objects,
2243
				int max_order, int fract_leftover, int reserved)
C
Christoph Lameter 已提交
2244 2245 2246
{
	int order;
	int rem;
2247
	int min_order = slub_min_order;
C
Christoph Lameter 已提交
2248

2249
	if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
2250
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
2251

2252
	for (order = max(min_order,
2253 2254
				fls(min_objects * size - 1) - PAGE_SHIFT);
			order <= max_order; order++) {
C
Christoph Lameter 已提交
2255

2256
		unsigned long slab_size = PAGE_SIZE << order;
C
Christoph Lameter 已提交
2257

2258
		if (slab_size < min_objects * size + reserved)
C
Christoph Lameter 已提交
2259 2260
			continue;

2261
		rem = (slab_size - reserved) % size;
C
Christoph Lameter 已提交
2262

2263
		if (rem <= slab_size / fract_leftover)
C
Christoph Lameter 已提交
2264 2265 2266
			break;

	}
C
Christoph Lameter 已提交
2267

C
Christoph Lameter 已提交
2268 2269 2270
	return order;
}

2271
static inline int calculate_order(int size, int reserved)
2272 2273 2274 2275
{
	int order;
	int min_objects;
	int fraction;
2276
	int max_objects;
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 *
	 * First we reduce the acceptable waste in a slab. Then
	 * we reduce the minimum objects required in a slab.
	 */
	min_objects = slub_min_objects;
2287 2288
	if (!min_objects)
		min_objects = 4 * (fls(nr_cpu_ids) + 1);
2289
	max_objects = order_objects(slub_max_order, size, reserved);
2290 2291
	min_objects = min(min_objects, max_objects);

2292
	while (min_objects > 1) {
C
Christoph Lameter 已提交
2293
		fraction = 16;
2294 2295
		while (fraction >= 4) {
			order = slab_order(size, min_objects,
2296
					slub_max_order, fraction, reserved);
2297 2298 2299 2300
			if (order <= slub_max_order)
				return order;
			fraction /= 2;
		}
2301
		min_objects--;
2302 2303 2304 2305 2306 2307
	}

	/*
	 * We were unable to place multiple objects in a slab. Now
	 * lets see if we can place a single object there.
	 */
2308
	order = slab_order(size, 1, slub_max_order, 1, reserved);
2309 2310 2311 2312 2313 2314
	if (order <= slub_max_order)
		return order;

	/*
	 * Doh this slab cannot be placed using slub_max_order.
	 */
2315
	order = slab_order(size, 1, MAX_ORDER, 1, reserved);
D
David Rientjes 已提交
2316
	if (order < MAX_ORDER)
2317 2318 2319 2320
		return order;
	return -ENOSYS;
}

C
Christoph Lameter 已提交
2321
/*
C
Christoph Lameter 已提交
2322
 * Figure out what the alignment of the objects will be.
C
Christoph Lameter 已提交
2323 2324 2325 2326 2327
 */
static unsigned long calculate_alignment(unsigned long flags,
		unsigned long align, unsigned long size)
{
	/*
C
Christoph Lameter 已提交
2328 2329
	 * If the user wants hardware cache aligned objects then follow that
	 * suggestion if the object is sufficiently large.
C
Christoph Lameter 已提交
2330
	 *
C
Christoph Lameter 已提交
2331 2332
	 * The hardware cache alignment cannot override the specified
	 * alignment though. If that is greater then use it.
C
Christoph Lameter 已提交
2333
	 */
2334 2335 2336 2337 2338 2339
	if (flags & SLAB_HWCACHE_ALIGN) {
		unsigned long ralign = cache_line_size();
		while (size <= ralign / 2)
			ralign /= 2;
		align = max(align, ralign);
	}
C
Christoph Lameter 已提交
2340 2341

	if (align < ARCH_SLAB_MINALIGN)
2342
		align = ARCH_SLAB_MINALIGN;
C
Christoph Lameter 已提交
2343 2344 2345 2346

	return ALIGN(align, sizeof(void *));
}

2347 2348
static void
init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
C
Christoph Lameter 已提交
2349 2350 2351 2352
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
2353
#ifdef CONFIG_SLUB_DEBUG
2354
	atomic_long_set(&n->nr_slabs, 0);
2355
	atomic_long_set(&n->total_objects, 0);
2356
	INIT_LIST_HEAD(&n->full);
2357
#endif
C
Christoph Lameter 已提交
2358 2359
}

2360
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
2361
{
2362 2363
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
			SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
2364

2365 2366 2367 2368 2369 2370 2371 2372
#ifdef CONFIG_CMPXCHG_LOCAL
	/*
	 * Must align to double word boundary for the double cmpxchg instructions
	 * to work.
	 */
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 2 * sizeof(void *));
#else
	/* Regular alignment is sufficient */
2373
	s->cpu_slab = alloc_percpu(struct kmem_cache_cpu);
2374 2375 2376 2377 2378 2379
#endif

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
2380

2381
	return 1;
2382 2383
}

2384 2385
static struct kmem_cache *kmem_cache_node;

C
Christoph Lameter 已提交
2386 2387 2388 2389 2390 2391
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
 * Note that this function only works on the kmalloc_node_cache
2392 2393
 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
 * memory on a fresh node that has no slab structures yet.
C
Christoph Lameter 已提交
2394
 */
2395
static void early_kmem_cache_node_alloc(int node)
C
Christoph Lameter 已提交
2396 2397 2398
{
	struct page *page;
	struct kmem_cache_node *n;
R
root 已提交
2399
	unsigned long flags;
C
Christoph Lameter 已提交
2400

2401
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
C
Christoph Lameter 已提交
2402

2403
	page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
C
Christoph Lameter 已提交
2404 2405

	BUG_ON(!page);
2406 2407 2408 2409 2410 2411 2412
	if (page_to_nid(page) != node) {
		printk(KERN_ERR "SLUB: Unable to allocate memory from "
				"node %d\n", node);
		printk(KERN_ERR "SLUB: Allocating a useless per node structure "
				"in order to be able to continue\n");
	}

C
Christoph Lameter 已提交
2413 2414
	n = page->freelist;
	BUG_ON(!n);
2415
	page->freelist = get_freepointer(kmem_cache_node, n);
C
Christoph Lameter 已提交
2416
	page->inuse++;
2417
	kmem_cache_node->node[node] = n;
2418
#ifdef CONFIG_SLUB_DEBUG
2419
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
2420
	init_tracking(kmem_cache_node, n);
2421
#endif
2422 2423
	init_kmem_cache_node(n, kmem_cache_node);
	inc_slabs_node(kmem_cache_node, node, page->objects);
C
Christoph Lameter 已提交
2424

R
root 已提交
2425 2426 2427 2428 2429 2430
	/*
	 * lockdep requires consistent irq usage for each lock
	 * so even though there cannot be a race this early in
	 * the boot sequence, we still disable irqs.
	 */
	local_irq_save(flags);
2431
	add_partial(n, page, 0);
R
root 已提交
2432
	local_irq_restore(flags);
C
Christoph Lameter 已提交
2433 2434 2435 2436 2437 2438
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;

C
Christoph Lameter 已提交
2439
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
2440
		struct kmem_cache_node *n = s->node[node];
2441

2442
		if (n)
2443 2444
			kmem_cache_free(kmem_cache_node, n);

C
Christoph Lameter 已提交
2445 2446 2447 2448
		s->node[node] = NULL;
	}
}

2449
static int init_kmem_cache_nodes(struct kmem_cache *s)
C
Christoph Lameter 已提交
2450 2451 2452
{
	int node;

C
Christoph Lameter 已提交
2453
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
2454 2455
		struct kmem_cache_node *n;

2456
		if (slab_state == DOWN) {
2457
			early_kmem_cache_node_alloc(node);
2458 2459
			continue;
		}
2460
		n = kmem_cache_alloc_node(kmem_cache_node,
2461
						GFP_KERNEL, node);
C
Christoph Lameter 已提交
2462

2463 2464 2465
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
C
Christoph Lameter 已提交
2466
		}
2467

C
Christoph Lameter 已提交
2468
		s->node[node] = n;
2469
		init_kmem_cache_node(n, s);
C
Christoph Lameter 已提交
2470 2471 2472 2473
	}
	return 1;
}

2474
static void set_min_partial(struct kmem_cache *s, unsigned long min)
2475 2476 2477 2478 2479 2480 2481 2482
{
	if (min < MIN_PARTIAL)
		min = MIN_PARTIAL;
	else if (min > MAX_PARTIAL)
		min = MAX_PARTIAL;
	s->min_partial = min;
}

C
Christoph Lameter 已提交
2483 2484 2485 2486
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
2487
static int calculate_sizes(struct kmem_cache *s, int forced_order)
C
Christoph Lameter 已提交
2488 2489 2490 2491
{
	unsigned long flags = s->flags;
	unsigned long size = s->objsize;
	unsigned long align = s->align;
2492
	int order;
C
Christoph Lameter 已提交
2493

2494 2495 2496 2497 2498 2499 2500 2501
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
2502 2503 2504 2505 2506 2507
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
	if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
2508
			!s->ctor)
C
Christoph Lameter 已提交
2509 2510 2511 2512 2513 2514
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
C
Christoph Lameter 已提交
2515
	 * If we are Redzoning then check if there is some space between the
C
Christoph Lameter 已提交
2516
	 * end of the object and the free pointer. If not then add an
C
Christoph Lameter 已提交
2517
	 * additional word to have some bytes to store Redzone information.
C
Christoph Lameter 已提交
2518 2519 2520
	 */
	if ((flags & SLAB_RED_ZONE) && size == s->objsize)
		size += sizeof(void *);
C
Christoph Lameter 已提交
2521
#endif
C
Christoph Lameter 已提交
2522 2523

	/*
C
Christoph Lameter 已提交
2524 2525
	 * With that we have determined the number of bytes in actual use
	 * by the object. This is the potential offset to the free pointer.
C
Christoph Lameter 已提交
2526 2527 2528 2529
	 */
	s->inuse = size;

	if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
2530
		s->ctor)) {
C
Christoph Lameter 已提交
2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
		 * destructor or are poisoning the objects.
		 */
		s->offset = size;
		size += sizeof(void *);
	}

2543
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
2544 2545 2546 2547 2548 2549 2550
	if (flags & SLAB_STORE_USER)
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);

2551
	if (flags & SLAB_RED_ZONE)
C
Christoph Lameter 已提交
2552 2553 2554 2555
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
2556
		 * corrupted if a user writes before the start
C
Christoph Lameter 已提交
2557 2558 2559
		 * of the object.
		 */
		size += sizeof(void *);
C
Christoph Lameter 已提交
2560
#endif
C
Christoph Lameter 已提交
2561

C
Christoph Lameter 已提交
2562 2563
	/*
	 * Determine the alignment based on various parameters that the
2564 2565
	 * user specified and the dynamic determination of cache line size
	 * on bootup.
C
Christoph Lameter 已提交
2566 2567
	 */
	align = calculate_alignment(flags, align, s->objsize);
2568
	s->align = align;
C
Christoph Lameter 已提交
2569 2570 2571 2572 2573 2574 2575 2576

	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
	size = ALIGN(size, align);
	s->size = size;
2577 2578 2579
	if (forced_order >= 0)
		order = forced_order;
	else
2580
		order = calculate_order(size, s->reserved);
C
Christoph Lameter 已提交
2581

2582
	if (order < 0)
C
Christoph Lameter 已提交
2583 2584
		return 0;

2585
	s->allocflags = 0;
2586
	if (order)
2587 2588 2589 2590 2591 2592 2593 2594
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
		s->allocflags |= SLUB_DMA;

	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

C
Christoph Lameter 已提交
2595 2596 2597
	/*
	 * Determine the number of objects per slab
	 */
2598 2599
	s->oo = oo_make(order, size, s->reserved);
	s->min = oo_make(get_order(size), size, s->reserved);
2600 2601
	if (oo_objects(s->oo) > oo_objects(s->max))
		s->max = s->oo;
C
Christoph Lameter 已提交
2602

2603
	return !!oo_objects(s->oo);
C
Christoph Lameter 已提交
2604 2605 2606

}

2607
static int kmem_cache_open(struct kmem_cache *s,
C
Christoph Lameter 已提交
2608 2609
		const char *name, size_t size,
		size_t align, unsigned long flags,
2610
		void (*ctor)(void *))
C
Christoph Lameter 已提交
2611 2612 2613 2614 2615 2616
{
	memset(s, 0, kmem_size);
	s->name = name;
	s->ctor = ctor;
	s->objsize = size;
	s->align = align;
2617
	s->flags = kmem_cache_flags(size, flags, name, ctor);
2618
	s->reserved = 0;
C
Christoph Lameter 已提交
2619

2620 2621
	if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
		s->reserved = sizeof(struct rcu_head);
C
Christoph Lameter 已提交
2622

2623
	if (!calculate_sizes(s, -1))
C
Christoph Lameter 已提交
2624
		goto error;
2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
		if (get_order(s->size) > get_order(s->objsize)) {
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
			if (!calculate_sizes(s, -1))
				goto error;
		}
	}
C
Christoph Lameter 已提交
2637

2638 2639 2640 2641
	/*
	 * The larger the object size is, the more pages we want on the partial
	 * list to avoid pounding the page allocator excessively.
	 */
2642
	set_min_partial(s, ilog2(s->size));
C
Christoph Lameter 已提交
2643 2644
	s->refcount = 1;
#ifdef CONFIG_NUMA
2645
	s->remote_node_defrag_ratio = 1000;
C
Christoph Lameter 已提交
2646
#endif
2647
	if (!init_kmem_cache_nodes(s))
2648
		goto error;
C
Christoph Lameter 已提交
2649

2650
	if (alloc_kmem_cache_cpus(s))
C
Christoph Lameter 已提交
2651
		return 1;
2652

2653
	free_kmem_cache_nodes(s);
C
Christoph Lameter 已提交
2654 2655 2656 2657
error:
	if (flags & SLAB_PANIC)
		panic("Cannot create slab %s size=%lu realsize=%u "
			"order=%u offset=%u flags=%lx\n",
2658
			s->name, (unsigned long)size, s->size, oo_order(s->oo),
C
Christoph Lameter 已提交
2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671
			s->offset, flags);
	return 0;
}

/*
 * Determine the size of a slab object
 */
unsigned int kmem_cache_size(struct kmem_cache *s)
{
	return s->objsize;
}
EXPORT_SYMBOL(kmem_cache_size);

2672 2673 2674 2675 2676 2677
static void list_slab_objects(struct kmem_cache *s, struct page *page,
							const char *text)
{
#ifdef CONFIG_SLUB_DEBUG
	void *addr = page_address(page);
	void *p;
N
Namhyung Kim 已提交
2678 2679
	unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
				     sizeof(long), GFP_ATOMIC);
E
Eric Dumazet 已提交
2680 2681
	if (!map)
		return;
2682 2683 2684
	slab_err(s, page, "%s", text);
	slab_lock(page);

2685
	get_map(s, page, map);
2686 2687 2688 2689 2690 2691 2692 2693 2694
	for_each_object(p, s, addr, page->objects) {

		if (!test_bit(slab_index(p, s, addr), map)) {
			printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
							p, p - addr);
			print_tracking(s, p);
		}
	}
	slab_unlock(page);
E
Eric Dumazet 已提交
2695
	kfree(map);
2696 2697 2698
#endif
}

C
Christoph Lameter 已提交
2699
/*
C
Christoph Lameter 已提交
2700
 * Attempt to free all partial slabs on a node.
C
Christoph Lameter 已提交
2701
 */
C
Christoph Lameter 已提交
2702
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
C
Christoph Lameter 已提交
2703 2704 2705 2706 2707
{
	unsigned long flags;
	struct page *page, *h;

	spin_lock_irqsave(&n->list_lock, flags);
2708
	list_for_each_entry_safe(page, h, &n->partial, lru) {
C
Christoph Lameter 已提交
2709
		if (!page->inuse) {
2710
			__remove_partial(n, page);
C
Christoph Lameter 已提交
2711
			discard_slab(s, page);
2712 2713 2714
		} else {
			list_slab_objects(s, page,
				"Objects remaining on kmem_cache_close()");
C
Christoph Lameter 已提交
2715
		}
2716
	}
C
Christoph Lameter 已提交
2717 2718 2719 2720
	spin_unlock_irqrestore(&n->list_lock, flags);
}

/*
C
Christoph Lameter 已提交
2721
 * Release all resources used by a slab cache.
C
Christoph Lameter 已提交
2722
 */
2723
static inline int kmem_cache_close(struct kmem_cache *s)
C
Christoph Lameter 已提交
2724 2725 2726 2727
{
	int node;

	flush_all(s);
2728
	free_percpu(s->cpu_slab);
C
Christoph Lameter 已提交
2729
	/* Attempt to free all objects */
C
Christoph Lameter 已提交
2730
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
2731 2732
		struct kmem_cache_node *n = get_node(s, node);

C
Christoph Lameter 已提交
2733 2734
		free_partial(s, n);
		if (n->nr_partial || slabs_node(s, node))
C
Christoph Lameter 已提交
2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750
			return 1;
	}
	free_kmem_cache_nodes(s);
	return 0;
}

/*
 * Close a cache and release the kmem_cache structure
 * (must be used for caches created using kmem_cache_create)
 */
void kmem_cache_destroy(struct kmem_cache *s)
{
	down_write(&slub_lock);
	s->refcount--;
	if (!s->refcount) {
		list_del(&s->list);
2751 2752 2753 2754 2755
		if (kmem_cache_close(s)) {
			printk(KERN_ERR "SLUB %s: %s called for cache that "
				"still has objects.\n", s->name, __func__);
			dump_stack();
		}
2756 2757
		if (s->flags & SLAB_DESTROY_BY_RCU)
			rcu_barrier();
C
Christoph Lameter 已提交
2758
		sysfs_slab_remove(s);
2759 2760
	}
	up_write(&slub_lock);
C
Christoph Lameter 已提交
2761 2762 2763 2764 2765 2766 2767
}
EXPORT_SYMBOL(kmem_cache_destroy);

/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

2768
struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
C
Christoph Lameter 已提交
2769 2770
EXPORT_SYMBOL(kmalloc_caches);

2771 2772
static struct kmem_cache *kmem_cache;

2773
#ifdef CONFIG_ZONE_DMA
2774
static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
2775 2776
#endif

C
Christoph Lameter 已提交
2777 2778
static int __init setup_slub_min_order(char *str)
{
P
Pekka Enberg 已提交
2779
	get_option(&str, &slub_min_order);
C
Christoph Lameter 已提交
2780 2781 2782 2783 2784 2785 2786 2787

	return 1;
}

__setup("slub_min_order=", setup_slub_min_order);

static int __init setup_slub_max_order(char *str)
{
P
Pekka Enberg 已提交
2788
	get_option(&str, &slub_max_order);
D
David Rientjes 已提交
2789
	slub_max_order = min(slub_max_order, MAX_ORDER - 1);
C
Christoph Lameter 已提交
2790 2791 2792 2793 2794 2795 2796 2797

	return 1;
}

__setup("slub_max_order=", setup_slub_max_order);

static int __init setup_slub_min_objects(char *str)
{
P
Pekka Enberg 已提交
2798
	get_option(&str, &slub_min_objects);
C
Christoph Lameter 已提交
2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812

	return 1;
}

__setup("slub_min_objects=", setup_slub_min_objects);

static int __init setup_slub_nomerge(char *str)
{
	slub_nomerge = 1;
	return 1;
}

__setup("slub_nomerge", setup_slub_nomerge);

2813 2814
static struct kmem_cache *__init create_kmalloc_cache(const char *name,
						int size, unsigned int flags)
C
Christoph Lameter 已提交
2815
{
2816 2817 2818 2819
	struct kmem_cache *s;

	s = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);

2820 2821 2822 2823
	/*
	 * This function is called with IRQs disabled during early-boot on
	 * single CPU so there's no need to take slub_lock here.
	 */
2824
	if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
2825
								flags, NULL))
C
Christoph Lameter 已提交
2826 2827 2828
		goto panic;

	list_add(&s->list, &slab_caches);
2829
	return s;
C
Christoph Lameter 已提交
2830 2831 2832

panic:
	panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
2833
	return NULL;
C
Christoph Lameter 已提交
2834 2835
}

2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868
/*
 * Conversion table for small slabs sizes / 8 to the index in the
 * kmalloc array. This is necessary for slabs < 192 since we have non power
 * of two cache sizes there. The size of larger slabs can be determined using
 * fls.
 */
static s8 size_index[24] = {
	3,	/* 8 */
	4,	/* 16 */
	5,	/* 24 */
	5,	/* 32 */
	6,	/* 40 */
	6,	/* 48 */
	6,	/* 56 */
	6,	/* 64 */
	1,	/* 72 */
	1,	/* 80 */
	1,	/* 88 */
	1,	/* 96 */
	7,	/* 104 */
	7,	/* 112 */
	7,	/* 120 */
	7,	/* 128 */
	2,	/* 136 */
	2,	/* 144 */
	2,	/* 152 */
	2,	/* 160 */
	2,	/* 168 */
	2,	/* 176 */
	2,	/* 184 */
	2	/* 192 */
};

2869 2870 2871 2872 2873
static inline int size_index_elem(size_t bytes)
{
	return (bytes - 1) / 8;
}

C
Christoph Lameter 已提交
2874 2875
static struct kmem_cache *get_slab(size_t size, gfp_t flags)
{
2876
	int index;
C
Christoph Lameter 已提交
2877

2878 2879 2880
	if (size <= 192) {
		if (!size)
			return ZERO_SIZE_PTR;
C
Christoph Lameter 已提交
2881

2882
		index = size_index[size_index_elem(size)];
2883
	} else
2884
		index = fls(size - 1);
C
Christoph Lameter 已提交
2885 2886

#ifdef CONFIG_ZONE_DMA
2887
	if (unlikely((flags & SLUB_DMA)))
2888
		return kmalloc_dma_caches[index];
2889

C
Christoph Lameter 已提交
2890
#endif
2891
	return kmalloc_caches[index];
C
Christoph Lameter 已提交
2892 2893 2894 2895
}

void *__kmalloc(size_t size, gfp_t flags)
{
2896
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
2897
	void *ret;
C
Christoph Lameter 已提交
2898

2899
	if (unlikely(size > SLUB_MAX_SIZE))
2900
		return kmalloc_large(size, flags);
2901 2902 2903 2904

	s = get_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
2905 2906
		return s;

2907
	ret = slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2908

2909
	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
E
Eduard - Gabriel Munteanu 已提交
2910 2911

	return ret;
C
Christoph Lameter 已提交
2912 2913 2914
}
EXPORT_SYMBOL(__kmalloc);

2915
#ifdef CONFIG_NUMA
2916 2917
static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
2918
	struct page *page;
2919
	void *ptr = NULL;
2920

2921 2922
	flags |= __GFP_COMP | __GFP_NOTRACK;
	page = alloc_pages_node(node, flags, get_order(size));
2923
	if (page)
2924 2925 2926 2927
		ptr = page_address(page);

	kmemleak_alloc(ptr, size, 1, flags);
	return ptr;
2928 2929
}

C
Christoph Lameter 已提交
2930 2931
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
2932
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
2933
	void *ret;
C
Christoph Lameter 已提交
2934

I
Ingo Molnar 已提交
2935
	if (unlikely(size > SLUB_MAX_SIZE)) {
E
Eduard - Gabriel Munteanu 已提交
2936 2937
		ret = kmalloc_large_node(size, flags, node);

2938 2939 2940
		trace_kmalloc_node(_RET_IP_, ret,
				   size, PAGE_SIZE << get_order(size),
				   flags, node);
E
Eduard - Gabriel Munteanu 已提交
2941 2942 2943

		return ret;
	}
2944 2945 2946 2947

	s = get_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
2948 2949
		return s;

E
Eduard - Gabriel Munteanu 已提交
2950 2951
	ret = slab_alloc(s, flags, node, _RET_IP_);

2952
	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
E
Eduard - Gabriel Munteanu 已提交
2953 2954

	return ret;
C
Christoph Lameter 已提交
2955 2956 2957 2958 2959 2960
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

size_t ksize(const void *object)
{
2961
	struct page *page;
C
Christoph Lameter 已提交
2962

2963
	if (unlikely(object == ZERO_SIZE_PTR))
2964 2965
		return 0;

2966 2967
	page = virt_to_head_page(object);

P
Pekka Enberg 已提交
2968 2969
	if (unlikely(!PageSlab(page))) {
		WARN_ON(!PageCompound(page));
2970
		return PAGE_SIZE << compound_order(page);
P
Pekka Enberg 已提交
2971
	}
C
Christoph Lameter 已提交
2972

2973
	return slab_ksize(page->slab);
C
Christoph Lameter 已提交
2974
}
K
Kirill A. Shutemov 已提交
2975
EXPORT_SYMBOL(ksize);
C
Christoph Lameter 已提交
2976 2977 2978 2979

void kfree(const void *x)
{
	struct page *page;
2980
	void *object = (void *)x;
C
Christoph Lameter 已提交
2981

2982 2983
	trace_kfree(_RET_IP_, x);

2984
	if (unlikely(ZERO_OR_NULL_PTR(x)))
C
Christoph Lameter 已提交
2985 2986
		return;

2987
	page = virt_to_head_page(x);
2988
	if (unlikely(!PageSlab(page))) {
2989
		BUG_ON(!PageCompound(page));
2990
		kmemleak_free(x);
2991 2992 2993
		put_page(page);
		return;
	}
2994
	slab_free(page->slab, page, object, _RET_IP_);
C
Christoph Lameter 已提交
2995 2996 2997
}
EXPORT_SYMBOL(kfree);

2998
/*
C
Christoph Lameter 已提交
2999 3000 3001 3002 3003 3004 3005 3006
 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
 * the remaining slabs by the number of items in use. The slabs with the
 * most items in use come first. New allocations will then fill those up
 * and thus they can be removed from the partial lists.
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
3007 3008 3009 3010 3011 3012 3013 3014
 */
int kmem_cache_shrink(struct kmem_cache *s)
{
	int node;
	int i;
	struct kmem_cache_node *n;
	struct page *page;
	struct page *t;
3015
	int objects = oo_objects(s->max);
3016
	struct list_head *slabs_by_inuse =
3017
		kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
3018 3019 3020 3021 3022 3023
	unsigned long flags;

	if (!slabs_by_inuse)
		return -ENOMEM;

	flush_all(s);
C
Christoph Lameter 已提交
3024
	for_each_node_state(node, N_NORMAL_MEMORY) {
3025 3026 3027 3028 3029
		n = get_node(s, node);

		if (!n->nr_partial)
			continue;

3030
		for (i = 0; i < objects; i++)
3031 3032 3033 3034 3035
			INIT_LIST_HEAD(slabs_by_inuse + i);

		spin_lock_irqsave(&n->list_lock, flags);

		/*
C
Christoph Lameter 已提交
3036
		 * Build lists indexed by the items in use in each slab.
3037
		 *
C
Christoph Lameter 已提交
3038 3039
		 * Note that concurrent frees may occur while we hold the
		 * list_lock. page->inuse here is the upper limit.
3040 3041 3042 3043 3044 3045 3046 3047
		 */
		list_for_each_entry_safe(page, t, &n->partial, lru) {
			if (!page->inuse && slab_trylock(page)) {
				/*
				 * Must hold slab lock here because slab_free
				 * may have freed the last object and be
				 * waiting to release the slab.
				 */
3048
				__remove_partial(n, page);
3049 3050 3051
				slab_unlock(page);
				discard_slab(s, page);
			} else {
3052 3053
				list_move(&page->lru,
				slabs_by_inuse + page->inuse);
3054 3055 3056 3057
			}
		}

		/*
C
Christoph Lameter 已提交
3058 3059
		 * Rebuild the partial list with the slabs filled up most
		 * first and the least used slabs at the end.
3060
		 */
3061
		for (i = objects - 1; i >= 0; i--)
3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
			list_splice(slabs_by_inuse + i, n->partial.prev);

		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	kfree(slabs_by_inuse);
	return 0;
}
EXPORT_SYMBOL(kmem_cache_shrink);

P
Pekka Enberg 已提交
3072
#if defined(CONFIG_MEMORY_HOTPLUG)
3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list)
		kmem_cache_shrink(s);
	up_read(&slub_lock);

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int offline_node;

	offline_node = marg->status_change_nid;

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list) {
		n = get_node(s, offline_node);
		if (n) {
			/*
			 * if n->nr_slabs > 0, slabs still exist on the node
			 * that is going down. We were unable to free them,
3108
			 * and offline_pages() function shouldn't call this
3109 3110
			 * callback. So, we must fail.
			 */
3111
			BUG_ON(slabs_node(s, offline_node));
3112 3113

			s->node[offline_node] = NULL;
3114
			kmem_cache_free(kmem_cache_node, n);
3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135
		}
	}
	up_read(&slub_lock);
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int nid = marg->status_change_nid;
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
3136
	 * We are bringing a node online. No memory is available yet. We must
3137 3138 3139 3140 3141 3142 3143 3144 3145 3146
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
	down_read(&slub_lock);
	list_for_each_entry(s, &slab_caches, list) {
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
3147
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
3148 3149 3150 3151
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
3152
		init_kmem_cache_node(n, s);
3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179
		s->node[nid] = n;
	}
out:
	up_read(&slub_lock);
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
3180 3181 3182 3183
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
3184 3185 3186 3187 3188
	return ret;
}

#endif /* CONFIG_MEMORY_HOTPLUG */

C
Christoph Lameter 已提交
3189 3190 3191 3192
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212
/*
 * Used for early kmem_cache structures that were allocated using
 * the page allocator
 */

static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
{
	int node;

	list_add(&s->list, &slab_caches);
	s->refcount = -1;

	for_each_node_state(node, N_NORMAL_MEMORY) {
		struct kmem_cache_node *n = get_node(s, node);
		struct page *p;

		if (n) {
			list_for_each_entry(p, &n->partial, lru)
				p->slab = s;

L
Li Zefan 已提交
3213
#ifdef CONFIG_SLUB_DEBUG
3214 3215 3216 3217 3218 3219 3220
			list_for_each_entry(p, &n->full, lru)
				p->slab = s;
#endif
		}
	}
}

C
Christoph Lameter 已提交
3221 3222 3223
void __init kmem_cache_init(void)
{
	int i;
3224
	int caches = 0;
3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237
	struct kmem_cache *temp_kmem_cache;
	int order;
	struct kmem_cache *temp_kmem_cache_node;
	unsigned long kmalloc_size;

	kmem_size = offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *);

	/* Allocate two kmem_caches from the page allocator */
	kmalloc_size = ALIGN(kmem_size, cache_line_size());
	order = get_order(2 * kmalloc_size);
	kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order);

C
Christoph Lameter 已提交
3238 3239
	/*
	 * Must first have the slab cache available for the allocations of the
C
Christoph Lameter 已提交
3240
	 * struct kmem_cache_node's. There is special bootstrap code in
C
Christoph Lameter 已提交
3241 3242
	 * kmem_cache_open for slab_state == DOWN.
	 */
3243 3244 3245 3246 3247
	kmem_cache_node = (void *)kmem_cache + kmalloc_size;

	kmem_cache_open(kmem_cache_node, "kmem_cache_node",
		sizeof(struct kmem_cache_node),
		0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
3248

3249
	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
C
Christoph Lameter 已提交
3250 3251 3252 3253

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

3254 3255 3256 3257 3258
	temp_kmem_cache = kmem_cache;
	kmem_cache_open(kmem_cache, "kmem_cache", kmem_size,
		0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
	kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
	memcpy(kmem_cache, temp_kmem_cache, kmem_size);
C
Christoph Lameter 已提交
3259

3260 3261 3262 3263 3264 3265
	/*
	 * Allocate kmem_cache_node properly from the kmem_cache slab.
	 * kmem_cache_node is separately allocated so no need to
	 * update any list pointers.
	 */
	temp_kmem_cache_node = kmem_cache_node;
C
Christoph Lameter 已提交
3266

3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
	kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
	memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);

	kmem_cache_bootstrap_fixup(kmem_cache_node);

	caches++;
	kmem_cache_bootstrap_fixup(kmem_cache);
	caches++;
	/* Free temporary boot structure */
	free_pages((unsigned long)temp_kmem_cache, order);

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
3279 3280 3281 3282

	/*
	 * Patch up the size_index table if we have strange large alignment
	 * requirements for the kmalloc array. This is only the case for
C
Christoph Lameter 已提交
3283
	 * MIPS it seems. The standard arches will not generate any code here.
3284 3285 3286 3287 3288 3289 3290 3291 3292 3293
	 *
	 * Largest permitted alignment is 256 bytes due to the way we
	 * handle the index determination for the smaller caches.
	 *
	 * Make sure that nothing crazy happens if someone starts tinkering
	 * around with ARCH_KMALLOC_MINALIGN
	 */
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
		(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

3294 3295 3296 3297 3298 3299
	for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
		int elem = size_index_elem(i);
		if (elem >= ARRAY_SIZE(size_index))
			break;
		size_index[elem] = KMALLOC_SHIFT_LOW;
	}
3300

3301 3302 3303 3304 3305 3306 3307 3308
	if (KMALLOC_MIN_SIZE == 64) {
		/*
		 * The 96 byte size cache is not used if the alignment
		 * is 64 byte.
		 */
		for (i = 64 + 8; i <= 96; i += 8)
			size_index[size_index_elem(i)] = 7;
	} else if (KMALLOC_MIN_SIZE == 128) {
3309 3310 3311 3312 3313 3314
		/*
		 * The 192 byte sized cache is not used if the alignment
		 * is 128 byte. Redirect kmalloc to use the 256 byte cache
		 * instead.
		 */
		for (i = 128 + 8; i <= 192; i += 8)
3315
			size_index[size_index_elem(i)] = 8;
3316 3317
	}

3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333
	/* Caches that are not of the two-to-the-power-of size */
	if (KMALLOC_MIN_SIZE <= 32) {
		kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
		caches++;
	}

	if (KMALLOC_MIN_SIZE <= 64) {
		kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
		caches++;
	}

	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
		kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
		caches++;
	}

C
Christoph Lameter 已提交
3334 3335 3336
	slab_state = UP;

	/* Provide the correct kmalloc names now that the caches are up */
P
Pekka Enberg 已提交
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346
	if (KMALLOC_MIN_SIZE <= 32) {
		kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
		BUG_ON(!kmalloc_caches[1]->name);
	}

	if (KMALLOC_MIN_SIZE <= 64) {
		kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
		BUG_ON(!kmalloc_caches[2]->name);
	}

3347 3348 3349 3350
	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
		char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);

		BUG_ON(!s);
3351
		kmalloc_caches[i]->name = s;
3352
	}
C
Christoph Lameter 已提交
3353 3354 3355

#ifdef CONFIG_SMP
	register_cpu_notifier(&slab_notifier);
3356
#endif
C
Christoph Lameter 已提交
3357

3358
#ifdef CONFIG_ZONE_DMA
3359 3360
	for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
		struct kmem_cache *s = kmalloc_caches[i];
3361

3362
		if (s && s->size) {
3363 3364 3365 3366
			char *name = kasprintf(GFP_NOWAIT,
				 "dma-kmalloc-%d", s->objsize);

			BUG_ON(!name);
3367 3368
			kmalloc_dma_caches[i] = create_kmalloc_cache(name,
				s->objsize, SLAB_CACHE_DMA);
3369 3370 3371
		}
	}
#endif
I
Ingo Molnar 已提交
3372 3373
	printk(KERN_INFO
		"SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
3374 3375
		" CPUs=%d, Nodes=%d\n",
		caches, cache_line_size(),
C
Christoph Lameter 已提交
3376 3377 3378 3379
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

3380 3381 3382 3383
void __init kmem_cache_init_late(void)
{
}

C
Christoph Lameter 已提交
3384 3385 3386 3387 3388 3389 3390 3391
/*
 * Find a mergeable slab cache
 */
static int slab_unmergeable(struct kmem_cache *s)
{
	if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
		return 1;

3392
	if (s->ctor)
C
Christoph Lameter 已提交
3393 3394
		return 1;

3395 3396 3397 3398 3399 3400
	/*
	 * We may have set a slab to be unmergeable during bootstrap.
	 */
	if (s->refcount < 0)
		return 1;

C
Christoph Lameter 已提交
3401 3402 3403 3404
	return 0;
}

static struct kmem_cache *find_mergeable(size_t size,
3405
		size_t align, unsigned long flags, const char *name,
3406
		void (*ctor)(void *))
C
Christoph Lameter 已提交
3407
{
3408
	struct kmem_cache *s;
C
Christoph Lameter 已提交
3409 3410 3411 3412

	if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
		return NULL;

3413
	if (ctor)
C
Christoph Lameter 已提交
3414 3415 3416 3417 3418
		return NULL;

	size = ALIGN(size, sizeof(void *));
	align = calculate_alignment(flags, align, size);
	size = ALIGN(size, align);
3419
	flags = kmem_cache_flags(size, flags, name, NULL);
C
Christoph Lameter 已提交
3420

3421
	list_for_each_entry(s, &slab_caches, list) {
C
Christoph Lameter 已提交
3422 3423 3424 3425 3426 3427
		if (slab_unmergeable(s))
			continue;

		if (size > s->size)
			continue;

3428
		if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
C
Christoph Lameter 已提交
3429 3430 3431 3432 3433
				continue;
		/*
		 * Check if alignment is compatible.
		 * Courtesy of Adrian Drzewiecki
		 */
P
Pekka Enberg 已提交
3434
		if ((s->size & ~(align - 1)) != s->size)
C
Christoph Lameter 已提交
3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445
			continue;

		if (s->size - size >= sizeof(void *))
			continue;

		return s;
	}
	return NULL;
}

struct kmem_cache *kmem_cache_create(const char *name, size_t size,
3446
		size_t align, unsigned long flags, void (*ctor)(void *))
C
Christoph Lameter 已提交
3447 3448
{
	struct kmem_cache *s;
P
Pekka Enberg 已提交
3449
	char *n;
C
Christoph Lameter 已提交
3450

3451 3452 3453
	if (WARN_ON(!name))
		return NULL;

C
Christoph Lameter 已提交
3454
	down_write(&slub_lock);
3455
	s = find_mergeable(size, align, flags, name, ctor);
C
Christoph Lameter 已提交
3456 3457 3458 3459 3460 3461 3462 3463
	if (s) {
		s->refcount++;
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
		s->objsize = max(s->objsize, (int)size);
		s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
C
Christoph Lameter 已提交
3464

3465 3466
		if (sysfs_slab_alias(s, name)) {
			s->refcount--;
C
Christoph Lameter 已提交
3467
			goto err;
3468
		}
3469
		up_write(&slub_lock);
3470 3471
		return s;
	}
C
Christoph Lameter 已提交
3472

P
Pekka Enberg 已提交
3473 3474 3475 3476
	n = kstrdup(name, GFP_KERNEL);
	if (!n)
		goto err;

3477 3478
	s = kmalloc(kmem_size, GFP_KERNEL);
	if (s) {
P
Pekka Enberg 已提交
3479
		if (kmem_cache_open(s, n,
3480
				size, align, flags, ctor)) {
C
Christoph Lameter 已提交
3481
			list_add(&s->list, &slab_caches);
3482 3483
			if (sysfs_slab_add(s)) {
				list_del(&s->list);
P
Pekka Enberg 已提交
3484
				kfree(n);
3485
				kfree(s);
3486
				goto err;
3487
			}
3488
			up_write(&slub_lock);
3489 3490
			return s;
		}
P
Pekka Enberg 已提交
3491
		kfree(n);
3492
		kfree(s);
C
Christoph Lameter 已提交
3493
	}
3494
err:
C
Christoph Lameter 已提交
3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506
	up_write(&slub_lock);

	if (flags & SLAB_PANIC)
		panic("Cannot create slabcache %s\n", name);
	else
		s = NULL;
	return s;
}
EXPORT_SYMBOL(kmem_cache_create);

#ifdef CONFIG_SMP
/*
C
Christoph Lameter 已提交
3507 3508
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
C
Christoph Lameter 已提交
3509 3510 3511 3512 3513
 */
static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
		unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
3514 3515
	struct kmem_cache *s;
	unsigned long flags;
C
Christoph Lameter 已提交
3516 3517 3518

	switch (action) {
	case CPU_UP_CANCELED:
3519
	case CPU_UP_CANCELED_FROZEN:
C
Christoph Lameter 已提交
3520
	case CPU_DEAD:
3521
	case CPU_DEAD_FROZEN:
3522 3523 3524 3525 3526 3527 3528
		down_read(&slub_lock);
		list_for_each_entry(s, &slab_caches, list) {
			local_irq_save(flags);
			__flush_cpu_slab(s, cpu);
			local_irq_restore(flags);
		}
		up_read(&slub_lock);
C
Christoph Lameter 已提交
3529 3530 3531 3532 3533 3534 3535
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

P
Pekka Enberg 已提交
3536
static struct notifier_block __cpuinitdata slab_notifier = {
I
Ingo Molnar 已提交
3537
	.notifier_call = slab_cpuup_callback
P
Pekka Enberg 已提交
3538
};
C
Christoph Lameter 已提交
3539 3540 3541

#endif

3542
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
C
Christoph Lameter 已提交
3543
{
3544
	struct kmem_cache *s;
3545
	void *ret;
3546

3547
	if (unlikely(size > SLUB_MAX_SIZE))
3548 3549
		return kmalloc_large(size, gfpflags);

3550
	s = get_slab(size, gfpflags);
C
Christoph Lameter 已提交
3551

3552
	if (unlikely(ZERO_OR_NULL_PTR(s)))
3553
		return s;
C
Christoph Lameter 已提交
3554

3555
	ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller);
3556 3557

	/* Honor the call site pointer we recieved. */
3558
	trace_kmalloc(caller, ret, size, s->size, gfpflags);
3559 3560

	return ret;
C
Christoph Lameter 已提交
3561 3562
}

3563
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
3564
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3565
					int node, unsigned long caller)
C
Christoph Lameter 已提交
3566
{
3567
	struct kmem_cache *s;
3568
	void *ret;
3569

3570 3571 3572 3573 3574 3575 3576 3577 3578
	if (unlikely(size > SLUB_MAX_SIZE)) {
		ret = kmalloc_large_node(size, gfpflags, node);

		trace_kmalloc_node(caller, ret,
				   size, PAGE_SIZE << get_order(size),
				   gfpflags, node);

		return ret;
	}
3579

3580
	s = get_slab(size, gfpflags);
C
Christoph Lameter 已提交
3581

3582
	if (unlikely(ZERO_OR_NULL_PTR(s)))
3583
		return s;
C
Christoph Lameter 已提交
3584

3585 3586 3587
	ret = slab_alloc(s, gfpflags, node, caller);

	/* Honor the call site pointer we recieved. */
3588
	trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
3589 3590

	return ret;
C
Christoph Lameter 已提交
3591
}
3592
#endif
C
Christoph Lameter 已提交
3593

3594
#ifdef CONFIG_SYSFS
3595 3596 3597 3598 3599 3600 3601 3602 3603
static int count_inuse(struct page *page)
{
	return page->inuse;
}

static int count_total(struct page *page)
{
	return page->objects;
}
3604
#endif
3605

3606
#ifdef CONFIG_SLUB_DEBUG
3607 3608
static int validate_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
3609 3610
{
	void *p;
3611
	void *addr = page_address(page);
3612 3613 3614 3615 3616 3617

	if (!check_slab(s, page) ||
			!on_freelist(s, page, NULL))
		return 0;

	/* Now we know that a valid freelist exists */
3618
	bitmap_zero(map, page->objects);
3619

3620 3621 3622 3623 3624
	get_map(s, page, map);
	for_each_object(p, s, addr, page->objects) {
		if (test_bit(slab_index(p, s, addr), map))
			if (!check_object(s, page, p, SLUB_RED_INACTIVE))
				return 0;
3625 3626
	}

3627
	for_each_object(p, s, addr, page->objects)
3628
		if (!test_bit(slab_index(p, s, addr), map))
3629
			if (!check_object(s, page, p, SLUB_RED_ACTIVE))
3630 3631 3632 3633
				return 0;
	return 1;
}

3634 3635
static void validate_slab_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
3636 3637
{
	if (slab_trylock(page)) {
3638
		validate_slab(s, page, map);
3639 3640 3641 3642 3643 3644
		slab_unlock(page);
	} else
		printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
			s->name, page);
}

3645 3646
static int validate_slab_node(struct kmem_cache *s,
		struct kmem_cache_node *n, unsigned long *map)
3647 3648 3649 3650 3651 3652 3653 3654
{
	unsigned long count = 0;
	struct page *page;
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

	list_for_each_entry(page, &n->partial, lru) {
3655
		validate_slab_slab(s, page, map);
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665
		count++;
	}
	if (count != n->nr_partial)
		printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
			"counter=%ld\n", s->name, count, n->nr_partial);

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

	list_for_each_entry(page, &n->full, lru) {
3666
		validate_slab_slab(s, page, map);
3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678
		count++;
	}
	if (count != atomic_long_read(&n->nr_slabs))
		printk(KERN_ERR "SLUB: %s %ld slabs counted but "
			"counter=%ld\n", s->name, count,
			atomic_long_read(&n->nr_slabs));

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

3679
static long validate_slab_cache(struct kmem_cache *s)
3680 3681 3682
{
	int node;
	unsigned long count = 0;
3683
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
3684 3685 3686 3687
				sizeof(unsigned long), GFP_KERNEL);

	if (!map)
		return -ENOMEM;
3688 3689

	flush_all(s);
C
Christoph Lameter 已提交
3690
	for_each_node_state(node, N_NORMAL_MEMORY) {
3691 3692
		struct kmem_cache_node *n = get_node(s, node);

3693
		count += validate_slab_node(s, n, map);
3694
	}
3695
	kfree(map);
3696 3697
	return count;
}
3698
/*
C
Christoph Lameter 已提交
3699
 * Generate lists of code addresses where slabcache objects are allocated
3700 3701 3702 3703 3704
 * and freed.
 */

struct location {
	unsigned long count;
3705
	unsigned long addr;
3706 3707 3708 3709 3710
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
R
Rusty Russell 已提交
3711
	DECLARE_BITMAP(cpus, NR_CPUS);
3712
	nodemask_t nodes;
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
};

static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

3728
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
3729 3730 3731 3732 3733 3734
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

3735
	l = (void *)__get_free_pages(flags, order);
3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
3749
				const struct track *track)
3750 3751 3752
{
	long start, end, pos;
	struct location *l;
3753
	unsigned long caddr;
3754
	unsigned long age = jiffies - track->when;
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769

	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

		caddr = t->loc[pos].addr;
3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785
		if (track->addr == caddr) {

			l = &t->loc[pos];
			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

R
Rusty Russell 已提交
3786 3787
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
3788 3789
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
3790 3791 3792
			return 1;
		}

3793
		if (track->addr < caddr)
3794 3795 3796 3797 3798 3799
			end = pos;
		else
			start = pos;
	}

	/*
C
Christoph Lameter 已提交
3800
	 * Not found. Insert new tracking element.
3801
	 */
3802
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
3803 3804 3805 3806 3807 3808 3809 3810
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
3811 3812 3813 3814 3815 3816
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
R
Rusty Russell 已提交
3817 3818
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
3819 3820
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
3821 3822 3823 3824
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
E
Eric Dumazet 已提交
3825
		struct page *page, enum track_item alloc,
N
Namhyung Kim 已提交
3826
		unsigned long *map)
3827
{
3828
	void *addr = page_address(page);
3829 3830
	void *p;

3831
	bitmap_zero(map, page->objects);
3832
	get_map(s, page, map);
3833

3834
	for_each_object(p, s, addr, page->objects)
3835 3836
		if (!test_bit(slab_index(p, s, addr), map))
			add_location(t, s, get_track(s, p, alloc));
3837 3838 3839 3840 3841
}

static int list_locations(struct kmem_cache *s, char *buf,
					enum track_item alloc)
{
3842
	int len = 0;
3843
	unsigned long i;
3844
	struct loc_track t = { 0, 0, NULL };
3845
	int node;
E
Eric Dumazet 已提交
3846 3847
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
				     sizeof(unsigned long), GFP_KERNEL);
3848

E
Eric Dumazet 已提交
3849 3850 3851
	if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
				     GFP_TEMPORARY)) {
		kfree(map);
3852
		return sprintf(buf, "Out of memory\n");
E
Eric Dumazet 已提交
3853
	}
3854 3855 3856
	/* Push back cpu slabs */
	flush_all(s);

C
Christoph Lameter 已提交
3857
	for_each_node_state(node, N_NORMAL_MEMORY) {
3858 3859 3860 3861
		struct kmem_cache_node *n = get_node(s, node);
		unsigned long flags;
		struct page *page;

3862
		if (!atomic_long_read(&n->nr_slabs))
3863 3864 3865 3866
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry(page, &n->partial, lru)
E
Eric Dumazet 已提交
3867
			process_slab(&t, s, page, alloc, map);
3868
		list_for_each_entry(page, &n->full, lru)
E
Eric Dumazet 已提交
3869
			process_slab(&t, s, page, alloc, map);
3870 3871 3872 3873
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	for (i = 0; i < t.count; i++) {
3874
		struct location *l = &t.loc[i];
3875

H
Hugh Dickins 已提交
3876
		if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
3877
			break;
3878
		len += sprintf(buf + len, "%7ld ", l->count);
3879 3880

		if (l->addr)
J
Joe Perches 已提交
3881
			len += sprintf(buf + len, "%pS", (void *)l->addr);
3882
		else
3883
			len += sprintf(buf + len, "<not-available>");
3884 3885

		if (l->sum_time != l->min_time) {
3886
			len += sprintf(buf + len, " age=%ld/%ld/%ld",
R
Roman Zippel 已提交
3887 3888 3889
				l->min_time,
				(long)div_u64(l->sum_time, l->count),
				l->max_time);
3890
		} else
3891
			len += sprintf(buf + len, " age=%ld",
3892 3893 3894
				l->min_time);

		if (l->min_pid != l->max_pid)
3895
			len += sprintf(buf + len, " pid=%ld-%ld",
3896 3897
				l->min_pid, l->max_pid);
		else
3898
			len += sprintf(buf + len, " pid=%ld",
3899 3900
				l->min_pid);

R
Rusty Russell 已提交
3901 3902
		if (num_online_cpus() > 1 &&
				!cpumask_empty(to_cpumask(l->cpus)) &&
3903 3904 3905
				len < PAGE_SIZE - 60) {
			len += sprintf(buf + len, " cpus=");
			len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
R
Rusty Russell 已提交
3906
						 to_cpumask(l->cpus));
3907 3908
		}

3909
		if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
3910 3911 3912
				len < PAGE_SIZE - 60) {
			len += sprintf(buf + len, " nodes=");
			len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
3913 3914 3915
					l->nodes);
		}

3916
		len += sprintf(buf + len, "\n");
3917 3918 3919
	}

	free_loc_track(&t);
E
Eric Dumazet 已提交
3920
	kfree(map);
3921
	if (!t.count)
3922 3923
		len += sprintf(buf, "No data\n");
	return len;
3924
}
3925
#endif
3926

3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988
#ifdef SLUB_RESILIENCY_TEST
static void resiliency_test(void)
{
	u8 *p;

	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || SLUB_PAGE_SHIFT < 10);

	printk(KERN_ERR "SLUB resiliency testing\n");
	printk(KERN_ERR "-----------------------\n");
	printk(KERN_ERR "A. Corruption after allocation\n");

	p = kzalloc(16, GFP_KERNEL);
	p[16] = 0x12;
	printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
			" 0x12->0x%p\n\n", p + 16);

	validate_slab_cache(kmalloc_caches[4]);

	/* Hmmm... The next two are dangerous */
	p = kzalloc(32, GFP_KERNEL);
	p[32 + sizeof(void *)] = 0x34;
	printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
			" 0x34 -> -0x%p\n", p);
	printk(KERN_ERR
		"If allocated object is overwritten then not detectable\n\n");

	validate_slab_cache(kmalloc_caches[5]);
	p = kzalloc(64, GFP_KERNEL);
	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
	*p = 0x56;
	printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
									p);
	printk(KERN_ERR
		"If allocated object is overwritten then not detectable\n\n");
	validate_slab_cache(kmalloc_caches[6]);

	printk(KERN_ERR "\nB. Corruption after free\n");
	p = kzalloc(128, GFP_KERNEL);
	kfree(p);
	*p = 0x78;
	printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[7]);

	p = kzalloc(256, GFP_KERNEL);
	kfree(p);
	p[50] = 0x9a;
	printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
			p);
	validate_slab_cache(kmalloc_caches[8]);

	p = kzalloc(512, GFP_KERNEL);
	kfree(p);
	p[512] = 0xab;
	printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
	validate_slab_cache(kmalloc_caches[9]);
}
#else
#ifdef CONFIG_SYSFS
static void resiliency_test(void) {};
#endif
#endif

3989
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
3990
enum slab_stat_type {
3991 3992 3993 3994 3995
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
C
Christoph Lameter 已提交
3996 3997
};

3998
#define SO_ALL		(1 << SL_ALL)
C
Christoph Lameter 已提交
3999 4000 4001
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
4002
#define SO_TOTAL	(1 << SL_TOTAL)
C
Christoph Lameter 已提交
4003

4004 4005
static ssize_t show_slab_objects(struct kmem_cache *s,
			    char *buf, unsigned long flags)
C
Christoph Lameter 已提交
4006 4007 4008 4009 4010 4011 4012 4013
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;
	unsigned long *per_cpu;

	nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4014 4015
	if (!nodes)
		return -ENOMEM;
C
Christoph Lameter 已提交
4016 4017
	per_cpu = nodes + nr_node_ids;

4018 4019
	if (flags & SO_CPU) {
		int cpu;
C
Christoph Lameter 已提交
4020

4021
		for_each_possible_cpu(cpu) {
4022
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4023

4024 4025 4026 4027 4028 4029 4030 4031
			if (!c || c->node < 0)
				continue;

			if (c->page) {
					if (flags & SO_TOTAL)
						x = c->page->objects;
				else if (flags & SO_OBJECTS)
					x = c->page->inuse;
C
Christoph Lameter 已提交
4032 4033
				else
					x = 1;
4034

C
Christoph Lameter 已提交
4035
				total += x;
4036
				nodes[c->node] += x;
C
Christoph Lameter 已提交
4037
			}
4038
			per_cpu[c->node]++;
C
Christoph Lameter 已提交
4039 4040 4041
		}
	}

4042
	lock_memory_hotplug();
4043
#ifdef CONFIG_SLUB_DEBUG
4044 4045 4046 4047 4048 4049 4050 4051 4052
	if (flags & SO_ALL) {
		for_each_node_state(node, N_NORMAL_MEMORY) {
			struct kmem_cache_node *n = get_node(s, node);

		if (flags & SO_TOTAL)
			x = atomic_long_read(&n->total_objects);
		else if (flags & SO_OBJECTS)
			x = atomic_long_read(&n->total_objects) -
				count_partial(n, count_free);
C
Christoph Lameter 已提交
4053 4054

			else
4055
				x = atomic_long_read(&n->nr_slabs);
C
Christoph Lameter 已提交
4056 4057 4058 4059
			total += x;
			nodes[node] += x;
		}

4060 4061 4062
	} else
#endif
	if (flags & SO_PARTIAL) {
4063 4064
		for_each_node_state(node, N_NORMAL_MEMORY) {
			struct kmem_cache_node *n = get_node(s, node);
C
Christoph Lameter 已提交
4065

4066 4067 4068 4069
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
C
Christoph Lameter 已提交
4070
			else
4071
				x = n->nr_partial;
C
Christoph Lameter 已提交
4072 4073 4074 4075 4076 4077
			total += x;
			nodes[node] += x;
		}
	}
	x = sprintf(buf, "%lu", total);
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4078
	for_each_node_state(node, N_NORMAL_MEMORY)
C
Christoph Lameter 已提交
4079 4080 4081 4082
		if (nodes[node])
			x += sprintf(buf + x, " N%d=%lu",
					node, nodes[node]);
#endif
4083
	unlock_memory_hotplug();
C
Christoph Lameter 已提交
4084 4085 4086 4087
	kfree(nodes);
	return x + sprintf(buf + x, "\n");
}

4088
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4089 4090 4091 4092
static int any_slab_objects(struct kmem_cache *s)
{
	int node;

4093
	for_each_online_node(node) {
C
Christoph Lameter 已提交
4094 4095
		struct kmem_cache_node *n = get_node(s, node);

4096 4097 4098
		if (!n)
			continue;

4099
		if (atomic_long_read(&n->total_objects))
C
Christoph Lameter 已提交
4100 4101 4102 4103
			return 1;
	}
	return 0;
}
4104
#endif
C
Christoph Lameter 已提交
4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
#define to_slab(n) container_of(n, struct kmem_cache, kobj);

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = __ATTR_RO(_name)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0644, _name##_show, _name##_store)

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->size);
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->align);
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->objsize);
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
4142
	return sprintf(buf, "%d\n", oo_objects(s->oo));
C
Christoph Lameter 已提交
4143 4144 4145
}
SLAB_ATTR_RO(objs_per_slab);

4146 4147 4148
static ssize_t order_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
4149 4150 4151 4152 4153 4154
	unsigned long order;
	int err;

	err = strict_strtoul(buf, 10, &order);
	if (err)
		return err;
4155 4156 4157 4158 4159 4160 4161 4162

	if (order > slub_max_order || order < slub_min_order)
		return -EINVAL;

	calculate_sizes(s, order);
	return length;
}

C
Christoph Lameter 已提交
4163 4164
static ssize_t order_show(struct kmem_cache *s, char *buf)
{
4165
	return sprintf(buf, "%d\n", oo_order(s->oo));
C
Christoph Lameter 已提交
4166
}
4167
SLAB_ATTR(order);
C
Christoph Lameter 已提交
4168

4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%lu\n", s->min_partial);
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

	err = strict_strtoul(buf, 10, &min);
	if (err)
		return err;

4184
	set_min_partial(s, min);
4185 4186 4187 4188
	return length;
}
SLAB_ATTR(min_partial);

C
Christoph Lameter 已提交
4189 4190
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
J
Joe Perches 已提交
4191 4192 4193
	if (!s->ctor)
		return 0;
	return sprintf(buf, "%pS\n", s->ctor);
C
Christoph Lameter 已提交
4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->refcount - 1);
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
4205
	return show_slab_objects(s, buf, SO_PARTIAL);
C
Christoph Lameter 已提交
4206 4207 4208 4209 4210
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
4211
	return show_slab_objects(s, buf, SO_CPU);
C
Christoph Lameter 已提交
4212 4213 4214 4215 4216
}
SLAB_ATTR_RO(cpu_slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
4217
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
C
Christoph Lameter 已提交
4218 4219 4220
}
SLAB_ATTR_RO(objects);

4221 4222 4223 4224 4225 4226
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}

static ssize_t reclaim_account_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
	if (buf[0] == '1')
		s->flags |= SLAB_RECLAIM_ACCOUNT;
	return length;
}
SLAB_ATTR(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
}
SLAB_ATTR_RO(destroy_by_rcu);

4262 4263 4264 4265 4266 4267
static ssize_t reserved_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->reserved);
}
SLAB_ATTR_RO(reserved);

4268
#ifdef CONFIG_SLUB_DEBUG
4269 4270 4271 4272 4273 4274
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

4275 4276 4277 4278 4279 4280
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

C
Christoph Lameter 已提交
4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
}

static ssize_t sanity_checks_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_DEBUG_FREE;
	if (buf[0] == '1')
		s->flags |= SLAB_DEBUG_FREE;
	return length;
}
SLAB_ATTR(sanity_checks);

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
}

static ssize_t trace_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
	s->flags &= ~SLAB_TRACE;
	if (buf[0] == '1')
		s->flags |= SLAB_TRACE;
	return length;
}
SLAB_ATTR(trace);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}

static ssize_t red_zone_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_RED_ZONE;
	if (buf[0] == '1')
		s->flags |= SLAB_RED_ZONE;
4325
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343
	return length;
}
SLAB_ATTR(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}

static ssize_t poison_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_POISON;
	if (buf[0] == '1')
		s->flags |= SLAB_POISON;
4344
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362
	return length;
}
SLAB_ATTR(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}

static ssize_t store_user_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_STORE_USER;
	if (buf[0] == '1')
		s->flags |= SLAB_STORE_USER;
4363
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4364 4365 4366 4367
	return length;
}
SLAB_ATTR(store_user);

4368 4369 4370 4371 4372 4373 4374 4375
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
4376 4377 4378 4379 4380 4381 4382 4383
	int ret = -EINVAL;

	if (buf[0] == '1') {
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
4384 4385
}
SLAB_ATTR(validate);
4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418

static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_ALLOC);
}
SLAB_ATTR_RO(alloc_calls);

static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_FREE);
}
SLAB_ATTR_RO(free_calls);
#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
}

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
	s->flags &= ~SLAB_FAILSLAB;
	if (buf[0] == '1')
		s->flags |= SLAB_FAILSLAB;
	return length;
}
SLAB_ATTR(failslab);
4419
#endif
4420

4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
	if (buf[0] == '1') {
		int rc = kmem_cache_shrink(s);

		if (rc)
			return rc;
	} else
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

C
Christoph Lameter 已提交
4440
#ifdef CONFIG_NUMA
4441
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
C
Christoph Lameter 已提交
4442
{
4443
	return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
C
Christoph Lameter 已提交
4444 4445
}

4446
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
C
Christoph Lameter 已提交
4447 4448
				const char *buf, size_t length)
{
4449 4450 4451 4452 4453 4454 4455
	unsigned long ratio;
	int err;

	err = strict_strtoul(buf, 10, &ratio);
	if (err)
		return err;

4456
	if (ratio <= 100)
4457
		s->remote_node_defrag_ratio = ratio * 10;
C
Christoph Lameter 已提交
4458 4459 4460

	return length;
}
4461
SLAB_ATTR(remote_node_defrag_ratio);
C
Christoph Lameter 已提交
4462 4463
#endif

4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
	int len;
	int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
4476
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
4477 4478 4479 4480 4481 4482 4483

		data[cpu] = x;
		sum += x;
	}

	len = sprintf(buf, "%lu", sum);

4484
#ifdef CONFIG_SMP
4485 4486
	for_each_online_cpu(cpu) {
		if (data[cpu] && len < PAGE_SIZE - 20)
4487
			len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
4488
	}
4489
#endif
4490 4491 4492 4493
	kfree(data);
	return len + sprintf(buf + len, "\n");
}

D
David Rientjes 已提交
4494 4495 4496 4497 4498
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
4499
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
D
David Rientjes 已提交
4500 4501
}

4502 4503 4504 4505 4506
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
D
David Rientjes 已提交
4507 4508 4509 4510 4511 4512 4513 4514 4515
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
4534
STAT_ATTR(ORDER_FALLBACK, order_fallback);
4535 4536
#endif

P
Pekka Enberg 已提交
4537
static struct attribute *slab_attrs[] = {
C
Christoph Lameter 已提交
4538 4539 4540 4541
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
4542
	&min_partial_attr.attr,
C
Christoph Lameter 已提交
4543
	&objects_attr.attr,
4544
	&objects_partial_attr.attr,
C
Christoph Lameter 已提交
4545 4546 4547 4548 4549 4550 4551 4552
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
4553
	&shrink_attr.attr,
4554
	&reserved_attr.attr,
4555
#ifdef CONFIG_SLUB_DEBUG
4556 4557 4558 4559
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
C
Christoph Lameter 已提交
4560 4561 4562
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
4563
	&validate_attr.attr,
4564 4565
	&alloc_calls_attr.attr,
	&free_calls_attr.attr,
4566
#endif
C
Christoph Lameter 已提交
4567 4568 4569 4570
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
4571
	&remote_node_defrag_ratio_attr.attr,
4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
4591
	&order_fallback_attr.attr,
C
Christoph Lameter 已提交
4592
#endif
4593 4594 4595 4596
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif

C
Christoph Lameter 已提交
4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
	NULL
};

static struct attribute_group slab_attr_group = {
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);

	return err;
}

C
Christoph Lameter 已提交
4642 4643 4644 4645
static void kmem_cache_release(struct kobject *kobj)
{
	struct kmem_cache *s = to_slab(kobj);

P
Pekka Enberg 已提交
4646
	kfree(s->name);
C
Christoph Lameter 已提交
4647 4648 4649
	kfree(s);
}

4650
static const struct sysfs_ops slab_sysfs_ops = {
C
Christoph Lameter 已提交
4651 4652 4653 4654 4655 4656
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
C
Christoph Lameter 已提交
4657
	.release = kmem_cache_release
C
Christoph Lameter 已提交
4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;
}

4669
static const struct kset_uevent_ops slab_uevent_ops = {
C
Christoph Lameter 已提交
4670 4671 4672
	.filter = uevent_filter,
};

4673
static struct kset *slab_kset;
C
Christoph Lameter 已提交
4674 4675 4676 4677

#define ID_STR_LENGTH 64

/* Create a unique string id for a slab cache:
C
Christoph Lameter 已提交
4678 4679
 *
 * Format	:[flags-]size
C
Christoph Lameter 已提交
4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

	BUG_ON(!name);

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
	if (s->flags & SLAB_DEBUG_FREE)
		*p++ = 'F';
V
Vegard Nossum 已提交
4702 4703
	if (!(s->flags & SLAB_NOTRACK))
		*p++ = 't';
C
Christoph Lameter 已提交
4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727
	if (p != name + 1)
		*p++ = '-';
	p += sprintf(p, "%07d", s->size);
	BUG_ON(p > name + ID_STR_LENGTH - 1);
	return name;
}

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
	int unmergeable;

	if (slab_state < SYSFS)
		/* Defer until later */
		return 0;

	unmergeable = slab_unmergeable(s);
	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
4728
		sysfs_remove_link(&slab_kset->kobj, s->name);
C
Christoph Lameter 已提交
4729 4730 4731 4732 4733 4734 4735 4736 4737
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
	}

4738
	s->kobj.kset = slab_kset;
4739 4740 4741
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
	if (err) {
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
4742
		return err;
4743
	}
C
Christoph Lameter 已提交
4744 4745

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
4746 4747 4748
	if (err) {
		kobject_del(&s->kobj);
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
4749
		return err;
4750
	}
C
Christoph Lameter 已提交
4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761
	kobject_uevent(&s->kobj, KOBJ_ADD);
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
		kfree(name);
	}
	return 0;
}

static void sysfs_slab_remove(struct kmem_cache *s)
{
4762 4763 4764 4765 4766 4767 4768
	if (slab_state < SYSFS)
		/*
		 * Sysfs has not been setup yet so no need to remove the
		 * cache from sysfs.
		 */
		return;

C
Christoph Lameter 已提交
4769 4770
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
	kobject_del(&s->kobj);
C
Christoph Lameter 已提交
4771
	kobject_put(&s->kobj);
C
Christoph Lameter 已提交
4772 4773 4774 4775
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
N
Nick Andrew 已提交
4776
 * available lest we lose that information.
C
Christoph Lameter 已提交
4777 4778 4779 4780 4781 4782 4783
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

A
Adrian Bunk 已提交
4784
static struct saved_alias *alias_list;
C
Christoph Lameter 已提交
4785 4786 4787 4788 4789 4790 4791 4792 4793

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

	if (slab_state == SYSFS) {
		/*
		 * If we have a leftover link then remove it.
		 */
4794 4795
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
C
Christoph Lameter 已提交
4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
	return 0;
}

static int __init slab_sysfs_init(void)
{
4811
	struct kmem_cache *s;
C
Christoph Lameter 已提交
4812 4813
	int err;

4814 4815
	down_write(&slub_lock);

4816
	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
4817
	if (!slab_kset) {
4818
		up_write(&slub_lock);
C
Christoph Lameter 已提交
4819 4820 4821 4822
		printk(KERN_ERR "Cannot register slab subsystem.\n");
		return -ENOSYS;
	}

4823 4824
	slab_state = SYSFS;

4825
	list_for_each_entry(s, &slab_caches, list) {
4826
		err = sysfs_slab_add(s);
4827 4828 4829
		if (err)
			printk(KERN_ERR "SLUB: Unable to add boot slab %s"
						" to sysfs\n", s->name);
4830
	}
C
Christoph Lameter 已提交
4831 4832 4833 4834 4835 4836

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
4837 4838 4839
		if (err)
			printk(KERN_ERR "SLUB: Unable to add boot slab alias"
					" %s to sysfs\n", s->name);
C
Christoph Lameter 已提交
4840 4841 4842
		kfree(al);
	}

4843
	up_write(&slub_lock);
C
Christoph Lameter 已提交
4844 4845 4846 4847 4848
	resiliency_test();
	return 0;
}

__initcall(slab_sysfs_init);
4849
#endif /* CONFIG_SYSFS */
P
Pekka J Enberg 已提交
4850 4851 4852 4853

/*
 * The /proc/slabinfo ABI
 */
4854
#ifdef CONFIG_SLABINFO
P
Pekka J Enberg 已提交
4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890
static void print_slabinfo_header(struct seq_file *m)
{
	seq_puts(m, "slabinfo - version: 2.1\n");
	seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
		 "<objperslab> <pagesperslab>");
	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
	seq_putc(m, '\n');
}

static void *s_start(struct seq_file *m, loff_t *pos)
{
	loff_t n = *pos;

	down_read(&slub_lock);
	if (!n)
		print_slabinfo_header(m);

	return seq_list_start(&slab_caches, *pos);
}

static void *s_next(struct seq_file *m, void *p, loff_t *pos)
{
	return seq_list_next(p, &slab_caches, pos);
}

static void s_stop(struct seq_file *m, void *p)
{
	up_read(&slub_lock);
}

static int s_show(struct seq_file *m, void *p)
{
	unsigned long nr_partials = 0;
	unsigned long nr_slabs = 0;
	unsigned long nr_inuse = 0;
4891 4892
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
P
Pekka J Enberg 已提交
4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905
	struct kmem_cache *s;
	int node;

	s = list_entry(p, struct kmem_cache, list);

	for_each_online_node(node) {
		struct kmem_cache_node *n = get_node(s, node);

		if (!n)
			continue;

		nr_partials += n->nr_partial;
		nr_slabs += atomic_long_read(&n->nr_slabs);
4906 4907
		nr_objs += atomic_long_read(&n->total_objects);
		nr_free += count_partial(n, count_free);
P
Pekka J Enberg 已提交
4908 4909
	}

4910
	nr_inuse = nr_objs - nr_free;
P
Pekka J Enberg 已提交
4911 4912

	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
4913 4914
		   nr_objs, s->size, oo_objects(s->oo),
		   (1 << oo_order(s->oo)));
P
Pekka J Enberg 已提交
4915 4916 4917 4918 4919 4920 4921
	seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
	seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
		   0UL);
	seq_putc(m, '\n');
	return 0;
}

4922
static const struct seq_operations slabinfo_op = {
P
Pekka J Enberg 已提交
4923 4924 4925 4926 4927 4928
	.start = s_start,
	.next = s_next,
	.stop = s_stop,
	.show = s_show,
};

4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942
static int slabinfo_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &slabinfo_op);
}

static const struct file_operations proc_slabinfo_operations = {
	.open		= slabinfo_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init slab_proc_init(void)
{
4943
	proc_create("slabinfo", S_IRUGO, NULL, &proc_slabinfo_operations);
4944 4945 4946
	return 0;
}
module_init(slab_proc_init);
4947
#endif /* CONFIG_SLABINFO */