slub.c 133.8 KB
Newer Older
C
Christoph Lameter 已提交
1 2 3 4
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
5 6
 * The allocator synchronizes using per slab locks or atomic operatios
 * and only uses a centralized lock to manage a pool of partial slabs.
C
Christoph Lameter 已提交
7
 *
C
Christoph Lameter 已提交
8
 * (C) 2007 SGI, Christoph Lameter
9
 * (C) 2011 Linux Foundation, Christoph Lameter
C
Christoph Lameter 已提交
10 11 12
 */

#include <linux/mm.h>
N
Nick Piggin 已提交
13
#include <linux/swap.h> /* struct reclaim_state */
C
Christoph Lameter 已提交
14 15 16 17 18
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/slab.h>
19
#include "slab.h"
20
#include <linux/proc_fs.h>
21
#include <linux/notifier.h>
C
Christoph Lameter 已提交
22
#include <linux/seq_file.h>
23
#include <linux/kasan.h>
V
Vegard Nossum 已提交
24
#include <linux/kmemcheck.h>
C
Christoph Lameter 已提交
25 26 27 28
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
29
#include <linux/debugobjects.h>
C
Christoph Lameter 已提交
30
#include <linux/kallsyms.h>
31
#include <linux/memory.h>
R
Roman Zippel 已提交
32
#include <linux/math64.h>
A
Akinobu Mita 已提交
33
#include <linux/fault-inject.h>
34
#include <linux/stacktrace.h>
35
#include <linux/prefetch.h>
36
#include <linux/memcontrol.h>
C
Christoph Lameter 已提交
37

38 39
#include <trace/events/kmem.h>

40 41
#include "internal.h"

C
Christoph Lameter 已提交
42 43
/*
 * Lock order:
44
 *   1. slab_mutex (Global Mutex)
45 46
 *   2. node->list_lock
 *   3. slab_lock(page) (Only on some arches and for debugging)
C
Christoph Lameter 已提交
47
 *
48
 *   slab_mutex
49
 *
50
 *   The role of the slab_mutex is to protect the list of all the slabs
51 52 53 54 55 56 57 58 59 60 61 62 63 64
 *   and to synchronize major metadata changes to slab cache structures.
 *
 *   The slab_lock is only used for debugging and on arches that do not
 *   have the ability to do a cmpxchg_double. It only protects the second
 *   double word in the page struct. Meaning
 *	A. page->freelist	-> List of object free in a page
 *	B. page->counters	-> Counters of objects
 *	C. page->frozen		-> frozen state
 *
 *   If a slab is frozen then it is exempt from list management. It is not
 *   on any list. The processor that froze the slab is the one who can
 *   perform list operations on the page. Other processors may put objects
 *   onto the freelist but the processor that froze the slab is the only
 *   one that can retrieve the objects from the page's freelist.
C
Christoph Lameter 已提交
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
 *
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
 *   Interrupts are disabled during allocation and deallocation in order to
 *   make the slab allocator safe to use in the context of an irq. In addition
 *   interrupts are disabled to ensure that the processor does not change
 *   while handling per_cpu slabs, due to kernel preemption.
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
C
Christoph Lameter 已提交
85 86
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
C
Christoph Lameter 已提交
87
 * freed then the slab will show up again on the partial lists.
C
Christoph Lameter 已提交
88 89
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
C
Christoph Lameter 已提交
90 91 92 93 94 95 96
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
 * Overloading of page flags that are otherwise used for LRU management.
 *
97 98 99 100 101 102 103 104 105 106 107 108
 * PageActive 		The slab is frozen and exempt from list processing.
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
109
 * 			freelist that allows lockless access to
110 111
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
C
Christoph Lameter 已提交
112 113 114
 *
 * PageError		Slab requires special handling due to debug
 * 			options set. This moves	slab handling out of
115
 * 			the fast path and disables lockless freelists.
C
Christoph Lameter 已提交
116 117
 */

118 119
static inline int kmem_cache_debug(struct kmem_cache *s)
{
120
#ifdef CONFIG_SLUB_DEBUG
121
	return unlikely(s->flags & SLAB_DEBUG_FLAGS);
122
#else
123
	return 0;
124
#endif
125
}
126

J
Joonsoo Kim 已提交
127 128 129 130 131 132 133 134
static inline void *fixup_red_left(struct kmem_cache *s, void *p)
{
	if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE)
		p += s->red_left_pad;

	return p;
}

135 136 137 138 139 140 141 142 143
static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	return !kmem_cache_debug(s);
#else
	return false;
#endif
}

C
Christoph Lameter 已提交
144 145 146 147 148 149 150 151 152 153 154
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST

155 156 157
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG

158 159 160 161
/*
 * Mininum number of partial slabs. These will be left on the partial
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
C
Christoph Lameter 已提交
162
#define MIN_PARTIAL 5
C
Christoph Lameter 已提交
163

164 165 166
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
Z
Zhi Yong Wu 已提交
167
 * sort the partial list by the number of objects in use.
168 169 170
 */
#define MAX_PARTIAL 10

171
#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
C
Christoph Lameter 已提交
172
				SLAB_POISON | SLAB_STORE_USER)
C
Christoph Lameter 已提交
173

174 175 176 177 178 179 180 181
/*
 * These debug flags cannot use CMPXCHG because there might be consistency
 * issues when checking or reading debug information
 */
#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)


182
/*
183 184 185
 * Debugging flags that require metadata to be stored in the slab.  These get
 * disabled when slub_debug=O is used and a cache's min order increases with
 * metadata.
186
 */
187
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
188

189 190
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
191
#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */
192

C
Christoph Lameter 已提交
193
/* Internal SLUB flags */
C
Christoph Lameter 已提交
194
#define __OBJECT_POISON		0x80000000UL /* Poison object */
195
#define __CMPXCHG_DOUBLE	0x40000000UL /* Use cmpxchg_double */
C
Christoph Lameter 已提交
196 197 198 199 200

#ifdef CONFIG_SMP
static struct notifier_block slab_notifier;
#endif

201 202 203
/*
 * Tracking user of a slab.
 */
204
#define TRACK_ADDRS_COUNT 16
205
struct track {
206
	unsigned long addr;	/* Called from address */
207 208 209
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
210 211 212 213 214 215 216
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

217
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
218 219
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
220
static void memcg_propagate_slab_attrs(struct kmem_cache *s);
C
Christoph Lameter 已提交
221
#else
222 223 224
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
225
static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
C
Christoph Lameter 已提交
226 227
#endif

228
static inline void stat(const struct kmem_cache *s, enum stat_item si)
229 230
{
#ifdef CONFIG_SLUB_STATS
231 232 233 234 235
	/*
	 * The rmw is racy on a preemptible kernel but this is acceptable, so
	 * avoid this_cpu_add()'s irq-disable overhead.
	 */
	raw_cpu_inc(s->cpu_slab->stat[si]);
236 237 238
#endif
}

C
Christoph Lameter 已提交
239 240 241 242
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

243 244 245 246 247
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
	return *(void **)(object + s->offset);
}

248 249 250 251 252
static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{
	prefetch(object + s->offset);
}

253 254 255 256
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
	void *p;

257 258 259
	if (!debug_pagealloc_enabled())
		return get_freepointer(s, object);

260 261 262 263
	probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
	return p;
}

264 265 266 267 268 269
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
	*(void **)(object + s->offset) = fp;
}

/* Loop over all objects in a slab */
270
#define for_each_object(__p, __s, __addr, __objects) \
J
Joonsoo Kim 已提交
271 272 273
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)
274

275
#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
J
Joonsoo Kim 已提交
276 277 278
	for (__p = fixup_red_left(__s, __addr), __idx = 1; \
		__idx <= __objects; \
		__p += (__s)->size, __idx++)
279

280 281 282 283 284 285
/* Determine object index from a given position */
static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
{
	return (p - addr) / s->size;
}

286 287 288 289 290
static inline int order_objects(int order, unsigned long size, int reserved)
{
	return ((PAGE_SIZE << order) - reserved) / size;
}

291
static inline struct kmem_cache_order_objects oo_make(int order,
292
		unsigned long size, int reserved)
293 294
{
	struct kmem_cache_order_objects x = {
295
		(order << OO_SHIFT) + order_objects(order, size, reserved)
296 297 298 299 300 301 302
	};

	return x;
}

static inline int oo_order(struct kmem_cache_order_objects x)
{
303
	return x.x >> OO_SHIFT;
304 305 306 307
}

static inline int oo_objects(struct kmem_cache_order_objects x)
{
308
	return x.x & OO_MASK;
309 310
}

311 312 313 314 315
/*
 * Per slab locking using the pagelock
 */
static __always_inline void slab_lock(struct page *page)
{
316
	VM_BUG_ON_PAGE(PageTail(page), page);
317 318 319 320 321
	bit_spin_lock(PG_locked, &page->flags);
}

static __always_inline void slab_unlock(struct page *page)
{
322
	VM_BUG_ON_PAGE(PageTail(page), page);
323 324 325
	__bit_spin_unlock(PG_locked, &page->flags);
}

326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
static inline void set_page_slub_counters(struct page *page, unsigned long counters_new)
{
	struct page tmp;
	tmp.counters = counters_new;
	/*
	 * page->counters can cover frozen/inuse/objects as well
	 * as page->_count.  If we assign to ->counters directly
	 * we run the risk of losing updates to page->_count, so
	 * be careful and only assign to the fields we need.
	 */
	page->frozen  = tmp.frozen;
	page->inuse   = tmp.inuse;
	page->objects = tmp.objects;
}

341 342 343 344 345 346 347
/* Interrupts must be disabled (for the fallback code to work right) */
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
	VM_BUG_ON(!irqs_disabled());
348 349
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
350
	if (s->flags & __CMPXCHG_DOUBLE) {
351
		if (cmpxchg_double(&page->freelist, &page->counters,
352 353
				   freelist_old, counters_old,
				   freelist_new, counters_new))
354
			return true;
355 356 357 358
	} else
#endif
	{
		slab_lock(page);
359 360
		if (page->freelist == freelist_old &&
					page->counters == counters_old) {
361
			page->freelist = freelist_new;
362
			set_page_slub_counters(page, counters_new);
363
			slab_unlock(page);
364
			return true;
365 366 367 368 369 370 371 372
		}
		slab_unlock(page);
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
373
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
374 375
#endif

376
	return false;
377 378
}

379 380 381 382 383
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
384 385
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
386
	if (s->flags & __CMPXCHG_DOUBLE) {
387
		if (cmpxchg_double(&page->freelist, &page->counters,
388 389
				   freelist_old, counters_old,
				   freelist_new, counters_new))
390
			return true;
391 392 393
	} else
#endif
	{
394 395 396
		unsigned long flags;

		local_irq_save(flags);
397
		slab_lock(page);
398 399
		if (page->freelist == freelist_old &&
					page->counters == counters_old) {
400
			page->freelist = freelist_new;
401
			set_page_slub_counters(page, counters_new);
402
			slab_unlock(page);
403
			local_irq_restore(flags);
404
			return true;
405
		}
406
		slab_unlock(page);
407
		local_irq_restore(flags);
408 409 410 411 412 413
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
414
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
415 416
#endif

417
	return false;
418 419
}

C
Christoph Lameter 已提交
420
#ifdef CONFIG_SLUB_DEBUG
421 422 423
/*
 * Determine a map of object in use on a page.
 *
424
 * Node listlock must be held to guarantee that the page does
425 426 427 428 429 430 431 432 433 434 435
 * not vanish from under us.
 */
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
{
	void *p;
	void *addr = page_address(page);

	for (p = page->freelist; p; p = get_freepointer(s, p))
		set_bit(slab_index(p, s, addr), map);
}

J
Joonsoo Kim 已提交
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
static inline int size_from_object(struct kmem_cache *s)
{
	if (s->flags & SLAB_RED_ZONE)
		return s->size - s->red_left_pad;

	return s->size;
}

static inline void *restore_red_left(struct kmem_cache *s, void *p)
{
	if (s->flags & SLAB_RED_ZONE)
		p -= s->red_left_pad;

	return p;
}

C
Christoph Lameter 已提交
452 453 454
/*
 * Debug settings:
 */
455
#if defined(CONFIG_SLUB_DEBUG_ON)
456
static int slub_debug = DEBUG_DEFAULT_FLAGS;
457 458
#elif defined(CONFIG_KASAN)
static int slub_debug = SLAB_STORE_USER;
459
#else
C
Christoph Lameter 已提交
460
static int slub_debug;
461
#endif
C
Christoph Lameter 已提交
462 463

static char *slub_debug_slabs;
464
static int disable_higher_order_debug;
C
Christoph Lameter 已提交
465

466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
/*
 * slub is about to manipulate internal object metadata.  This memory lies
 * outside the range of the allocated object, so accessing it would normally
 * be reported by kasan as a bounds error.  metadata_access_enable() is used
 * to tell kasan that these accesses are OK.
 */
static inline void metadata_access_enable(void)
{
	kasan_disable_current();
}

static inline void metadata_access_disable(void)
{
	kasan_enable_current();
}

C
Christoph Lameter 已提交
482 483 484
/*
 * Object debugging
 */
J
Joonsoo Kim 已提交
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504

/* Verify that a pointer has an address that is valid within a slab page */
static inline int check_valid_pointer(struct kmem_cache *s,
				struct page *page, void *object)
{
	void *base;

	if (!object)
		return 1;

	base = page_address(page);
	object = restore_red_left(s, object);
	if (object < base || object >= base + page->objects * s->size ||
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

C
Christoph Lameter 已提交
505 506
static void print_section(char *text, u8 *addr, unsigned int length)
{
507
	metadata_access_enable();
508 509
	print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
			length, 1);
510
	metadata_access_disable();
C
Christoph Lameter 已提交
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
}

static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
527
			enum track_item alloc, unsigned long addr)
C
Christoph Lameter 已提交
528
{
A
Akinobu Mita 已提交
529
	struct track *p = get_track(s, object, alloc);
C
Christoph Lameter 已提交
530 531

	if (addr) {
532 533 534 535 536 537 538 539
#ifdef CONFIG_STACKTRACE
		struct stack_trace trace;
		int i;

		trace.nr_entries = 0;
		trace.max_entries = TRACK_ADDRS_COUNT;
		trace.entries = p->addrs;
		trace.skip = 3;
540
		metadata_access_enable();
541
		save_stack_trace(&trace);
542
		metadata_access_disable();
543 544 545 546 547 548 549 550 551

		/* See rant in lockdep.c */
		if (trace.nr_entries != 0 &&
		    trace.entries[trace.nr_entries - 1] == ULONG_MAX)
			trace.nr_entries--;

		for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
			p->addrs[i] = 0;
#endif
C
Christoph Lameter 已提交
552 553
		p->addr = addr;
		p->cpu = smp_processor_id();
A
Alexey Dobriyan 已提交
554
		p->pid = current->pid;
C
Christoph Lameter 已提交
555 556 557 558 559 560 561
		p->when = jiffies;
	} else
		memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
562 563 564
	if (!(s->flags & SLAB_STORE_USER))
		return;

565 566
	set_track(s, object, TRACK_FREE, 0UL);
	set_track(s, object, TRACK_ALLOC, 0UL);
C
Christoph Lameter 已提交
567 568 569 570 571 572 573
}

static void print_track(const char *s, struct track *t)
{
	if (!t->addr)
		return;

574 575
	pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
	       s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
576 577 578 579 580
#ifdef CONFIG_STACKTRACE
	{
		int i;
		for (i = 0; i < TRACK_ADDRS_COUNT; i++)
			if (t->addrs[i])
581
				pr_err("\t%pS\n", (void *)t->addrs[i]);
582 583 584 585
			else
				break;
	}
#endif
586 587 588 589 590 591 592 593 594 595 596 597 598
}

static void print_tracking(struct kmem_cache *s, void *object)
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

	print_track("Allocated", get_track(s, object, TRACK_ALLOC));
	print_track("Freed", get_track(s, object, TRACK_FREE));
}

static void print_page_info(struct page *page)
{
599
	pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
600
	       page, page->objects, page->inuse, page->freelist, page->flags);
601 602 603 604 605

}

static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
606
	struct va_format vaf;
607 608 609
	va_list args;

	va_start(args, fmt);
610 611
	vaf.fmt = fmt;
	vaf.va = &args;
612
	pr_err("=============================================================================\n");
613
	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
614
	pr_err("-----------------------------------------------------------------------------\n\n");
615

616
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
617
	va_end(args);
C
Christoph Lameter 已提交
618 619
}

620 621
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
622
	struct va_format vaf;
623 624 625
	va_list args;

	va_start(args, fmt);
626 627 628
	vaf.fmt = fmt;
	vaf.va = &args;
	pr_err("FIX %s: %pV\n", s->name, &vaf);
629 630 631 632
	va_end(args);
}

static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
C
Christoph Lameter 已提交
633 634
{
	unsigned int off;	/* Offset of last byte */
635
	u8 *addr = page_address(page);
636 637 638 639 640

	print_tracking(s, p);

	print_page_info(page);

641 642
	pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
	       p, p - addr, get_freepointer(s, p));
643

J
Joonsoo Kim 已提交
644 645 646
	if (s->flags & SLAB_RED_ZONE)
		print_section("Redzone ", p - s->red_left_pad, s->red_left_pad);
	else if (p > addr + 16)
647
		print_section("Bytes b4 ", p - 16, 16);
C
Christoph Lameter 已提交
648

649
	print_section("Object ", p, min_t(unsigned long, s->object_size,
650
				PAGE_SIZE));
C
Christoph Lameter 已提交
651
	if (s->flags & SLAB_RED_ZONE)
652 653
		print_section("Redzone ", p + s->object_size,
			s->inuse - s->object_size);
C
Christoph Lameter 已提交
654 655 656 657 658 659

	if (s->offset)
		off = s->offset + sizeof(void *);
	else
		off = s->inuse;

660
	if (s->flags & SLAB_STORE_USER)
C
Christoph Lameter 已提交
661 662
		off += 2 * sizeof(struct track);

J
Joonsoo Kim 已提交
663
	if (off != size_from_object(s))
C
Christoph Lameter 已提交
664
		/* Beginning of the filler is the free pointer */
J
Joonsoo Kim 已提交
665
		print_section("Padding ", p + off, size_from_object(s) - off);
666 667

	dump_stack();
C
Christoph Lameter 已提交
668 669
}

670
void object_err(struct kmem_cache *s, struct page *page,
C
Christoph Lameter 已提交
671 672
			u8 *object, char *reason)
{
673
	slab_bug(s, "%s", reason);
674
	print_trailer(s, page, object);
C
Christoph Lameter 已提交
675 676
}

677 678
static void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
C
Christoph Lameter 已提交
679 680 681 682
{
	va_list args;
	char buf[100];

683 684
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
C
Christoph Lameter 已提交
685
	va_end(args);
686
	slab_bug(s, "%s", buf);
687
	print_page_info(page);
C
Christoph Lameter 已提交
688 689 690
	dump_stack();
}

691
static void init_object(struct kmem_cache *s, void *object, u8 val)
C
Christoph Lameter 已提交
692 693 694
{
	u8 *p = object;

J
Joonsoo Kim 已提交
695 696 697
	if (s->flags & SLAB_RED_ZONE)
		memset(p - s->red_left_pad, val, s->red_left_pad);

C
Christoph Lameter 已提交
698
	if (s->flags & __OBJECT_POISON) {
699 700
		memset(p, POISON_FREE, s->object_size - 1);
		p[s->object_size - 1] = POISON_END;
C
Christoph Lameter 已提交
701 702 703
	}

	if (s->flags & SLAB_RED_ZONE)
704
		memset(p + s->object_size, val, s->inuse - s->object_size);
C
Christoph Lameter 已提交
705 706
}

707 708 709 710 711 712 713 714 715
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
	memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
			u8 *object, char *what,
P
Pekka Enberg 已提交
716
			u8 *start, unsigned int value, unsigned int bytes)
717 718 719 720
{
	u8 *fault;
	u8 *end;

721
	metadata_access_enable();
722
	fault = memchr_inv(start, value, bytes);
723
	metadata_access_disable();
724 725 726 727 728 729 730 731
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
732
	pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
733 734 735 736 737
					fault, end - 1, fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
C
Christoph Lameter 已提交
738 739 740 741 742 743 744 745 746
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
 * 	pointer is the first word of the object.
C
Christoph Lameter 已提交
747
 *
C
Christoph Lameter 已提交
748 749 750
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
751
 * object + s->object_size
C
Christoph Lameter 已提交
752
 * 	Padding to reach word boundary. This is also used for Redzoning.
C
Christoph Lameter 已提交
753
 * 	Padding is extended by another word if Redzoning is enabled and
754
 * 	object_size == inuse.
C
Christoph Lameter 已提交
755
 *
C
Christoph Lameter 已提交
756 757 758 759
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
C
Christoph Lameter 已提交
760 761
 * 	Meta data starts here.
 *
C
Christoph Lameter 已提交
762 763
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
C
Christoph Lameter 已提交
764
 * 	C. Padding to reach required alignment boundary or at mininum
C
Christoph Lameter 已提交
765
 * 		one word if debugging is on to be able to detect writes
C
Christoph Lameter 已提交
766 767 768
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
C
Christoph Lameter 已提交
769 770
 *
 * object + s->size
C
Christoph Lameter 已提交
771
 * 	Nothing is used beyond s->size.
C
Christoph Lameter 已提交
772
 *
773
 * If slabcaches are merged then the object_size and inuse boundaries are mostly
C
Christoph Lameter 已提交
774
 * ignored. And therefore no slab options that rely on these boundaries
C
Christoph Lameter 已提交
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
 * may be used with merged slabcaches.
 */

static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
{
	unsigned long off = s->inuse;	/* The end of info */

	if (s->offset)
		/* Freepointer is placed after the object. */
		off += sizeof(void *);

	if (s->flags & SLAB_STORE_USER)
		/* We also have user information there */
		off += 2 * sizeof(struct track);

J
Joonsoo Kim 已提交
790
	if (size_from_object(s) == off)
C
Christoph Lameter 已提交
791 792
		return 1;

793
	return check_bytes_and_report(s, page, p, "Object padding",
J
Joonsoo Kim 已提交
794
			p + off, POISON_INUSE, size_from_object(s) - off);
C
Christoph Lameter 已提交
795 796
}

797
/* Check the pad bytes at the end of a slab page */
C
Christoph Lameter 已提交
798 799
static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
800 801 802 803 804
	u8 *start;
	u8 *fault;
	u8 *end;
	int length;
	int remainder;
C
Christoph Lameter 已提交
805 806 807 808

	if (!(s->flags & SLAB_POISON))
		return 1;

809
	start = page_address(page);
810
	length = (PAGE_SIZE << compound_order(page)) - s->reserved;
811 812
	end = start + length;
	remainder = length % s->size;
C
Christoph Lameter 已提交
813 814 815
	if (!remainder)
		return 1;

816
	metadata_access_enable();
817
	fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
818
	metadata_access_disable();
819 820 821 822 823 824
	if (!fault)
		return 1;
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
825
	print_section("Padding ", end - remainder, remainder);
826

E
Eric Dumazet 已提交
827
	restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
828
	return 0;
C
Christoph Lameter 已提交
829 830 831
}

static int check_object(struct kmem_cache *s, struct page *page,
832
					void *object, u8 val)
C
Christoph Lameter 已提交
833 834
{
	u8 *p = object;
835
	u8 *endobject = object + s->object_size;
C
Christoph Lameter 已提交
836 837

	if (s->flags & SLAB_RED_ZONE) {
J
Joonsoo Kim 已提交
838 839 840 841
		if (!check_bytes_and_report(s, page, object, "Redzone",
			object - s->red_left_pad, val, s->red_left_pad))
			return 0;

842
		if (!check_bytes_and_report(s, page, object, "Redzone",
843
			endobject, val, s->inuse - s->object_size))
C
Christoph Lameter 已提交
844 845
			return 0;
	} else {
846
		if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
I
Ingo Molnar 已提交
847
			check_bytes_and_report(s, page, p, "Alignment padding",
848 849
				endobject, POISON_INUSE,
				s->inuse - s->object_size);
I
Ingo Molnar 已提交
850
		}
C
Christoph Lameter 已提交
851 852 853
	}

	if (s->flags & SLAB_POISON) {
854
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
855
			(!check_bytes_and_report(s, page, p, "Poison", p,
856
					POISON_FREE, s->object_size - 1) ||
857
			 !check_bytes_and_report(s, page, p, "Poison",
858
				p + s->object_size - 1, POISON_END, 1)))
C
Christoph Lameter 已提交
859 860 861 862 863 864 865
			return 0;
		/*
		 * check_pad_bytes cleans up on its own.
		 */
		check_pad_bytes(s, page, p);
	}

866
	if (!s->offset && val == SLUB_RED_ACTIVE)
C
Christoph Lameter 已提交
867 868 869 870 871 872 873 874 875 876
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
	if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
		object_err(s, page, p, "Freepointer corrupt");
		/*
N
Nick Andrew 已提交
877
		 * No choice but to zap it and thus lose the remainder
C
Christoph Lameter 已提交
878
		 * of the free objects in this slab. May cause
C
Christoph Lameter 已提交
879
		 * another error because the object count is now wrong.
C
Christoph Lameter 已提交
880
		 */
881
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
882 883 884 885 886 887 888
		return 0;
	}
	return 1;
}

static int check_slab(struct kmem_cache *s, struct page *page)
{
889 890
	int maxobj;

C
Christoph Lameter 已提交
891 892 893
	VM_BUG_ON(!irqs_disabled());

	if (!PageSlab(page)) {
894
		slab_err(s, page, "Not a valid slab page");
C
Christoph Lameter 已提交
895 896
		return 0;
	}
897

898
	maxobj = order_objects(compound_order(page), s->size, s->reserved);
899 900
	if (page->objects > maxobj) {
		slab_err(s, page, "objects %u > max %u",
901
			page->objects, maxobj);
902 903 904
		return 0;
	}
	if (page->inuse > page->objects) {
905
		slab_err(s, page, "inuse %u > max %u",
906
			page->inuse, page->objects);
C
Christoph Lameter 已提交
907 908 909 910 911 912 913 914
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
	slab_pad_check(s, page);
	return 1;
}

/*
C
Christoph Lameter 已提交
915 916
 * Determine if a certain object on a page is on the freelist. Must hold the
 * slab lock to guarantee that the chains are in a consistent state.
C
Christoph Lameter 已提交
917 918 919 920
 */
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
	int nr = 0;
921
	void *fp;
C
Christoph Lameter 已提交
922
	void *object = NULL;
923
	int max_objects;
C
Christoph Lameter 已提交
924

925
	fp = page->freelist;
926
	while (fp && nr <= page->objects) {
C
Christoph Lameter 已提交
927 928 929 930 931 932
		if (fp == search)
			return 1;
		if (!check_valid_pointer(s, page, fp)) {
			if (object) {
				object_err(s, page, object,
					"Freechain corrupt");
933
				set_freepointer(s, object, NULL);
C
Christoph Lameter 已提交
934
			} else {
935
				slab_err(s, page, "Freepointer corrupt");
936
				page->freelist = NULL;
937
				page->inuse = page->objects;
938
				slab_fix(s, "Freelist cleared");
C
Christoph Lameter 已提交
939 940 941 942 943 944 945 946 947
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

948
	max_objects = order_objects(compound_order(page), s->size, s->reserved);
949 950
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
951 952

	if (page->objects != max_objects) {
J
Joe Perches 已提交
953 954
		slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
			 page->objects, max_objects);
955 956 957
		page->objects = max_objects;
		slab_fix(s, "Number of objects adjusted.");
	}
958
	if (page->inuse != page->objects - nr) {
J
Joe Perches 已提交
959 960
		slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
			 page->inuse, page->objects - nr);
961
		page->inuse = page->objects - nr;
962
		slab_fix(s, "Object count adjusted.");
C
Christoph Lameter 已提交
963 964 965 966
	}
	return search == NULL;
}

967 968
static void trace(struct kmem_cache *s, struct page *page, void *object,
								int alloc)
C
Christoph Lameter 已提交
969 970
{
	if (s->flags & SLAB_TRACE) {
971
		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
C
Christoph Lameter 已提交
972 973 974 975 976 977
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
978 979
			print_section("Object ", (void *)object,
					s->object_size);
C
Christoph Lameter 已提交
980 981 982 983 984

		dump_stack();
	}
}

985
/*
C
Christoph Lameter 已提交
986
 * Tracking of fully allocated slabs for debugging purposes.
987
 */
988 989
static void add_full(struct kmem_cache *s,
	struct kmem_cache_node *n, struct page *page)
990
{
991 992 993
	if (!(s->flags & SLAB_STORE_USER))
		return;

994
	lockdep_assert_held(&n->list_lock);
995 996 997
	list_add(&page->lru, &n->full);
}

P
Peter Zijlstra 已提交
998
static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
999 1000 1001 1002
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

1003
	lockdep_assert_held(&n->list_lock);
1004 1005 1006
	list_del(&page->lru);
}

1007 1008 1009 1010 1011 1012 1013 1014
/* Tracking of the number of slabs for debugging purposes */
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
{
	struct kmem_cache_node *n = get_node(s, node);

	return atomic_long_read(&n->nr_slabs);
}

1015 1016 1017 1018 1019
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

1020
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1021 1022 1023 1024 1025 1026 1027 1028 1029
{
	struct kmem_cache_node *n = get_node(s, node);

	/*
	 * May be called early in order to allocate a slab for the
	 * kmem_cache_node structure. Solve the chicken-egg
	 * dilemma by deferring the increment of the count during
	 * bootstrap (see early_kmem_cache_node_alloc).
	 */
1030
	if (likely(n)) {
1031
		atomic_long_inc(&n->nr_slabs);
1032 1033
		atomic_long_add(objects, &n->total_objects);
	}
1034
}
1035
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1036 1037 1038 1039
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
1040
	atomic_long_sub(objects, &n->total_objects);
1041 1042 1043
}

/* Object debug checks for alloc/free paths */
C
Christoph Lameter 已提交
1044 1045 1046 1047 1048 1049
static void setup_object_debug(struct kmem_cache *s, struct page *page,
								void *object)
{
	if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
		return;

1050
	init_object(s, object, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1051 1052 1053
	init_tracking(s, object);
}

1054
static inline int alloc_consistency_checks(struct kmem_cache *s,
1055
					struct page *page,
1056
					void *object, unsigned long addr)
C
Christoph Lameter 已提交
1057 1058
{
	if (!check_slab(s, page))
1059
		return 0;
C
Christoph Lameter 已提交
1060 1061 1062

	if (!check_valid_pointer(s, page, object)) {
		object_err(s, page, object, "Freelist Pointer check fails");
1063
		return 0;
C
Christoph Lameter 已提交
1064 1065
	}

1066
	if (!check_object(s, page, object, SLUB_RED_INACTIVE))
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
		return 0;

	return 1;
}

static noinline int alloc_debug_processing(struct kmem_cache *s,
					struct page *page,
					void *object, unsigned long addr)
{
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!alloc_consistency_checks(s, page, object, addr))
			goto bad;
	}
C
Christoph Lameter 已提交
1080

C
Christoph Lameter 已提交
1081 1082 1083 1084
	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
	trace(s, page, object, 1);
1085
	init_object(s, object, SLUB_RED_ACTIVE);
C
Christoph Lameter 已提交
1086
	return 1;
C
Christoph Lameter 已提交
1087

C
Christoph Lameter 已提交
1088 1089 1090 1091 1092
bad:
	if (PageSlab(page)) {
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
C
Christoph Lameter 已提交
1093
		 * as used avoids touching the remaining objects.
C
Christoph Lameter 已提交
1094
		 */
1095
		slab_fix(s, "Marking all objects used");
1096
		page->inuse = page->objects;
1097
		page->freelist = NULL;
C
Christoph Lameter 已提交
1098 1099 1100 1101
	}
	return 0;
}

1102 1103
static inline int free_consistency_checks(struct kmem_cache *s,
		struct page *page, void *object, unsigned long addr)
C
Christoph Lameter 已提交
1104 1105
{
	if (!check_valid_pointer(s, page, object)) {
1106
		slab_err(s, page, "Invalid object pointer 0x%p", object);
1107
		return 0;
C
Christoph Lameter 已提交
1108 1109 1110
	}

	if (on_freelist(s, page, object)) {
1111
		object_err(s, page, object, "Object already free");
1112
		return 0;
C
Christoph Lameter 已提交
1113 1114
	}

1115
	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1116
		return 0;
C
Christoph Lameter 已提交
1117

1118
	if (unlikely(s != page->slab_cache)) {
I
Ingo Molnar 已提交
1119
		if (!PageSlab(page)) {
J
Joe Perches 已提交
1120 1121
			slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
				 object);
1122
		} else if (!page->slab_cache) {
1123 1124
			pr_err("SLUB <none>: no slab for object 0x%p.\n",
			       object);
1125
			dump_stack();
P
Pekka Enberg 已提交
1126
		} else
1127 1128
			object_err(s, page, object,
					"page slab pointer corrupt.");
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
		return 0;
	}
	return 1;
}

/* Supports checking bulk free of a constructed freelist */
static noinline int free_debug_processing(
	struct kmem_cache *s, struct page *page,
	void *head, void *tail, int bulk_cnt,
	unsigned long addr)
{
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	void *object = head;
	int cnt = 0;
	unsigned long uninitialized_var(flags);
	int ret = 0;

	spin_lock_irqsave(&n->list_lock, flags);
	slab_lock(page);

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!check_slab(s, page))
			goto out;
	}

next_object:
	cnt++;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!free_consistency_checks(s, page, object, addr))
			goto out;
C
Christoph Lameter 已提交
1160
	}
C
Christoph Lameter 已提交
1161 1162 1163 1164

	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
1165
	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
1166
	init_object(s, object, SLUB_RED_INACTIVE);
1167 1168 1169 1170 1171 1172

	/* Reached end of constructed freelist yet? */
	if (object != tail) {
		object = get_freepointer(s, object);
		goto next_object;
	}
1173 1174
	ret = 1;

1175
out:
1176 1177 1178 1179
	if (cnt != bulk_cnt)
		slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
			 bulk_cnt, cnt);

1180
	slab_unlock(page);
1181
	spin_unlock_irqrestore(&n->list_lock, flags);
1182 1183 1184
	if (!ret)
		slab_fix(s, "Object at 0x%p not freed", object);
	return ret;
C
Christoph Lameter 已提交
1185 1186
}

C
Christoph Lameter 已提交
1187 1188
static int __init setup_slub_debug(char *str)
{
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
	slub_debug = DEBUG_DEFAULT_FLAGS;
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	if (*str == ',')
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
		goto check_slabs;

	slub_debug = 0;
	if (*str == '-')
		/*
		 * Switch off all debugging measures.
		 */
		goto out;

	/*
	 * Determine which debug features should be switched on
	 */
P
Pekka Enberg 已提交
1213
	for (; *str && *str != ','; str++) {
1214 1215
		switch (tolower(*str)) {
		case 'f':
1216
			slub_debug |= SLAB_CONSISTENCY_CHECKS;
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229
			break;
		case 'z':
			slub_debug |= SLAB_RED_ZONE;
			break;
		case 'p':
			slub_debug |= SLAB_POISON;
			break;
		case 'u':
			slub_debug |= SLAB_STORE_USER;
			break;
		case 't':
			slub_debug |= SLAB_TRACE;
			break;
1230 1231 1232
		case 'a':
			slub_debug |= SLAB_FAILSLAB;
			break;
1233 1234 1235 1236 1237 1238 1239
		case 'o':
			/*
			 * Avoid enabling debugging on caches if its minimum
			 * order would increase as a result.
			 */
			disable_higher_order_debug = 1;
			break;
1240
		default:
1241 1242
			pr_err("slub_debug option '%c' unknown. skipped\n",
			       *str);
1243
		}
C
Christoph Lameter 已提交
1244 1245
	}

1246
check_slabs:
C
Christoph Lameter 已提交
1247 1248
	if (*str == ',')
		slub_debug_slabs = str + 1;
1249
out:
C
Christoph Lameter 已提交
1250 1251 1252 1253 1254
	return 1;
}

__setup("slub_debug", setup_slub_debug);

1255
unsigned long kmem_cache_flags(unsigned long object_size,
1256
	unsigned long flags, const char *name,
1257
	void (*ctor)(void *))
C
Christoph Lameter 已提交
1258 1259
{
	/*
1260
	 * Enable debugging if selected on the kernel commandline.
C
Christoph Lameter 已提交
1261
	 */
1262 1263
	if (slub_debug && (!slub_debug_slabs || (name &&
		!strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)))))
1264
		flags |= slub_debug;
1265 1266

	return flags;
C
Christoph Lameter 已提交
1267
}
1268
#else /* !CONFIG_SLUB_DEBUG */
C
Christoph Lameter 已提交
1269 1270
static inline void setup_object_debug(struct kmem_cache *s,
			struct page *page, void *object) {}
C
Christoph Lameter 已提交
1271

C
Christoph Lameter 已提交
1272
static inline int alloc_debug_processing(struct kmem_cache *s,
1273
	struct page *page, void *object, unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1274

1275
static inline int free_debug_processing(
1276 1277
	struct kmem_cache *s, struct page *page,
	void *head, void *tail, int bulk_cnt,
1278
	unsigned long addr) { return 0; }
C
Christoph Lameter 已提交
1279 1280 1281 1282

static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
			{ return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page,
1283
			void *object, u8 val) { return 1; }
1284 1285
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
					struct page *page) {}
P
Peter Zijlstra 已提交
1286 1287
static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
					struct page *page) {}
1288
unsigned long kmem_cache_flags(unsigned long object_size,
1289
	unsigned long flags, const char *name,
1290
	void (*ctor)(void *))
1291 1292 1293
{
	return flags;
}
C
Christoph Lameter 已提交
1294
#define slub_debug 0
1295

1296 1297
#define disable_higher_order_debug 0

1298 1299
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }
1300 1301
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1302 1303 1304 1305
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1306

1307 1308 1309 1310 1311 1312
#endif /* CONFIG_SLUB_DEBUG */

/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
 */
1313 1314 1315
static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
{
	kmemleak_alloc(ptr, size, 1, flags);
1316
	kasan_kmalloc_large(ptr, size, flags);
1317 1318 1319 1320 1321
}

static inline void kfree_hook(const void *x)
{
	kmemleak_free(x);
1322
	kasan_kfree_large(x);
1323 1324 1325 1326 1327
}

static inline void slab_free_hook(struct kmem_cache *s, void *x)
{
	kmemleak_free_recursive(x, s->flags);
1328

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
	/*
	 * Trouble is that we may no longer disable interrupts in the fast path
	 * So in order to make the debug calls that expect irqs to be
	 * disabled we need to disable interrupts temporarily.
	 */
#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
	{
		unsigned long flags;

		local_irq_save(flags);
		kmemcheck_slab_free(s, x, s->object_size);
		debug_check_no_locks_freed(x, s->object_size);
		local_irq_restore(flags);
	}
#endif
	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->object_size);
1346 1347

	kasan_slab_free(s, x);
1348
}
1349

1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
static inline void slab_free_freelist_hook(struct kmem_cache *s,
					   void *head, void *tail)
{
/*
 * Compiler cannot detect this function can be removed if slab_free_hook()
 * evaluates to nothing.  Thus, catch all relevant config debug options here.
 */
#if defined(CONFIG_KMEMCHECK) ||		\
	defined(CONFIG_LOCKDEP)	||		\
	defined(CONFIG_DEBUG_KMEMLEAK) ||	\
	defined(CONFIG_DEBUG_OBJECTS_FREE) ||	\
	defined(CONFIG_KASAN)

	void *object = head;
	void *tail_obj = tail ? : head;

	do {
		slab_free_hook(s, object);
	} while ((object != tail_obj) &&
		 (object = get_freepointer(s, object)));
#endif
}

1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
static void setup_object(struct kmem_cache *s, struct page *page,
				void *object)
{
	setup_object_debug(s, page, object);
	if (unlikely(s->ctor)) {
		kasan_unpoison_object_data(s, object);
		s->ctor(object);
		kasan_poison_object_data(s, object);
	}
}

C
Christoph Lameter 已提交
1384 1385 1386
/*
 * Slab allocation and freeing
 */
1387 1388
static inline struct page *alloc_slab_page(struct kmem_cache *s,
		gfp_t flags, int node, struct kmem_cache_order_objects oo)
1389
{
1390
	struct page *page;
1391 1392
	int order = oo_order(oo);

1393 1394
	flags |= __GFP_NOTRACK;

1395
	if (node == NUMA_NO_NODE)
1396
		page = alloc_pages(flags, order);
1397
	else
1398
		page = __alloc_pages_node(node, flags, order);
1399

1400 1401 1402 1403
	if (page && memcg_charge_slab(page, flags, order, s)) {
		__free_pages(page, order);
		page = NULL;
	}
1404 1405

	return page;
1406 1407
}

C
Christoph Lameter 已提交
1408 1409
static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
{
P
Pekka Enberg 已提交
1410
	struct page *page;
1411
	struct kmem_cache_order_objects oo = s->oo;
1412
	gfp_t alloc_gfp;
1413 1414
	void *start, *p;
	int idx, order;
C
Christoph Lameter 已提交
1415

1416 1417
	flags &= gfp_allowed_mask;

1418
	if (gfpflags_allow_blocking(flags))
1419 1420
		local_irq_enable();

1421
	flags |= s->allocflags;
1422

1423 1424 1425 1426 1427
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1428
	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1429
		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
1430

1431
	page = alloc_slab_page(s, alloc_gfp, node, oo);
1432 1433
	if (unlikely(!page)) {
		oo = s->min;
1434
		alloc_gfp = flags;
1435 1436 1437 1438
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
1439
		page = alloc_slab_page(s, alloc_gfp, node, oo);
1440 1441 1442
		if (unlikely(!page))
			goto out;
		stat(s, ORDER_FALLBACK);
1443
	}
V
Vegard Nossum 已提交
1444

1445 1446
	if (kmemcheck_enabled &&
	    !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
1447 1448
		int pages = 1 << oo_order(oo);

1449
		kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
1450 1451 1452 1453 1454 1455 1456 1457 1458

		/*
		 * Objects from caches that have a constructor don't get
		 * cleared when they're allocated, so we need to do it here.
		 */
		if (s->ctor)
			kmemcheck_mark_uninitialized_pages(page, pages);
		else
			kmemcheck_mark_unallocated_pages(page, pages);
V
Vegard Nossum 已提交
1459 1460
	}

1461
	page->objects = oo_objects(oo);
C
Christoph Lameter 已提交
1462

G
Glauber Costa 已提交
1463
	order = compound_order(page);
1464
	page->slab_cache = s;
1465
	__SetPageSlab(page);
1466
	if (page_is_pfmemalloc(page))
1467
		SetPageSlabPfmemalloc(page);
C
Christoph Lameter 已提交
1468 1469 1470 1471

	start = page_address(page);

	if (unlikely(s->flags & SLAB_POISON))
G
Glauber Costa 已提交
1472
		memset(start, POISON_INUSE, PAGE_SIZE << order);
C
Christoph Lameter 已提交
1473

1474 1475
	kasan_poison_slab(page);

1476 1477 1478 1479 1480 1481
	for_each_object_idx(p, idx, s, start, page->objects) {
		setup_object(s, page, p);
		if (likely(idx < page->objects))
			set_freepointer(s, p, p + s->size);
		else
			set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
1482 1483
	}

J
Joonsoo Kim 已提交
1484
	page->freelist = fixup_red_left(s, start);
1485
	page->inuse = page->objects;
1486
	page->frozen = 1;
1487

C
Christoph Lameter 已提交
1488
out:
1489
	if (gfpflags_allow_blocking(flags))
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
		local_irq_disable();
	if (!page)
		return NULL;

	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
		1 << oo_order(oo));

	inc_slabs_node(s, page_to_nid(page), page->objects);

C
Christoph Lameter 已提交
1501 1502 1503
	return page;
}

1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514
static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
	if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
		pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
		BUG();
	}

	return allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
}

C
Christoph Lameter 已提交
1515 1516
static void __free_slab(struct kmem_cache *s, struct page *page)
{
1517 1518
	int order = compound_order(page);
	int pages = 1 << order;
C
Christoph Lameter 已提交
1519

1520
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
C
Christoph Lameter 已提交
1521 1522 1523
		void *p;

		slab_pad_check(s, page);
1524 1525
		for_each_object(p, s, page_address(page),
						page->objects)
1526
			check_object(s, page, p, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1527 1528
	}

1529
	kmemcheck_free_shadow(page, compound_order(page));
V
Vegard Nossum 已提交
1530

C
Christoph Lameter 已提交
1531 1532 1533
	mod_zone_page_state(page_zone(page),
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
P
Pekka Enberg 已提交
1534
		-pages);
C
Christoph Lameter 已提交
1535

1536
	__ClearPageSlabPfmemalloc(page);
1537
	__ClearPageSlab(page);
G
Glauber Costa 已提交
1538

1539
	page_mapcount_reset(page);
N
Nick Piggin 已提交
1540 1541
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
1542 1543
	memcg_uncharge_slab(page, order, s);
	__free_pages(page, order);
C
Christoph Lameter 已提交
1544 1545
}

1546 1547 1548
#define need_reserve_slab_rcu						\
	(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))

C
Christoph Lameter 已提交
1549 1550 1551 1552
static void rcu_free_slab(struct rcu_head *h)
{
	struct page *page;

1553 1554 1555 1556 1557
	if (need_reserve_slab_rcu)
		page = virt_to_head_page(h);
	else
		page = container_of((struct list_head *)h, struct page, lru);

1558
	__free_slab(page->slab_cache, page);
C
Christoph Lameter 已提交
1559 1560 1561 1562 1563
}

static void free_slab(struct kmem_cache *s, struct page *page)
{
	if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1564 1565 1566 1567 1568 1569 1570 1571 1572
		struct rcu_head *head;

		if (need_reserve_slab_rcu) {
			int order = compound_order(page);
			int offset = (PAGE_SIZE << order) - s->reserved;

			VM_BUG_ON(s->reserved != sizeof(*head));
			head = page_address(page) + offset;
		} else {
1573
			head = &page->rcu_head;
1574
		}
C
Christoph Lameter 已提交
1575 1576 1577 1578 1579 1580 1581 1582

		call_rcu(head, rcu_free_slab);
	} else
		__free_slab(s, page);
}

static void discard_slab(struct kmem_cache *s, struct page *page)
{
1583
	dec_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1584 1585 1586 1587
	free_slab(s, page);
}

/*
1588
 * Management of partially allocated slabs.
C
Christoph Lameter 已提交
1589
 */
1590 1591
static inline void
__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
C
Christoph Lameter 已提交
1592
{
C
Christoph Lameter 已提交
1593
	n->nr_partial++;
1594
	if (tail == DEACTIVATE_TO_TAIL)
1595 1596 1597
		list_add_tail(&page->lru, &n->partial);
	else
		list_add(&page->lru, &n->partial);
C
Christoph Lameter 已提交
1598 1599
}

1600 1601
static inline void add_partial(struct kmem_cache_node *n,
				struct page *page, int tail)
1602
{
P
Peter Zijlstra 已提交
1603
	lockdep_assert_held(&n->list_lock);
1604 1605
	__add_partial(n, page, tail);
}
P
Peter Zijlstra 已提交
1606

1607 1608 1609 1610
static inline void remove_partial(struct kmem_cache_node *n,
					struct page *page)
{
	lockdep_assert_held(&n->list_lock);
1611 1612
	list_del(&page->lru);
	n->nr_partial--;
1613 1614
}

C
Christoph Lameter 已提交
1615
/*
1616 1617
 * Remove slab from the partial list, freeze it and
 * return the pointer to the freelist.
C
Christoph Lameter 已提交
1618
 *
1619
 * Returns a list of objects or NULL if it fails.
C
Christoph Lameter 已提交
1620
 */
1621
static inline void *acquire_slab(struct kmem_cache *s,
1622
		struct kmem_cache_node *n, struct page *page,
1623
		int mode, int *objects)
C
Christoph Lameter 已提交
1624
{
1625 1626 1627 1628
	void *freelist;
	unsigned long counters;
	struct page new;

P
Peter Zijlstra 已提交
1629 1630
	lockdep_assert_held(&n->list_lock);

1631 1632 1633 1634 1635
	/*
	 * Zap the freelist and set the frozen bit.
	 * The old freelist is the list of objects for the
	 * per cpu allocation list.
	 */
1636 1637 1638
	freelist = page->freelist;
	counters = page->counters;
	new.counters = counters;
1639
	*objects = new.objects - new.inuse;
1640
	if (mode) {
1641
		new.inuse = page->objects;
1642 1643 1644 1645
		new.freelist = NULL;
	} else {
		new.freelist = freelist;
	}
1646

1647
	VM_BUG_ON(new.frozen);
1648
	new.frozen = 1;
1649

1650
	if (!__cmpxchg_double_slab(s, page,
1651
			freelist, counters,
1652
			new.freelist, new.counters,
1653 1654
			"acquire_slab"))
		return NULL;
1655 1656

	remove_partial(n, page);
1657
	WARN_ON(!freelist);
1658
	return freelist;
C
Christoph Lameter 已提交
1659 1660
}

1661
static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
1662
static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
1663

C
Christoph Lameter 已提交
1664
/*
C
Christoph Lameter 已提交
1665
 * Try to allocate a partial slab from a specific node.
C
Christoph Lameter 已提交
1666
 */
1667 1668
static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
				struct kmem_cache_cpu *c, gfp_t flags)
C
Christoph Lameter 已提交
1669
{
1670 1671
	struct page *page, *page2;
	void *object = NULL;
1672 1673
	int available = 0;
	int objects;
C
Christoph Lameter 已提交
1674 1675 1676 1677

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
C
Christoph Lameter 已提交
1678 1679
	 * partial slab and there is none available then get_partials()
	 * will return NULL.
C
Christoph Lameter 已提交
1680 1681 1682 1683 1684
	 */
	if (!n || !n->nr_partial)
		return NULL;

	spin_lock(&n->list_lock);
1685
	list_for_each_entry_safe(page, page2, &n->partial, lru) {
1686
		void *t;
1687

1688 1689 1690
		if (!pfmemalloc_match(page, flags))
			continue;

1691
		t = acquire_slab(s, n, page, object == NULL, &objects);
1692 1693 1694
		if (!t)
			break;

1695
		available += objects;
1696
		if (!object) {
1697 1698 1699 1700
			c->page = page;
			stat(s, ALLOC_FROM_PARTIAL);
			object = t;
		} else {
1701
			put_cpu_partial(s, page, 0);
1702
			stat(s, CPU_PARTIAL_NODE);
1703
		}
1704 1705
		if (!kmem_cache_has_cpu_partial(s)
			|| available > s->cpu_partial / 2)
1706 1707
			break;

1708
	}
C
Christoph Lameter 已提交
1709
	spin_unlock(&n->list_lock);
1710
	return object;
C
Christoph Lameter 已提交
1711 1712 1713
}

/*
C
Christoph Lameter 已提交
1714
 * Get a page from somewhere. Search in increasing NUMA distances.
C
Christoph Lameter 已提交
1715
 */
1716
static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
1717
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1718 1719 1720
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
1721
	struct zoneref *z;
1722 1723
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
1724
	void *object;
1725
	unsigned int cpuset_mems_cookie;
C
Christoph Lameter 已提交
1726 1727

	/*
C
Christoph Lameter 已提交
1728 1729 1730 1731
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
C
Christoph Lameter 已提交
1732
	 *
C
Christoph Lameter 已提交
1733 1734 1735 1736
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
C
Christoph Lameter 已提交
1737
	 *
C
Christoph Lameter 已提交
1738
	 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
C
Christoph Lameter 已提交
1739 1740 1741 1742 1743
	 * defrag_ratio = 1000) then every (well almost) allocation will
	 * first attempt to defrag slab caches on other nodes. This means
	 * scanning over all nodes to look for partial slabs which may be
	 * expensive if we do it every time we are trying to find a slab
	 * with available objects.
C
Christoph Lameter 已提交
1744
	 */
1745 1746
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
C
Christoph Lameter 已提交
1747 1748
		return NULL;

1749
	do {
1750
		cpuset_mems_cookie = read_mems_allowed_begin();
1751
		zonelist = node_zonelist(mempolicy_slab_node(), flags);
1752 1753 1754 1755 1756
		for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
			struct kmem_cache_node *n;

			n = get_node(s, zone_to_nid(zone));

1757
			if (n && cpuset_zone_allowed(zone, flags) &&
1758
					n->nr_partial > s->min_partial) {
1759
				object = get_partial_node(s, n, c, flags);
1760 1761
				if (object) {
					/*
1762 1763 1764 1765 1766
					 * Don't check read_mems_allowed_retry()
					 * here - if mems_allowed was updated in
					 * parallel, that was a harmless race
					 * between allocation and the cpuset
					 * update
1767 1768 1769
					 */
					return object;
				}
1770
			}
C
Christoph Lameter 已提交
1771
		}
1772
	} while (read_mems_allowed_retry(cpuset_mems_cookie));
C
Christoph Lameter 已提交
1773 1774 1775 1776 1777 1778 1779
#endif
	return NULL;
}

/*
 * Get a partial page, lock it and return it.
 */
1780
static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
1781
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1782
{
1783
	void *object;
1784 1785 1786 1787 1788 1789
	int searchnode = node;

	if (node == NUMA_NO_NODE)
		searchnode = numa_mem_id();
	else if (!node_present_pages(node))
		searchnode = node_to_mem_node(node);
C
Christoph Lameter 已提交
1790

1791
	object = get_partial_node(s, get_node(s, searchnode), c, flags);
1792 1793
	if (object || node != NUMA_NO_NODE)
		return object;
C
Christoph Lameter 已提交
1794

1795
	return get_any_partial(s, flags, c);
C
Christoph Lameter 已提交
1796 1797
}

1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838
#ifdef CONFIG_PREEMPT
/*
 * Calculate the next globally unique transaction for disambiguiation
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
#endif

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

1839
	pr_info("%s %s: cmpxchg redo ", n, s->name);
1840 1841 1842

#ifdef CONFIG_PREEMPT
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
1843
		pr_warn("due to cpu change %d -> %d\n",
1844 1845 1846 1847
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
1848
		pr_warn("due to cpu running other code. Event %ld->%ld\n",
1849 1850
			tid_to_event(tid), tid_to_event(actual_tid));
	else
1851
		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
1852 1853
			actual_tid, tid, next_tid(tid));
#endif
1854
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1855 1856
}

1857
static void init_kmem_cache_cpus(struct kmem_cache *s)
1858 1859 1860 1861 1862 1863
{
	int cpu;

	for_each_possible_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
}
1864

C
Christoph Lameter 已提交
1865 1866 1867
/*
 * Remove the cpu slab
 */
1868 1869
static void deactivate_slab(struct kmem_cache *s, struct page *page,
				void *freelist)
C
Christoph Lameter 已提交
1870
{
1871 1872 1873 1874 1875
	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	int lock = 0;
	enum slab_modes l = M_NONE, m = M_NONE;
	void *nextfree;
1876
	int tail = DEACTIVATE_TO_HEAD;
1877 1878 1879 1880
	struct page new;
	struct page old;

	if (page->freelist) {
1881
		stat(s, DEACTIVATE_REMOTE_FREES);
1882
		tail = DEACTIVATE_TO_TAIL;
1883 1884
	}

1885
	/*
1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
	 * Stage one: Free all available per cpu objects back
	 * to the page freelist while it is still frozen. Leave the
	 * last one.
	 *
	 * There is no need to take the list->lock because the page
	 * is still frozen.
	 */
	while (freelist && (nextfree = get_freepointer(s, freelist))) {
		void *prior;
		unsigned long counters;

		do {
			prior = page->freelist;
			counters = page->counters;
			set_freepointer(s, freelist, prior);
			new.counters = counters;
			new.inuse--;
1903
			VM_BUG_ON(!new.frozen);
1904

1905
		} while (!__cmpxchg_double_slab(s, page,
1906 1907 1908 1909 1910 1911 1912
			prior, counters,
			freelist, new.counters,
			"drain percpu freelist"));

		freelist = nextfree;
	}

1913
	/*
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
	 * Stage two: Ensure that the page is unfrozen while the
	 * list presence reflects the actual number of objects
	 * during unfreeze.
	 *
	 * We setup the list membership and then perform a cmpxchg
	 * with the count. If there is a mismatch then the page
	 * is not unfrozen but the page is on the wrong list.
	 *
	 * Then we restart the process which may have to remove
	 * the page from the list that we just put it on again
	 * because the number of objects in the slab may have
	 * changed.
1926
	 */
1927
redo:
1928

1929 1930
	old.freelist = page->freelist;
	old.counters = page->counters;
1931
	VM_BUG_ON(!old.frozen);
1932

1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
	/* Determine target state of the slab */
	new.counters = old.counters;
	if (freelist) {
		new.inuse--;
		set_freepointer(s, freelist, old.freelist);
		new.freelist = freelist;
	} else
		new.freelist = old.freelist;

	new.frozen = 0;

1944
	if (!new.inuse && n->nr_partial >= s->min_partial)
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
		m = M_FREE;
	else if (new.freelist) {
		m = M_PARTIAL;
		if (!lock) {
			lock = 1;
			/*
			 * Taking the spinlock removes the possiblity
			 * that acquire_slab() will see a slab page that
			 * is frozen
			 */
			spin_lock(&n->list_lock);
		}
	} else {
		m = M_FULL;
		if (kmem_cache_debug(s) && !lock) {
			lock = 1;
			/*
			 * This also ensures that the scanning of full
			 * slabs from diagnostic functions will not see
			 * any frozen slabs.
			 */
			spin_lock(&n->list_lock);
		}
	}

	if (l != m) {

		if (l == M_PARTIAL)

			remove_partial(n, page);

		else if (l == M_FULL)
1977

P
Peter Zijlstra 已提交
1978
			remove_full(s, n, page);
1979 1980 1981 1982

		if (m == M_PARTIAL) {

			add_partial(n, page, tail);
1983
			stat(s, tail);
1984 1985

		} else if (m == M_FULL) {
1986

1987 1988 1989 1990 1991 1992 1993
			stat(s, DEACTIVATE_FULL);
			add_full(s, n, page);

		}
	}

	l = m;
1994
	if (!__cmpxchg_double_slab(s, page,
1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"))
		goto redo;

	if (lock)
		spin_unlock(&n->list_lock);

	if (m == M_FREE) {
		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
2007
	}
C
Christoph Lameter 已提交
2008 2009
}

2010 2011 2012
/*
 * Unfreeze all the cpu partial slabs.
 *
2013 2014 2015
 * This function must be called with interrupts disabled
 * for the cpu using c (or some other guarantee must be there
 * to guarantee no concurrent accesses).
2016
 */
2017 2018
static void unfreeze_partials(struct kmem_cache *s,
		struct kmem_cache_cpu *c)
2019
{
2020
#ifdef CONFIG_SLUB_CPU_PARTIAL
2021
	struct kmem_cache_node *n = NULL, *n2 = NULL;
2022
	struct page *page, *discard_page = NULL;
2023 2024 2025 2026 2027 2028

	while ((page = c->partial)) {
		struct page new;
		struct page old;

		c->partial = page->next;
2029 2030 2031 2032 2033 2034 2035 2036 2037

		n2 = get_node(s, page_to_nid(page));
		if (n != n2) {
			if (n)
				spin_unlock(&n->list_lock);

			n = n2;
			spin_lock(&n->list_lock);
		}
2038 2039 2040 2041 2042

		do {

			old.freelist = page->freelist;
			old.counters = page->counters;
2043
			VM_BUG_ON(!old.frozen);
2044 2045 2046 2047 2048 2049

			new.counters = old.counters;
			new.freelist = old.freelist;

			new.frozen = 0;

2050
		} while (!__cmpxchg_double_slab(s, page,
2051 2052 2053 2054
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"));

2055
		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
2056 2057
			page->next = discard_page;
			discard_page = page;
2058 2059 2060
		} else {
			add_partial(n, page, DEACTIVATE_TO_TAIL);
			stat(s, FREE_ADD_PARTIAL);
2061 2062 2063 2064 2065
		}
	}

	if (n)
		spin_unlock(&n->list_lock);
2066 2067 2068 2069 2070 2071 2072 2073 2074

	while (discard_page) {
		page = discard_page;
		discard_page = discard_page->next;

		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
	}
2075
#endif
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086
}

/*
 * Put a page that was just frozen (in __slab_free) into a partial page
 * slot if available. This is done without interrupts disabled and without
 * preemption disabled. The cmpxchg is racy and may put the partial page
 * onto a random cpus partial slot.
 *
 * If we did not find a slot then simply move all the partials to the
 * per node partial list.
 */
2087
static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
2088
{
2089
#ifdef CONFIG_SLUB_CPU_PARTIAL
2090 2091 2092 2093
	struct page *oldpage;
	int pages;
	int pobjects;

2094
	preempt_disable();
2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	do {
		pages = 0;
		pobjects = 0;
		oldpage = this_cpu_read(s->cpu_slab->partial);

		if (oldpage) {
			pobjects = oldpage->pobjects;
			pages = oldpage->pages;
			if (drain && pobjects > s->cpu_partial) {
				unsigned long flags;
				/*
				 * partial array is full. Move the existing
				 * set to the per node partial list.
				 */
				local_irq_save(flags);
2110
				unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2111
				local_irq_restore(flags);
2112
				oldpage = NULL;
2113 2114
				pobjects = 0;
				pages = 0;
2115
				stat(s, CPU_PARTIAL_DRAIN);
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
			}
		}

		pages++;
		pobjects += page->objects - page->inuse;

		page->pages = pages;
		page->pobjects = pobjects;
		page->next = oldpage;

2126 2127
	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
								!= oldpage);
2128 2129 2130 2131 2132 2133 2134 2135
	if (unlikely(!s->cpu_partial)) {
		unsigned long flags;

		local_irq_save(flags);
		unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
		local_irq_restore(flags);
	}
	preempt_enable();
2136
#endif
2137 2138
}

2139
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2140
{
2141
	stat(s, CPUSLAB_FLUSH);
2142 2143 2144 2145 2146
	deactivate_slab(s, c->page, c->freelist);

	c->tid = next_tid(c->tid);
	c->page = NULL;
	c->freelist = NULL;
C
Christoph Lameter 已提交
2147 2148 2149 2150
}

/*
 * Flush cpu slab.
C
Christoph Lameter 已提交
2151
 *
C
Christoph Lameter 已提交
2152 2153
 * Called from IPI handler with interrupts disabled.
 */
2154
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
C
Christoph Lameter 已提交
2155
{
2156
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
C
Christoph Lameter 已提交
2157

2158 2159 2160 2161
	if (likely(c)) {
		if (c->page)
			flush_slab(s, c);

2162
		unfreeze_partials(s, c);
2163
	}
C
Christoph Lameter 已提交
2164 2165 2166 2167 2168 2169
}

static void flush_cpu_slab(void *d)
{
	struct kmem_cache *s = d;

2170
	__flush_cpu_slab(s, smp_processor_id());
C
Christoph Lameter 已提交
2171 2172
}

2173 2174 2175 2176 2177
static bool has_cpu_slab(int cpu, void *info)
{
	struct kmem_cache *s = info;
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

2178
	return c->page || c->partial;
2179 2180
}

C
Christoph Lameter 已提交
2181 2182
static void flush_all(struct kmem_cache *s)
{
2183
	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
C
Christoph Lameter 已提交
2184 2185
}

2186 2187 2188 2189
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
2190
static inline int node_match(struct page *page, int node)
2191 2192
{
#ifdef CONFIG_NUMA
2193
	if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node))
2194 2195 2196 2197 2198
		return 0;
#endif
	return 1;
}

2199
#ifdef CONFIG_SLUB_DEBUG
P
Pekka Enberg 已提交
2200 2201 2202 2203 2204
static int count_free(struct page *page)
{
	return page->objects - page->inuse;
}

2205 2206 2207 2208 2209 2210 2211
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->total_objects);
}
#endif /* CONFIG_SLUB_DEBUG */

#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
P
Pekka Enberg 已提交
2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224
static unsigned long count_partial(struct kmem_cache_node *n,
					int (*get_count)(struct page *))
{
	unsigned long flags;
	unsigned long x = 0;
	struct page *page;

	spin_lock_irqsave(&n->list_lock, flags);
	list_for_each_entry(page, &n->partial, lru)
		x += get_count(page);
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}
2225
#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
2226

P
Pekka Enberg 已提交
2227 2228 2229
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
2230 2231 2232
#ifdef CONFIG_SLUB_DEBUG
	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);
P
Pekka Enberg 已提交
2233
	int node;
C
Christoph Lameter 已提交
2234
	struct kmem_cache_node *n;
P
Pekka Enberg 已提交
2235

2236 2237 2238
	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
		return;

2239 2240
	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
		nid, gfpflags, &gfpflags);
2241 2242 2243
	pr_warn("  cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n",
		s->name, s->object_size, s->size, oo_order(s->oo),
		oo_order(s->min));
P
Pekka Enberg 已提交
2244

2245
	if (oo_order(s->min) > get_order(s->object_size))
2246 2247
		pr_warn("  %s debugging increased min order, use slub_debug=O to disable.\n",
			s->name);
2248

C
Christoph Lameter 已提交
2249
	for_each_kmem_cache_node(s, node, n) {
P
Pekka Enberg 已提交
2250 2251 2252 2253
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

2254 2255 2256
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
P
Pekka Enberg 已提交
2257

2258
		pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
P
Pekka Enberg 已提交
2259 2260
			node, nr_slabs, nr_objs, nr_free);
	}
2261
#endif
P
Pekka Enberg 已提交
2262 2263
}

2264 2265 2266
static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
			int node, struct kmem_cache_cpu **pc)
{
2267
	void *freelist;
2268 2269
	struct kmem_cache_cpu *c = *pc;
	struct page *page;
2270

2271
	freelist = get_partial(s, flags, node, c);
2272

2273 2274 2275 2276
	if (freelist)
		return freelist;

	page = new_slab(s, flags, node);
2277
	if (page) {
2278
		c = raw_cpu_ptr(s->cpu_slab);
2279 2280 2281 2282 2283 2284 2285
		if (c->page)
			flush_slab(s, c);

		/*
		 * No other reference to the page yet so we can
		 * muck around with it freely without cmpxchg
		 */
2286
		freelist = page->freelist;
2287 2288 2289 2290 2291 2292
		page->freelist = NULL;

		stat(s, ALLOC_SLAB);
		c->page = page;
		*pc = c;
	} else
2293
		freelist = NULL;
2294

2295
	return freelist;
2296 2297
}

2298 2299 2300 2301 2302 2303 2304 2305
static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
{
	if (unlikely(PageSlabPfmemalloc(page)))
		return gfp_pfmemalloc_allowed(gfpflags);

	return true;
}

2306
/*
2307 2308
 * Check the page->freelist of a page and either transfer the freelist to the
 * per cpu freelist or deactivate the page.
2309 2310 2311 2312
 *
 * The page is still frozen if the return value is not NULL.
 *
 * If this function returns NULL then the page has been unfrozen.
2313 2314
 *
 * This function must be called with interrupt disabled.
2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
 */
static inline void *get_freelist(struct kmem_cache *s, struct page *page)
{
	struct page new;
	unsigned long counters;
	void *freelist;

	do {
		freelist = page->freelist;
		counters = page->counters;
2325

2326
		new.counters = counters;
2327
		VM_BUG_ON(!new.frozen);
2328 2329 2330 2331

		new.inuse = page->objects;
		new.frozen = freelist != NULL;

2332
	} while (!__cmpxchg_double_slab(s, page,
2333 2334 2335 2336 2337 2338 2339
		freelist, counters,
		NULL, new.counters,
		"get_freelist"));

	return freelist;
}

C
Christoph Lameter 已提交
2340
/*
2341 2342 2343 2344 2345 2346
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
C
Christoph Lameter 已提交
2347
 *
2348 2349 2350
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
C
Christoph Lameter 已提交
2351
 *
2352
 * And if we were unable to get a new slab from the partial slab lists then
C
Christoph Lameter 已提交
2353 2354
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
2355 2356 2357
 *
 * Version of __slab_alloc to use when we know that interrupts are
 * already disabled (which is the case for bulk allocation).
C
Christoph Lameter 已提交
2358
 */
2359
static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2360
			  unsigned long addr, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2361
{
2362
	void *freelist;
2363
	struct page *page;
C
Christoph Lameter 已提交
2364

2365 2366
	page = c->page;
	if (!page)
C
Christoph Lameter 已提交
2367
		goto new_slab;
2368
redo:
2369

2370
	if (unlikely(!node_match(page, node))) {
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
		int searchnode = node;

		if (node != NUMA_NO_NODE && !node_present_pages(node))
			searchnode = node_to_mem_node(node);

		if (unlikely(!node_match(page, searchnode))) {
			stat(s, ALLOC_NODE_MISMATCH);
			deactivate_slab(s, page, c->freelist);
			c->page = NULL;
			c->freelist = NULL;
			goto new_slab;
		}
2383
	}
C
Christoph Lameter 已提交
2384

2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
	if (unlikely(!pfmemalloc_match(page, gfpflags))) {
		deactivate_slab(s, page, c->freelist);
		c->page = NULL;
		c->freelist = NULL;
		goto new_slab;
	}

2397
	/* must check again c->freelist in case of cpu migration or IRQ */
2398 2399
	freelist = c->freelist;
	if (freelist)
2400
		goto load_freelist;
2401

2402
	freelist = get_freelist(s, page);
C
Christoph Lameter 已提交
2403

2404
	if (!freelist) {
2405 2406
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
2407
		goto new_slab;
2408
	}
C
Christoph Lameter 已提交
2409

2410
	stat(s, ALLOC_REFILL);
C
Christoph Lameter 已提交
2411

2412
load_freelist:
2413 2414 2415 2416 2417
	/*
	 * freelist is pointing to the list of objects to be used.
	 * page is pointing to the page from which the objects are obtained.
	 * That page must be frozen for per cpu allocations to work.
	 */
2418
	VM_BUG_ON(!c->page->frozen);
2419
	c->freelist = get_freepointer(s, freelist);
2420
	c->tid = next_tid(c->tid);
2421
	return freelist;
C
Christoph Lameter 已提交
2422 2423

new_slab:
2424

2425
	if (c->partial) {
2426 2427
		page = c->page = c->partial;
		c->partial = page->next;
2428 2429 2430
		stat(s, CPU_PARTIAL_ALLOC);
		c->freelist = NULL;
		goto redo;
C
Christoph Lameter 已提交
2431 2432
	}

2433
	freelist = new_slab_objects(s, gfpflags, node, &c);
2434

2435
	if (unlikely(!freelist)) {
2436
		slab_out_of_memory(s, gfpflags, node);
2437
		return NULL;
C
Christoph Lameter 已提交
2438
	}
2439

2440
	page = c->page;
2441
	if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
2442
		goto load_freelist;
2443

2444
	/* Only entered in the debug case */
2445 2446
	if (kmem_cache_debug(s) &&
			!alloc_debug_processing(s, page, freelist, addr))
2447
		goto new_slab;	/* Slab failed checks. Next slab needed */
2448

2449
	deactivate_slab(s, page, get_freepointer(s, freelist));
2450 2451
	c->page = NULL;
	c->freelist = NULL;
2452
	return freelist;
2453 2454
}

2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479
/*
 * Another one that disabled interrupt and compensates for possible
 * cpu changes by refetching the per cpu area pointer.
 */
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *p;
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPT
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif

	p = ___slab_alloc(s, gfpflags, node, addr, c);
	local_irq_restore(flags);
	return p;
}

2480 2481 2482 2483 2484 2485 2486 2487 2488 2489
/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
2490
static __always_inline void *slab_alloc_node(struct kmem_cache *s,
2491
		gfp_t gfpflags, int node, unsigned long addr)
2492
{
2493
	void *object;
2494
	struct kmem_cache_cpu *c;
2495
	struct page *page;
2496
	unsigned long tid;
2497

2498 2499
	s = slab_pre_alloc_hook(s, gfpflags);
	if (!s)
A
Akinobu Mita 已提交
2500
		return NULL;
2501 2502 2503 2504 2505 2506
redo:
	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
2507
	 *
2508 2509 2510
	 * We should guarantee that tid and kmem_cache are retrieved on
	 * the same cpu. It could be different if CONFIG_PREEMPT so we need
	 * to check if it is matched or not.
2511
	 */
2512 2513 2514
	do {
		tid = this_cpu_read(s->cpu_slab->tid);
		c = raw_cpu_ptr(s->cpu_slab);
2515 2516
	} while (IS_ENABLED(CONFIG_PREEMPT) &&
		 unlikely(tid != READ_ONCE(c->tid)));
2517 2518 2519 2520 2521 2522 2523 2524 2525 2526

	/*
	 * Irqless object alloc/free algorithm used here depends on sequence
	 * of fetching cpu_slab's data. tid should be fetched before anything
	 * on c to guarantee that object and page associated with previous tid
	 * won't be used with current tid. If we fetch tid first, object and
	 * page could be one associated with next tid and our alloc/free
	 * request will be failed. In this case, we will retry. So, no problem.
	 */
	barrier();
2527 2528 2529 2530 2531 2532 2533 2534

	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */

2535
	object = c->freelist;
2536
	page = c->page;
D
Dave Hansen 已提交
2537
	if (unlikely(!object || !node_match(page, node))) {
2538
		object = __slab_alloc(s, gfpflags, node, addr, c);
D
Dave Hansen 已提交
2539 2540
		stat(s, ALLOC_SLOWPATH);
	} else {
2541 2542
		void *next_object = get_freepointer_safe(s, object);

2543
		/*
L
Lucas De Marchi 已提交
2544
		 * The cmpxchg will only match if there was no additional
2545 2546
		 * operation and if we are on the right processor.
		 *
2547 2548
		 * The cmpxchg does the following atomically (without lock
		 * semantics!)
2549 2550 2551 2552
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
2553 2554 2555
		 * Since this is without lock semantics the protection is only
		 * against code executing on this cpu *not* from access by
		 * other cpus.
2556
		 */
2557
		if (unlikely(!this_cpu_cmpxchg_double(
2558 2559
				s->cpu_slab->freelist, s->cpu_slab->tid,
				object, tid,
2560
				next_object, next_tid(tid)))) {
2561 2562 2563 2564

			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
2565
		prefetch_freepointer(s, next_object);
2566
		stat(s, ALLOC_FASTPATH);
2567
	}
2568

2569
	if (unlikely(gfpflags & __GFP_ZERO) && object)
2570
		memset(object, 0, s->object_size);
2571

2572
	slab_post_alloc_hook(s, gfpflags, 1, &object);
V
Vegard Nossum 已提交
2573

2574
	return object;
C
Christoph Lameter 已提交
2575 2576
}

2577 2578 2579 2580 2581 2582
static __always_inline void *slab_alloc(struct kmem_cache *s,
		gfp_t gfpflags, unsigned long addr)
{
	return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
}

C
Christoph Lameter 已提交
2583 2584
void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
2585
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2586

2587 2588
	trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
				s->size, gfpflags);
E
Eduard - Gabriel Munteanu 已提交
2589 2590

	return ret;
C
Christoph Lameter 已提交
2591 2592 2593
}
EXPORT_SYMBOL(kmem_cache_alloc);

2594
#ifdef CONFIG_TRACING
2595 2596
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
2597
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);
2598
	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2599
	kasan_kmalloc(s, ret, size, gfpflags);
2600 2601 2602
	return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);
E
Eduard - Gabriel Munteanu 已提交
2603 2604
#endif

C
Christoph Lameter 已提交
2605 2606 2607
#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
2608
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
2609

2610
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
2611
				    s->object_size, s->size, gfpflags, node);
E
Eduard - Gabriel Munteanu 已提交
2612 2613

	return ret;
C
Christoph Lameter 已提交
2614 2615 2616
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

2617
#ifdef CONFIG_TRACING
2618
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
E
Eduard - Gabriel Munteanu 已提交
2619
				    gfp_t gfpflags,
2620
				    int node, size_t size)
E
Eduard - Gabriel Munteanu 已提交
2621
{
2622
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
2623 2624 2625

	trace_kmalloc_node(_RET_IP_, ret,
			   size, s->size, gfpflags, node);
2626

2627
	kasan_kmalloc(s, ret, size, gfpflags);
2628
	return ret;
E
Eduard - Gabriel Munteanu 已提交
2629
}
2630
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
E
Eduard - Gabriel Munteanu 已提交
2631
#endif
2632
#endif
E
Eduard - Gabriel Munteanu 已提交
2633

C
Christoph Lameter 已提交
2634
/*
K
Kim Phillips 已提交
2635
 * Slow path handling. This may still be called frequently since objects
2636
 * have a longer lifetime than the cpu slabs in most processing loads.
C
Christoph Lameter 已提交
2637
 *
2638 2639 2640
 * So we still attempt to reduce cache line usage. Just take the slab
 * lock and free the item. If there is no additional partial page
 * handling required then we can return immediately.
C
Christoph Lameter 已提交
2641
 */
2642
static void __slab_free(struct kmem_cache *s, struct page *page,
2643 2644 2645
			void *head, void *tail, int cnt,
			unsigned long addr)

C
Christoph Lameter 已提交
2646 2647
{
	void *prior;
2648 2649 2650 2651
	int was_frozen;
	struct page new;
	unsigned long counters;
	struct kmem_cache_node *n = NULL;
2652
	unsigned long uninitialized_var(flags);
C
Christoph Lameter 已提交
2653

2654
	stat(s, FREE_SLOWPATH);
C
Christoph Lameter 已提交
2655

2656
	if (kmem_cache_debug(s) &&
2657
	    !free_debug_processing(s, page, head, tail, cnt, addr))
2658
		return;
C
Christoph Lameter 已提交
2659

2660
	do {
2661 2662 2663 2664
		if (unlikely(n)) {
			spin_unlock_irqrestore(&n->list_lock, flags);
			n = NULL;
		}
2665 2666
		prior = page->freelist;
		counters = page->counters;
2667
		set_freepointer(s, tail, prior);
2668 2669
		new.counters = counters;
		was_frozen = new.frozen;
2670
		new.inuse -= cnt;
2671
		if ((!new.inuse || !prior) && !was_frozen) {
2672

P
Peter Zijlstra 已提交
2673
			if (kmem_cache_has_cpu_partial(s) && !prior) {
2674 2675

				/*
2676 2677 2678 2679
				 * Slab was on no list before and will be
				 * partially empty
				 * We can defer the list move and instead
				 * freeze it.
2680 2681 2682
				 */
				new.frozen = 1;

P
Peter Zijlstra 已提交
2683
			} else { /* Needs to be taken off a list */
2684

2685
				n = get_node(s, page_to_nid(page));
2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696
				/*
				 * Speculatively acquire the list_lock.
				 * If the cmpxchg does not succeed then we may
				 * drop the list_lock without any processing.
				 *
				 * Otherwise the list_lock will synchronize with
				 * other processors updating the list of slabs.
				 */
				spin_lock_irqsave(&n->list_lock, flags);

			}
2697
		}
C
Christoph Lameter 已提交
2698

2699 2700
	} while (!cmpxchg_double_slab(s, page,
		prior, counters,
2701
		head, new.counters,
2702
		"__slab_free"));
C
Christoph Lameter 已提交
2703

2704
	if (likely(!n)) {
2705 2706 2707 2708 2709

		/*
		 * If we just froze the page then put it onto the
		 * per cpu partial list.
		 */
2710
		if (new.frozen && !was_frozen) {
2711
			put_cpu_partial(s, page, 1);
2712 2713
			stat(s, CPU_PARTIAL_FREE);
		}
2714
		/*
2715 2716 2717
		 * The list lock was not taken therefore no list
		 * activity can be necessary.
		 */
2718 2719 2720 2721
		if (was_frozen)
			stat(s, FREE_FROZEN);
		return;
	}
C
Christoph Lameter 已提交
2722

2723
	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
2724 2725
		goto slab_empty;

C
Christoph Lameter 已提交
2726
	/*
2727 2728
	 * Objects left in the slab. If it was not on the partial list before
	 * then add it.
C
Christoph Lameter 已提交
2729
	 */
2730 2731
	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
		if (kmem_cache_debug(s))
P
Peter Zijlstra 已提交
2732
			remove_full(s, n, page);
2733 2734
		add_partial(n, page, DEACTIVATE_TO_TAIL);
		stat(s, FREE_ADD_PARTIAL);
2735
	}
2736
	spin_unlock_irqrestore(&n->list_lock, flags);
C
Christoph Lameter 已提交
2737 2738 2739
	return;

slab_empty:
2740
	if (prior) {
C
Christoph Lameter 已提交
2741
		/*
2742
		 * Slab on the partial list.
C
Christoph Lameter 已提交
2743
		 */
2744
		remove_partial(n, page);
2745
		stat(s, FREE_REMOVE_PARTIAL);
P
Peter Zijlstra 已提交
2746
	} else {
2747
		/* Slab must be on the full list */
P
Peter Zijlstra 已提交
2748 2749
		remove_full(s, n, page);
	}
2750

2751
	spin_unlock_irqrestore(&n->list_lock, flags);
2752
	stat(s, FREE_SLAB);
C
Christoph Lameter 已提交
2753 2754 2755
	discard_slab(s, page);
}

2756 2757 2758 2759 2760 2761 2762 2763 2764 2765
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
2766 2767 2768 2769
 *
 * Bulk free of a freelist with several objects (all pointing to the
 * same page) possible by specifying head and tail ptr, plus objects
 * count (cnt). Bulk free indicated by tail pointer being set.
2770
 */
2771 2772 2773
static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
				      void *head, void *tail, int cnt,
				      unsigned long addr)
2774
{
2775
	void *tail_obj = tail ? : head;
2776
	struct kmem_cache_cpu *c;
2777
	unsigned long tid;
2778

2779
	slab_free_freelist_hook(s, head, tail);
2780

2781 2782 2783 2784 2785
redo:
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
2786
	 * during the cmpxchg then the free will succeed.
2787
	 */
2788 2789 2790
	do {
		tid = this_cpu_read(s->cpu_slab->tid);
		c = raw_cpu_ptr(s->cpu_slab);
2791 2792
	} while (IS_ENABLED(CONFIG_PREEMPT) &&
		 unlikely(tid != READ_ONCE(c->tid)));
2793

2794 2795
	/* Same with comment on barrier() in slab_alloc_node() */
	barrier();
2796

2797
	if (likely(page == c->page)) {
2798
		set_freepointer(s, tail_obj, c->freelist);
2799

2800
		if (unlikely(!this_cpu_cmpxchg_double(
2801 2802
				s->cpu_slab->freelist, s->cpu_slab->tid,
				c->freelist, tid,
2803
				head, next_tid(tid)))) {
2804 2805 2806 2807

			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
2808
		stat(s, FREE_FASTPATH);
2809
	} else
2810
		__slab_free(s, page, head, tail_obj, cnt, addr);
2811 2812 2813

}

C
Christoph Lameter 已提交
2814 2815
void kmem_cache_free(struct kmem_cache *s, void *x)
{
2816 2817
	s = cache_from_obj(s, x);
	if (!s)
2818
		return;
2819
	slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
2820
	trace_kmem_cache_free(_RET_IP_, x);
C
Christoph Lameter 已提交
2821 2822 2823
}
EXPORT_SYMBOL(kmem_cache_free);

2824
struct detached_freelist {
2825
	struct page *page;
2826 2827 2828
	void *tail;
	void *freelist;
	int cnt;
2829
	struct kmem_cache *s;
2830
};
2831

2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843
/*
 * This function progressively scans the array with free objects (with
 * a limited look ahead) and extract objects belonging to the same
 * page.  It builds a detached freelist directly within the given
 * page/objects.  This can happen without any need for
 * synchronization, because the objects are owned by running process.
 * The freelist is build up as a single linked list in the objects.
 * The idea is, that this detached freelist can then be bulk
 * transferred to the real freelist(s), but only requiring a single
 * synchronization primitive.  Look ahead in the array is limited due
 * to performance reasons.
 */
2844 2845 2846
static inline
int build_detached_freelist(struct kmem_cache *s, size_t size,
			    void **p, struct detached_freelist *df)
2847 2848 2849 2850
{
	size_t first_skipped_index = 0;
	int lookahead = 3;
	void *object;
2851
	struct page *page;
2852

2853 2854
	/* Always re-init detached_freelist */
	df->page = NULL;
2855

2856 2857
	do {
		object = p[--size];
2858
		/* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
2859
	} while (!object && size);
2860

2861 2862
	if (!object)
		return 0;
2863

2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878
	page = virt_to_head_page(object);
	if (!s) {
		/* Handle kalloc'ed objects */
		if (unlikely(!PageSlab(page))) {
			BUG_ON(!PageCompound(page));
			kfree_hook(object);
			__free_kmem_pages(page, compound_order(page));
			p[size] = NULL; /* mark object processed */
			return size;
		}
		/* Derive kmem_cache from object */
		df->s = page->slab_cache;
	} else {
		df->s = cache_from_obj(s, object); /* Support for memcg */
	}
2879

2880
	/* Start new detached freelist */
2881
	df->page = page;
2882
	set_freepointer(df->s, object, NULL);
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
	df->tail = object;
	df->freelist = object;
	p[size] = NULL; /* mark object processed */
	df->cnt = 1;

	while (size) {
		object = p[--size];
		if (!object)
			continue; /* Skip processed objects */

		/* df->page is always set at this point */
		if (df->page == virt_to_head_page(object)) {
			/* Opportunity build freelist */
2896
			set_freepointer(df->s, object, df->freelist);
2897 2898 2899 2900 2901
			df->freelist = object;
			df->cnt++;
			p[size] = NULL; /* mark object processed */

			continue;
2902
		}
2903 2904 2905 2906 2907 2908 2909

		/* Limit look ahead search */
		if (!--lookahead)
			break;

		if (!first_skipped_index)
			first_skipped_index = size + 1;
2910
	}
2911 2912 2913 2914 2915

	return first_skipped_index;
}

/* Note that interrupts must be enabled when calling this function. */
2916
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927
{
	if (WARN_ON(!size))
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
		if (unlikely(!df.page))
			continue;

2928
		slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
2929
	} while (likely(size));
2930 2931 2932
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

2933
/* Note that interrupts must be enabled when calling this function. */
2934 2935
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
			  void **p)
2936
{
2937 2938 2939
	struct kmem_cache_cpu *c;
	int i;

2940 2941 2942 2943
	/* memcg and kmem_cache debug support */
	s = slab_pre_alloc_hook(s, flags);
	if (unlikely(!s))
		return false;
2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
	/*
	 * Drain objects in the per cpu slab, while disabling local
	 * IRQs, which protects against PREEMPT and interrupts
	 * handlers invoking normal fastpath.
	 */
	local_irq_disable();
	c = this_cpu_ptr(s->cpu_slab);

	for (i = 0; i < size; i++) {
		void *object = c->freelist;

2955 2956 2957 2958 2959
		if (unlikely(!object)) {
			/*
			 * Invoking slow path likely have side-effect
			 * of re-populating per CPU c->freelist
			 */
2960
			p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
2961
					    _RET_IP_, c);
2962 2963 2964
			if (unlikely(!p[i]))
				goto error;

2965 2966 2967
			c = this_cpu_ptr(s->cpu_slab);
			continue; /* goto for-loop */
		}
2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981
		c->freelist = get_freepointer(s, object);
		p[i] = object;
	}
	c->tid = next_tid(c->tid);
	local_irq_enable();

	/* Clear memory outside IRQ disabled fastpath loop */
	if (unlikely(flags & __GFP_ZERO)) {
		int j;

		for (j = 0; j < i; j++)
			memset(p[j], 0, s->object_size);
	}

2982 2983
	/* memcg and kmem_cache debug support */
	slab_post_alloc_hook(s, flags, size, p);
2984
	return i;
2985 2986
error:
	local_irq_enable();
2987 2988
	slab_post_alloc_hook(s, flags, i, p);
	__kmem_cache_free_bulk(s, i, p);
2989
	return 0;
2990 2991 2992 2993
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk);


C
Christoph Lameter 已提交
2994
/*
C
Christoph Lameter 已提交
2995 2996 2997 2998
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
C
Christoph Lameter 已提交
2999 3000 3001 3002
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
C
Christoph Lameter 已提交
3003
 * must be moved on and off the partial lists and is therefore a factor in
C
Christoph Lameter 已提交
3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
 * locking overhead.
 */

/*
 * Mininum / Maximum order of slab pages. This influences locking overhead
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
static int slub_min_order;
3014
static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
3015
static int slub_min_objects;
C
Christoph Lameter 已提交
3016 3017 3018 3019

/*
 * Calculate the order of allocation given an slab object size.
 *
C
Christoph Lameter 已提交
3020 3021 3022 3023
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
C
Christoph Lameter 已提交
3024
 * unused space left. We go to a higher order if more than 1/16th of the slab
C
Christoph Lameter 已提交
3025 3026 3027 3028 3029 3030
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
C
Christoph Lameter 已提交
3031
 *
C
Christoph Lameter 已提交
3032 3033 3034 3035
 * slub_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slub_max_order then
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
C
Christoph Lameter 已提交
3036
 *
C
Christoph Lameter 已提交
3037 3038 3039 3040
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
 * requested a higher mininum order then we start with that one instead of
 * the smallest order which will fit the object.
C
Christoph Lameter 已提交
3041
 */
3042
static inline int slab_order(int size, int min_objects,
3043
				int max_order, int fract_leftover, int reserved)
C
Christoph Lameter 已提交
3044 3045 3046
{
	int order;
	int rem;
3047
	int min_order = slub_min_order;
C
Christoph Lameter 已提交
3048

3049
	if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
3050
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
3051

3052
	for (order = max(min_order, get_order(min_objects * size + reserved));
3053
			order <= max_order; order++) {
C
Christoph Lameter 已提交
3054

3055
		unsigned long slab_size = PAGE_SIZE << order;
C
Christoph Lameter 已提交
3056

3057
		rem = (slab_size - reserved) % size;
C
Christoph Lameter 已提交
3058

3059
		if (rem <= slab_size / fract_leftover)
C
Christoph Lameter 已提交
3060 3061
			break;
	}
C
Christoph Lameter 已提交
3062

C
Christoph Lameter 已提交
3063 3064 3065
	return order;
}

3066
static inline int calculate_order(int size, int reserved)
3067 3068 3069 3070
{
	int order;
	int min_objects;
	int fraction;
3071
	int max_objects;
3072 3073 3074 3075 3076 3077

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 *
3078
	 * First we increase the acceptable waste in a slab. Then
3079 3080 3081
	 * we reduce the minimum objects required in a slab.
	 */
	min_objects = slub_min_objects;
3082 3083
	if (!min_objects)
		min_objects = 4 * (fls(nr_cpu_ids) + 1);
3084
	max_objects = order_objects(slub_max_order, size, reserved);
3085 3086
	min_objects = min(min_objects, max_objects);

3087
	while (min_objects > 1) {
C
Christoph Lameter 已提交
3088
		fraction = 16;
3089 3090
		while (fraction >= 4) {
			order = slab_order(size, min_objects,
3091
					slub_max_order, fraction, reserved);
3092 3093 3094 3095
			if (order <= slub_max_order)
				return order;
			fraction /= 2;
		}
3096
		min_objects--;
3097 3098 3099 3100 3101 3102
	}

	/*
	 * We were unable to place multiple objects in a slab. Now
	 * lets see if we can place a single object there.
	 */
3103
	order = slab_order(size, 1, slub_max_order, 1, reserved);
3104 3105 3106 3107 3108 3109
	if (order <= slub_max_order)
		return order;

	/*
	 * Doh this slab cannot be placed using slub_max_order.
	 */
3110
	order = slab_order(size, 1, MAX_ORDER, 1, reserved);
D
David Rientjes 已提交
3111
	if (order < MAX_ORDER)
3112 3113 3114 3115
		return order;
	return -ENOSYS;
}

3116
static void
3117
init_kmem_cache_node(struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3118 3119 3120 3121
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
3122
#ifdef CONFIG_SLUB_DEBUG
3123
	atomic_long_set(&n->nr_slabs, 0);
3124
	atomic_long_set(&n->total_objects, 0);
3125
	INIT_LIST_HEAD(&n->full);
3126
#endif
C
Christoph Lameter 已提交
3127 3128
}

3129
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
3130
{
3131
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
3132
			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
3133

3134
	/*
3135 3136
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
3137
	 */
3138 3139
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));
3140 3141 3142 3143 3144

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
3145

3146
	return 1;
3147 3148
}

3149 3150
static struct kmem_cache *kmem_cache_node;

C
Christoph Lameter 已提交
3151 3152 3153 3154 3155
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
Z
Zhi Yong Wu 已提交
3156 3157
 * Note that this function only works on the kmem_cache_node
 * when allocating for the kmem_cache_node. This is used for bootstrapping
3158
 * memory on a fresh node that has no slab structures yet.
C
Christoph Lameter 已提交
3159
 */
3160
static void early_kmem_cache_node_alloc(int node)
C
Christoph Lameter 已提交
3161 3162 3163 3164
{
	struct page *page;
	struct kmem_cache_node *n;

3165
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
C
Christoph Lameter 已提交
3166

3167
	page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
C
Christoph Lameter 已提交
3168 3169

	BUG_ON(!page);
3170
	if (page_to_nid(page) != node) {
3171 3172
		pr_err("SLUB: Unable to allocate memory from node %d\n", node);
		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
3173 3174
	}

C
Christoph Lameter 已提交
3175 3176
	n = page->freelist;
	BUG_ON(!n);
3177
	page->freelist = get_freepointer(kmem_cache_node, n);
3178
	page->inuse = 1;
3179
	page->frozen = 0;
3180
	kmem_cache_node->node[node] = n;
3181
#ifdef CONFIG_SLUB_DEBUG
3182
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
3183
	init_tracking(kmem_cache_node, n);
3184
#endif
3185 3186
	kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
		      GFP_KERNEL);
3187
	init_kmem_cache_node(n);
3188
	inc_slabs_node(kmem_cache_node, node, page->objects);
C
Christoph Lameter 已提交
3189

3190
	/*
3191 3192
	 * No locks need to be taken here as it has just been
	 * initialized and there is no concurrent access.
3193
	 */
3194
	__add_partial(n, page, DEACTIVATE_TO_HEAD);
C
Christoph Lameter 已提交
3195 3196 3197 3198 3199
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;
C
Christoph Lameter 已提交
3200
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
3201

C
Christoph Lameter 已提交
3202 3203
	for_each_kmem_cache_node(s, node, n) {
		kmem_cache_free(kmem_cache_node, n);
C
Christoph Lameter 已提交
3204 3205 3206 3207
		s->node[node] = NULL;
	}
}

3208 3209 3210 3211 3212 3213
void __kmem_cache_release(struct kmem_cache *s)
{
	free_percpu(s->cpu_slab);
	free_kmem_cache_nodes(s);
}

3214
static int init_kmem_cache_nodes(struct kmem_cache *s)
C
Christoph Lameter 已提交
3215 3216 3217
{
	int node;

C
Christoph Lameter 已提交
3218
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
3219 3220
		struct kmem_cache_node *n;

3221
		if (slab_state == DOWN) {
3222
			early_kmem_cache_node_alloc(node);
3223 3224
			continue;
		}
3225
		n = kmem_cache_alloc_node(kmem_cache_node,
3226
						GFP_KERNEL, node);
C
Christoph Lameter 已提交
3227

3228 3229 3230
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
C
Christoph Lameter 已提交
3231
		}
3232

C
Christoph Lameter 已提交
3233
		s->node[node] = n;
3234
		init_kmem_cache_node(n);
C
Christoph Lameter 已提交
3235 3236 3237 3238
	}
	return 1;
}

3239
static void set_min_partial(struct kmem_cache *s, unsigned long min)
3240 3241 3242 3243 3244 3245 3246 3247
{
	if (min < MIN_PARTIAL)
		min = MIN_PARTIAL;
	else if (min > MAX_PARTIAL)
		min = MAX_PARTIAL;
	s->min_partial = min;
}

C
Christoph Lameter 已提交
3248 3249 3250 3251
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
3252
static int calculate_sizes(struct kmem_cache *s, int forced_order)
C
Christoph Lameter 已提交
3253 3254
{
	unsigned long flags = s->flags;
3255
	unsigned long size = s->object_size;
3256
	int order;
C
Christoph Lameter 已提交
3257

3258 3259 3260 3261 3262 3263 3264 3265
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
3266 3267 3268 3269 3270 3271
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
	if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
3272
			!s->ctor)
C
Christoph Lameter 已提交
3273 3274 3275 3276 3277 3278
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
C
Christoph Lameter 已提交
3279
	 * If we are Redzoning then check if there is some space between the
C
Christoph Lameter 已提交
3280
	 * end of the object and the free pointer. If not then add an
C
Christoph Lameter 已提交
3281
	 * additional word to have some bytes to store Redzone information.
C
Christoph Lameter 已提交
3282
	 */
3283
	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
C
Christoph Lameter 已提交
3284
		size += sizeof(void *);
C
Christoph Lameter 已提交
3285
#endif
C
Christoph Lameter 已提交
3286 3287

	/*
C
Christoph Lameter 已提交
3288 3289
	 * With that we have determined the number of bytes in actual use
	 * by the object. This is the potential offset to the free pointer.
C
Christoph Lameter 已提交
3290 3291 3292 3293
	 */
	s->inuse = size;

	if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
3294
		s->ctor)) {
C
Christoph Lameter 已提交
3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
		 * destructor or are poisoning the objects.
		 */
		s->offset = size;
		size += sizeof(void *);
	}

3307
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
3308 3309 3310 3311 3312 3313 3314
	if (flags & SLAB_STORE_USER)
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);

J
Joonsoo Kim 已提交
3315
	if (flags & SLAB_RED_ZONE) {
C
Christoph Lameter 已提交
3316 3317 3318 3319
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
3320
		 * corrupted if a user writes before the start
C
Christoph Lameter 已提交
3321 3322 3323
		 * of the object.
		 */
		size += sizeof(void *);
J
Joonsoo Kim 已提交
3324 3325 3326 3327 3328

		s->red_left_pad = sizeof(void *);
		s->red_left_pad = ALIGN(s->red_left_pad, s->align);
		size += s->red_left_pad;
	}
C
Christoph Lameter 已提交
3329
#endif
C
Christoph Lameter 已提交
3330

C
Christoph Lameter 已提交
3331 3332 3333 3334 3335
	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
3336
	size = ALIGN(size, s->align);
C
Christoph Lameter 已提交
3337
	s->size = size;
3338 3339 3340
	if (forced_order >= 0)
		order = forced_order;
	else
3341
		order = calculate_order(size, s->reserved);
C
Christoph Lameter 已提交
3342

3343
	if (order < 0)
C
Christoph Lameter 已提交
3344 3345
		return 0;

3346
	s->allocflags = 0;
3347
	if (order)
3348 3349 3350
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
3351
		s->allocflags |= GFP_DMA;
3352 3353 3354 3355

	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

C
Christoph Lameter 已提交
3356 3357 3358
	/*
	 * Determine the number of objects per slab
	 */
3359 3360
	s->oo = oo_make(order, size, s->reserved);
	s->min = oo_make(get_order(size), size, s->reserved);
3361 3362
	if (oo_objects(s->oo) > oo_objects(s->max))
		s->max = s->oo;
C
Christoph Lameter 已提交
3363

3364
	return !!oo_objects(s->oo);
C
Christoph Lameter 已提交
3365 3366
}

3367
static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
C
Christoph Lameter 已提交
3368
{
3369
	s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
3370
	s->reserved = 0;
C
Christoph Lameter 已提交
3371

3372 3373
	if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
		s->reserved = sizeof(struct rcu_head);
C
Christoph Lameter 已提交
3374

3375
	if (!calculate_sizes(s, -1))
C
Christoph Lameter 已提交
3376
		goto error;
3377 3378 3379 3380 3381
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
3382
		if (get_order(s->size) > get_order(s->object_size)) {
3383 3384 3385 3386 3387 3388
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
			if (!calculate_sizes(s, -1))
				goto error;
		}
	}
C
Christoph Lameter 已提交
3389

3390 3391
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3392
	if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
3393 3394 3395 3396
		/* Enable fast mode */
		s->flags |= __CMPXCHG_DOUBLE;
#endif

3397 3398 3399 3400
	/*
	 * The larger the object size is, the more pages we want on the partial
	 * list to avoid pounding the page allocator excessively.
	 */
3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
	set_min_partial(s, ilog2(s->size) / 2);

	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
	 * This setting also determines
	 *
	 * A) The number of objects from per cpu partial slabs dumped to the
	 *    per node list when we reach the limit.
3416
	 * B) The number of objects in cpu partial slabs to extract from the
3417 3418
	 *    per node list when we run out of per cpu objects. We only fetch
	 *    50% to keep some capacity around for frees.
3419
	 */
3420
	if (!kmem_cache_has_cpu_partial(s))
3421 3422
		s->cpu_partial = 0;
	else if (s->size >= PAGE_SIZE)
3423 3424 3425 3426 3427 3428 3429 3430
		s->cpu_partial = 2;
	else if (s->size >= 1024)
		s->cpu_partial = 6;
	else if (s->size >= 256)
		s->cpu_partial = 13;
	else
		s->cpu_partial = 30;

C
Christoph Lameter 已提交
3431
#ifdef CONFIG_NUMA
3432
	s->remote_node_defrag_ratio = 1000;
C
Christoph Lameter 已提交
3433
#endif
3434
	if (!init_kmem_cache_nodes(s))
3435
		goto error;
C
Christoph Lameter 已提交
3436

3437
	if (alloc_kmem_cache_cpus(s))
3438
		return 0;
3439

3440
	free_kmem_cache_nodes(s);
C
Christoph Lameter 已提交
3441 3442
error:
	if (flags & SLAB_PANIC)
J
Joe Perches 已提交
3443 3444 3445
		panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
		      s->name, (unsigned long)s->size, s->size,
		      oo_order(s->oo), s->offset, flags);
3446
	return -EINVAL;
C
Christoph Lameter 已提交
3447 3448
}

3449 3450 3451 3452 3453 3454
static void list_slab_objects(struct kmem_cache *s, struct page *page,
							const char *text)
{
#ifdef CONFIG_SLUB_DEBUG
	void *addr = page_address(page);
	void *p;
N
Namhyung Kim 已提交
3455 3456
	unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
				     sizeof(long), GFP_ATOMIC);
E
Eric Dumazet 已提交
3457 3458
	if (!map)
		return;
3459
	slab_err(s, page, text, s->name);
3460 3461
	slab_lock(page);

3462
	get_map(s, page, map);
3463 3464 3465
	for_each_object(p, s, addr, page->objects) {

		if (!test_bit(slab_index(p, s, addr), map)) {
3466
			pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
3467 3468 3469 3470
			print_tracking(s, p);
		}
	}
	slab_unlock(page);
E
Eric Dumazet 已提交
3471
	kfree(map);
3472 3473 3474
#endif
}

C
Christoph Lameter 已提交
3475
/*
C
Christoph Lameter 已提交
3476
 * Attempt to free all partial slabs on a node.
3477 3478
 * This is called from __kmem_cache_shutdown(). We must take list_lock
 * because sysfs file might still access partial list after the shutdowning.
C
Christoph Lameter 已提交
3479
 */
C
Christoph Lameter 已提交
3480
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3481 3482 3483
{
	struct page *page, *h;

3484 3485
	BUG_ON(irqs_disabled());
	spin_lock_irq(&n->list_lock);
3486
	list_for_each_entry_safe(page, h, &n->partial, lru) {
C
Christoph Lameter 已提交
3487
		if (!page->inuse) {
3488
			remove_partial(n, page);
C
Christoph Lameter 已提交
3489
			discard_slab(s, page);
3490 3491
		} else {
			list_slab_objects(s, page,
3492
			"Objects remaining in %s on __kmem_cache_shutdown()");
C
Christoph Lameter 已提交
3493
		}
3494
	}
3495
	spin_unlock_irq(&n->list_lock);
C
Christoph Lameter 已提交
3496 3497 3498
}

/*
C
Christoph Lameter 已提交
3499
 * Release all resources used by a slab cache.
C
Christoph Lameter 已提交
3500
 */
3501
int __kmem_cache_shutdown(struct kmem_cache *s)
C
Christoph Lameter 已提交
3502 3503
{
	int node;
C
Christoph Lameter 已提交
3504
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
3505 3506 3507

	flush_all(s);
	/* Attempt to free all objects */
C
Christoph Lameter 已提交
3508
	for_each_kmem_cache_node(s, node, n) {
C
Christoph Lameter 已提交
3509 3510
		free_partial(s, n);
		if (n->nr_partial || slabs_node(s, node))
C
Christoph Lameter 已提交
3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521
			return 1;
	}
	return 0;
}

/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

static int __init setup_slub_min_order(char *str)
{
P
Pekka Enberg 已提交
3522
	get_option(&str, &slub_min_order);
C
Christoph Lameter 已提交
3523 3524 3525 3526 3527 3528 3529 3530

	return 1;
}

__setup("slub_min_order=", setup_slub_min_order);

static int __init setup_slub_max_order(char *str)
{
P
Pekka Enberg 已提交
3531
	get_option(&str, &slub_max_order);
D
David Rientjes 已提交
3532
	slub_max_order = min(slub_max_order, MAX_ORDER - 1);
C
Christoph Lameter 已提交
3533 3534 3535 3536 3537 3538 3539 3540

	return 1;
}

__setup("slub_max_order=", setup_slub_max_order);

static int __init setup_slub_min_objects(char *str)
{
P
Pekka Enberg 已提交
3541
	get_option(&str, &slub_min_objects);
C
Christoph Lameter 已提交
3542 3543 3544 3545 3546 3547 3548 3549

	return 1;
}

__setup("slub_min_objects=", setup_slub_min_objects);

void *__kmalloc(size_t size, gfp_t flags)
{
3550
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
3551
	void *ret;
C
Christoph Lameter 已提交
3552

3553
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3554
		return kmalloc_large(size, flags);
3555

3556
	s = kmalloc_slab(size, flags);
3557 3558

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3559 3560
		return s;

3561
	ret = slab_alloc(s, flags, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3562

3563
	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
E
Eduard - Gabriel Munteanu 已提交
3564

3565
	kasan_kmalloc(s, ret, size, flags);
3566

E
Eduard - Gabriel Munteanu 已提交
3567
	return ret;
C
Christoph Lameter 已提交
3568 3569 3570
}
EXPORT_SYMBOL(__kmalloc);

3571
#ifdef CONFIG_NUMA
3572 3573
static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
3574
	struct page *page;
3575
	void *ptr = NULL;
3576

V
Vladimir Davydov 已提交
3577 3578
	flags |= __GFP_COMP | __GFP_NOTRACK;
	page = alloc_kmem_pages_node(node, flags, get_order(size));
3579
	if (page)
3580 3581
		ptr = page_address(page);

3582
	kmalloc_large_node_hook(ptr, size, flags);
3583
	return ptr;
3584 3585
}

C
Christoph Lameter 已提交
3586 3587
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
3588
	struct kmem_cache *s;
E
Eduard - Gabriel Munteanu 已提交
3589
	void *ret;
C
Christoph Lameter 已提交
3590

3591
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
E
Eduard - Gabriel Munteanu 已提交
3592 3593
		ret = kmalloc_large_node(size, flags, node);

3594 3595 3596
		trace_kmalloc_node(_RET_IP_, ret,
				   size, PAGE_SIZE << get_order(size),
				   flags, node);
E
Eduard - Gabriel Munteanu 已提交
3597 3598 3599

		return ret;
	}
3600

3601
	s = kmalloc_slab(size, flags);
3602 3603

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3604 3605
		return s;

3606
	ret = slab_alloc_node(s, flags, node, _RET_IP_);
E
Eduard - Gabriel Munteanu 已提交
3607

3608
	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
E
Eduard - Gabriel Munteanu 已提交
3609

3610
	kasan_kmalloc(s, ret, size, flags);
3611

E
Eduard - Gabriel Munteanu 已提交
3612
	return ret;
C
Christoph Lameter 已提交
3613 3614 3615 3616
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

3617
static size_t __ksize(const void *object)
C
Christoph Lameter 已提交
3618
{
3619
	struct page *page;
C
Christoph Lameter 已提交
3620

3621
	if (unlikely(object == ZERO_SIZE_PTR))
3622 3623
		return 0;

3624 3625
	page = virt_to_head_page(object);

P
Pekka Enberg 已提交
3626 3627
	if (unlikely(!PageSlab(page))) {
		WARN_ON(!PageCompound(page));
3628
		return PAGE_SIZE << compound_order(page);
P
Pekka Enberg 已提交
3629
	}
C
Christoph Lameter 已提交
3630

3631
	return slab_ksize(page->slab_cache);
C
Christoph Lameter 已提交
3632
}
3633 3634 3635 3636 3637 3638

size_t ksize(const void *object)
{
	size_t size = __ksize(object);
	/* We assume that ksize callers could use whole allocated area,
	   so we need unpoison this area. */
3639
	kasan_krealloc(object, size, GFP_NOWAIT);
3640 3641
	return size;
}
K
Kirill A. Shutemov 已提交
3642
EXPORT_SYMBOL(ksize);
C
Christoph Lameter 已提交
3643 3644 3645 3646

void kfree(const void *x)
{
	struct page *page;
3647
	void *object = (void *)x;
C
Christoph Lameter 已提交
3648

3649 3650
	trace_kfree(_RET_IP_, x);

3651
	if (unlikely(ZERO_OR_NULL_PTR(x)))
C
Christoph Lameter 已提交
3652 3653
		return;

3654
	page = virt_to_head_page(x);
3655
	if (unlikely(!PageSlab(page))) {
3656
		BUG_ON(!PageCompound(page));
3657
		kfree_hook(x);
V
Vladimir Davydov 已提交
3658
		__free_kmem_pages(page, compound_order(page));
3659 3660
		return;
	}
3661
	slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
C
Christoph Lameter 已提交
3662 3663 3664
}
EXPORT_SYMBOL(kfree);

3665 3666
#define SHRINK_PROMOTE_MAX 32

3667
/*
3668 3669 3670
 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
 * up most to the head of the partial lists. New allocations will then
 * fill those up and thus they can be removed from the partial lists.
C
Christoph Lameter 已提交
3671 3672 3673 3674
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
3675
 */
3676
int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
3677 3678 3679 3680 3681 3682
{
	int node;
	int i;
	struct kmem_cache_node *n;
	struct page *page;
	struct page *t;
3683 3684
	struct list_head discard;
	struct list_head promote[SHRINK_PROMOTE_MAX];
3685
	unsigned long flags;
3686
	int ret = 0;
3687

3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702
	if (deactivate) {
		/*
		 * Disable empty slabs caching. Used to avoid pinning offline
		 * memory cgroups by kmem pages that can be freed.
		 */
		s->cpu_partial = 0;
		s->min_partial = 0;

		/*
		 * s->cpu_partial is checked locklessly (see put_cpu_partial),
		 * so we have to make sure the change is visible.
		 */
		kick_all_cpus_sync();
	}

3703
	flush_all(s);
C
Christoph Lameter 已提交
3704
	for_each_kmem_cache_node(s, node, n) {
3705 3706 3707
		INIT_LIST_HEAD(&discard);
		for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
			INIT_LIST_HEAD(promote + i);
3708 3709 3710 3711

		spin_lock_irqsave(&n->list_lock, flags);

		/*
3712
		 * Build lists of slabs to discard or promote.
3713
		 *
C
Christoph Lameter 已提交
3714 3715
		 * Note that concurrent frees may occur while we hold the
		 * list_lock. page->inuse here is the upper limit.
3716 3717
		 */
		list_for_each_entry_safe(page, t, &n->partial, lru) {
3718 3719 3720 3721 3722 3723 3724 3725 3726 3727
			int free = page->objects - page->inuse;

			/* Do not reread page->inuse */
			barrier();

			/* We do not keep full slabs on the list */
			BUG_ON(free <= 0);

			if (free == page->objects) {
				list_move(&page->lru, &discard);
3728
				n->nr_partial--;
3729 3730
			} else if (free <= SHRINK_PROMOTE_MAX)
				list_move(&page->lru, promote + free - 1);
3731 3732 3733
		}

		/*
3734 3735
		 * Promote the slabs filled up most to the head of the
		 * partial list.
3736
		 */
3737 3738
		for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
			list_splice(promote + i, &n->partial);
3739 3740

		spin_unlock_irqrestore(&n->list_lock, flags);
3741 3742

		/* Release empty slabs */
3743
		list_for_each_entry_safe(page, t, &discard, lru)
3744
			discard_slab(s, page);
3745 3746 3747

		if (slabs_node(s, node))
			ret = 1;
3748 3749
	}

3750
	return ret;
3751 3752
}

3753 3754 3755 3756
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

3757
	mutex_lock(&slab_mutex);
3758
	list_for_each_entry(s, &slab_caches, list)
3759
		__kmem_cache_shrink(s, false);
3760
	mutex_unlock(&slab_mutex);
3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int offline_node;

3772
	offline_node = marg->status_change_nid_normal;
3773 3774 3775 3776 3777 3778 3779 3780

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

3781
	mutex_lock(&slab_mutex);
3782 3783 3784 3785 3786 3787
	list_for_each_entry(s, &slab_caches, list) {
		n = get_node(s, offline_node);
		if (n) {
			/*
			 * if n->nr_slabs > 0, slabs still exist on the node
			 * that is going down. We were unable to free them,
3788
			 * and offline_pages() function shouldn't call this
3789 3790
			 * callback. So, we must fail.
			 */
3791
			BUG_ON(slabs_node(s, offline_node));
3792 3793

			s->node[offline_node] = NULL;
3794
			kmem_cache_free(kmem_cache_node, n);
3795 3796
		}
	}
3797
	mutex_unlock(&slab_mutex);
3798 3799 3800 3801 3802 3803 3804
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
3805
	int nid = marg->status_change_nid_normal;
3806 3807 3808 3809 3810 3811 3812 3813 3814 3815
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
3816
	 * We are bringing a node online. No memory is available yet. We must
3817 3818 3819
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
3820
	mutex_lock(&slab_mutex);
3821 3822 3823 3824 3825 3826
	list_for_each_entry(s, &slab_caches, list) {
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
3827
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
3828 3829 3830 3831
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
3832
		init_kmem_cache_node(n);
3833 3834 3835
		s->node[nid] = n;
	}
out:
3836
	mutex_unlock(&slab_mutex);
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
3860 3861 3862 3863
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
3864 3865 3866
	return ret;
}

3867 3868 3869 3870
static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};
3871

C
Christoph Lameter 已提交
3872 3873 3874 3875
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

3876 3877
/*
 * Used for early kmem_cache structures that were allocated using
3878 3879
 * the page allocator. Allocate them properly then fix up the pointers
 * that may be pointing to the wrong kmem_cache structure.
3880 3881
 */

3882
static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
3883 3884
{
	int node;
3885
	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
C
Christoph Lameter 已提交
3886
	struct kmem_cache_node *n;
3887

3888
	memcpy(s, static_cache, kmem_cache->object_size);
3889

3890 3891 3892 3893 3894 3895
	/*
	 * This runs very early, and only the boot processor is supposed to be
	 * up.  Even if it weren't true, IRQs are not up so we couldn't fire
	 * IPIs around.
	 */
	__flush_cpu_slab(s, smp_processor_id());
C
Christoph Lameter 已提交
3896
	for_each_kmem_cache_node(s, node, n) {
3897 3898
		struct page *p;

C
Christoph Lameter 已提交
3899 3900
		list_for_each_entry(p, &n->partial, lru)
			p->slab_cache = s;
3901

L
Li Zefan 已提交
3902
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
3903 3904
		list_for_each_entry(p, &n->full, lru)
			p->slab_cache = s;
3905 3906
#endif
	}
3907
	slab_init_memcg_params(s);
3908 3909
	list_add(&s->list, &slab_caches);
	return s;
3910 3911
}

C
Christoph Lameter 已提交
3912 3913
void __init kmem_cache_init(void)
{
3914 3915
	static __initdata struct kmem_cache boot_kmem_cache,
		boot_kmem_cache_node;
3916

3917 3918 3919
	if (debug_guardpage_minorder())
		slub_max_order = 0;

3920 3921
	kmem_cache_node = &boot_kmem_cache_node;
	kmem_cache = &boot_kmem_cache;
3922

3923 3924
	create_boot_cache(kmem_cache_node, "kmem_cache_node",
		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
3925

3926
	register_hotmemory_notifier(&slab_memory_callback_nb);
C
Christoph Lameter 已提交
3927 3928 3929 3930

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

3931 3932 3933 3934
	create_boot_cache(kmem_cache, "kmem_cache",
			offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *),
		       SLAB_HWCACHE_ALIGN);
3935

3936
	kmem_cache = bootstrap(&boot_kmem_cache);
C
Christoph Lameter 已提交
3937

3938 3939 3940 3941 3942
	/*
	 * Allocate kmem_cache_node properly from the kmem_cache slab.
	 * kmem_cache_node is separately allocated so no need to
	 * update any list pointers.
	 */
3943
	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
3944 3945

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
3946
	setup_kmalloc_cache_index_table();
3947
	create_kmalloc_caches(0);
C
Christoph Lameter 已提交
3948 3949 3950

#ifdef CONFIG_SMP
	register_cpu_notifier(&slab_notifier);
3951
#endif
C
Christoph Lameter 已提交
3952

3953
	pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n",
3954
		cache_line_size(),
C
Christoph Lameter 已提交
3955 3956 3957 3958
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

3959 3960 3961 3962
void __init kmem_cache_init_late(void)
{
}

3963
struct kmem_cache *
3964 3965
__kmem_cache_alias(const char *name, size_t size, size_t align,
		   unsigned long flags, void (*ctor)(void *))
C
Christoph Lameter 已提交
3966
{
3967
	struct kmem_cache *s, *c;
C
Christoph Lameter 已提交
3968

3969
	s = find_mergeable(size, align, flags, name, ctor);
C
Christoph Lameter 已提交
3970 3971
	if (s) {
		s->refcount++;
3972

C
Christoph Lameter 已提交
3973 3974 3975 3976
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
3977
		s->object_size = max(s->object_size, (int)size);
C
Christoph Lameter 已提交
3978
		s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
C
Christoph Lameter 已提交
3979

3980
		for_each_memcg_cache(c, s) {
3981 3982 3983 3984 3985
			c->object_size = s->object_size;
			c->inuse = max_t(int, c->inuse,
					 ALIGN(size, sizeof(void *)));
		}

3986 3987
		if (sysfs_slab_alias(s, name)) {
			s->refcount--;
3988
			s = NULL;
3989
		}
3990
	}
C
Christoph Lameter 已提交
3991

3992 3993
	return s;
}
P
Pekka Enberg 已提交
3994

3995
int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
3996
{
3997 3998 3999 4000 4001
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;
4002

4003 4004 4005 4006
	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

4007
	memcg_propagate_slab_attrs(s);
4008 4009
	err = sysfs_slab_add(s);
	if (err)
4010
		__kmem_cache_release(s);
4011

4012
	return err;
C
Christoph Lameter 已提交
4013 4014 4015 4016
}

#ifdef CONFIG_SMP
/*
C
Christoph Lameter 已提交
4017 4018
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
C
Christoph Lameter 已提交
4019
 */
4020
static int slab_cpuup_callback(struct notifier_block *nfb,
C
Christoph Lameter 已提交
4021 4022 4023
		unsigned long action, void *hcpu)
{
	long cpu = (long)hcpu;
4024 4025
	struct kmem_cache *s;
	unsigned long flags;
C
Christoph Lameter 已提交
4026 4027 4028

	switch (action) {
	case CPU_UP_CANCELED:
4029
	case CPU_UP_CANCELED_FROZEN:
C
Christoph Lameter 已提交
4030
	case CPU_DEAD:
4031
	case CPU_DEAD_FROZEN:
4032
		mutex_lock(&slab_mutex);
4033 4034 4035 4036 4037
		list_for_each_entry(s, &slab_caches, list) {
			local_irq_save(flags);
			__flush_cpu_slab(s, cpu);
			local_irq_restore(flags);
		}
4038
		mutex_unlock(&slab_mutex);
C
Christoph Lameter 已提交
4039 4040 4041 4042 4043 4044 4045
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}

4046
static struct notifier_block slab_notifier = {
I
Ingo Molnar 已提交
4047
	.notifier_call = slab_cpuup_callback
P
Pekka Enberg 已提交
4048
};
C
Christoph Lameter 已提交
4049 4050 4051

#endif

4052
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
C
Christoph Lameter 已提交
4053
{
4054
	struct kmem_cache *s;
4055
	void *ret;
4056

4057
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
4058 4059
		return kmalloc_large(size, gfpflags);

4060
	s = kmalloc_slab(size, gfpflags);
C
Christoph Lameter 已提交
4061

4062
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4063
		return s;
C
Christoph Lameter 已提交
4064

4065
	ret = slab_alloc(s, gfpflags, caller);
4066

L
Lucas De Marchi 已提交
4067
	/* Honor the call site pointer we received. */
4068
	trace_kmalloc(caller, ret, size, s->size, gfpflags);
4069 4070

	return ret;
C
Christoph Lameter 已提交
4071 4072
}

4073
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4074
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4075
					int node, unsigned long caller)
C
Christoph Lameter 已提交
4076
{
4077
	struct kmem_cache *s;
4078
	void *ret;
4079

4080
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
4081 4082 4083 4084 4085 4086 4087 4088
		ret = kmalloc_large_node(size, gfpflags, node);

		trace_kmalloc_node(caller, ret,
				   size, PAGE_SIZE << get_order(size),
				   gfpflags, node);

		return ret;
	}
4089

4090
	s = kmalloc_slab(size, gfpflags);
C
Christoph Lameter 已提交
4091

4092
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4093
		return s;
C
Christoph Lameter 已提交
4094

4095
	ret = slab_alloc_node(s, gfpflags, node, caller);
4096

L
Lucas De Marchi 已提交
4097
	/* Honor the call site pointer we received. */
4098
	trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4099 4100

	return ret;
C
Christoph Lameter 已提交
4101
}
4102
#endif
C
Christoph Lameter 已提交
4103

4104
#ifdef CONFIG_SYSFS
4105 4106 4107 4108 4109 4110 4111 4112 4113
static int count_inuse(struct page *page)
{
	return page->inuse;
}

static int count_total(struct page *page)
{
	return page->objects;
}
4114
#endif
4115

4116
#ifdef CONFIG_SLUB_DEBUG
4117 4118
static int validate_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4119 4120
{
	void *p;
4121
	void *addr = page_address(page);
4122 4123 4124 4125 4126 4127

	if (!check_slab(s, page) ||
			!on_freelist(s, page, NULL))
		return 0;

	/* Now we know that a valid freelist exists */
4128
	bitmap_zero(map, page->objects);
4129

4130 4131 4132 4133 4134
	get_map(s, page, map);
	for_each_object(p, s, addr, page->objects) {
		if (test_bit(slab_index(p, s, addr), map))
			if (!check_object(s, page, p, SLUB_RED_INACTIVE))
				return 0;
4135 4136
	}

4137
	for_each_object(p, s, addr, page->objects)
4138
		if (!test_bit(slab_index(p, s, addr), map))
4139
			if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4140 4141 4142 4143
				return 0;
	return 1;
}

4144 4145
static void validate_slab_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4146
{
4147 4148 4149
	slab_lock(page);
	validate_slab(s, page, map);
	slab_unlock(page);
4150 4151
}

4152 4153
static int validate_slab_node(struct kmem_cache *s,
		struct kmem_cache_node *n, unsigned long *map)
4154 4155 4156 4157 4158 4159 4160 4161
{
	unsigned long count = 0;
	struct page *page;
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

	list_for_each_entry(page, &n->partial, lru) {
4162
		validate_slab_slab(s, page, map);
4163 4164 4165
		count++;
	}
	if (count != n->nr_partial)
4166 4167
		pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
		       s->name, count, n->nr_partial);
4168 4169 4170 4171 4172

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

	list_for_each_entry(page, &n->full, lru) {
4173
		validate_slab_slab(s, page, map);
4174 4175 4176
		count++;
	}
	if (count != atomic_long_read(&n->nr_slabs))
4177 4178
		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
		       s->name, count, atomic_long_read(&n->nr_slabs));
4179 4180 4181 4182 4183 4184

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

4185
static long validate_slab_cache(struct kmem_cache *s)
4186 4187 4188
{
	int node;
	unsigned long count = 0;
4189
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4190
				sizeof(unsigned long), GFP_KERNEL);
C
Christoph Lameter 已提交
4191
	struct kmem_cache_node *n;
4192 4193 4194

	if (!map)
		return -ENOMEM;
4195 4196

	flush_all(s);
C
Christoph Lameter 已提交
4197
	for_each_kmem_cache_node(s, node, n)
4198 4199
		count += validate_slab_node(s, n, map);
	kfree(map);
4200 4201
	return count;
}
4202
/*
C
Christoph Lameter 已提交
4203
 * Generate lists of code addresses where slabcache objects are allocated
4204 4205 4206 4207 4208
 * and freed.
 */

struct location {
	unsigned long count;
4209
	unsigned long addr;
4210 4211 4212 4213 4214
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
R
Rusty Russell 已提交
4215
	DECLARE_BITMAP(cpus, NR_CPUS);
4216
	nodemask_t nodes;
4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
};

static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

4232
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4233 4234 4235 4236 4237 4238
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

4239
	l = (void *)__get_free_pages(flags, order);
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
4253
				const struct track *track)
4254 4255 4256
{
	long start, end, pos;
	struct location *l;
4257
	unsigned long caddr;
4258
	unsigned long age = jiffies - track->when;
4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273

	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

		caddr = t->loc[pos].addr;
4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289
		if (track->addr == caddr) {

			l = &t->loc[pos];
			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

R
Rusty Russell 已提交
4290 4291
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
4292 4293
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
4294 4295 4296
			return 1;
		}

4297
		if (track->addr < caddr)
4298 4299 4300 4301 4302 4303
			end = pos;
		else
			start = pos;
	}

	/*
C
Christoph Lameter 已提交
4304
	 * Not found. Insert new tracking element.
4305
	 */
4306
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4307 4308 4309 4310 4311 4312 4313 4314
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
4315 4316 4317 4318 4319 4320
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
R
Rusty Russell 已提交
4321 4322
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4323 4324
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
4325 4326 4327 4328
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
E
Eric Dumazet 已提交
4329
		struct page *page, enum track_item alloc,
N
Namhyung Kim 已提交
4330
		unsigned long *map)
4331
{
4332
	void *addr = page_address(page);
4333 4334
	void *p;

4335
	bitmap_zero(map, page->objects);
4336
	get_map(s, page, map);
4337

4338
	for_each_object(p, s, addr, page->objects)
4339 4340
		if (!test_bit(slab_index(p, s, addr), map))
			add_location(t, s, get_track(s, p, alloc));
4341 4342 4343 4344 4345
}

static int list_locations(struct kmem_cache *s, char *buf,
					enum track_item alloc)
{
4346
	int len = 0;
4347
	unsigned long i;
4348
	struct loc_track t = { 0, 0, NULL };
4349
	int node;
E
Eric Dumazet 已提交
4350 4351
	unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
				     sizeof(unsigned long), GFP_KERNEL);
C
Christoph Lameter 已提交
4352
	struct kmem_cache_node *n;
4353

E
Eric Dumazet 已提交
4354 4355 4356
	if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
				     GFP_TEMPORARY)) {
		kfree(map);
4357
		return sprintf(buf, "Out of memory\n");
E
Eric Dumazet 已提交
4358
	}
4359 4360 4361
	/* Push back cpu slabs */
	flush_all(s);

C
Christoph Lameter 已提交
4362
	for_each_kmem_cache_node(s, node, n) {
4363 4364 4365
		unsigned long flags;
		struct page *page;

4366
		if (!atomic_long_read(&n->nr_slabs))
4367 4368 4369 4370
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry(page, &n->partial, lru)
E
Eric Dumazet 已提交
4371
			process_slab(&t, s, page, alloc, map);
4372
		list_for_each_entry(page, &n->full, lru)
E
Eric Dumazet 已提交
4373
			process_slab(&t, s, page, alloc, map);
4374 4375 4376 4377
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	for (i = 0; i < t.count; i++) {
4378
		struct location *l = &t.loc[i];
4379

H
Hugh Dickins 已提交
4380
		if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4381
			break;
4382
		len += sprintf(buf + len, "%7ld ", l->count);
4383 4384

		if (l->addr)
J
Joe Perches 已提交
4385
			len += sprintf(buf + len, "%pS", (void *)l->addr);
4386
		else
4387
			len += sprintf(buf + len, "<not-available>");
4388 4389

		if (l->sum_time != l->min_time) {
4390
			len += sprintf(buf + len, " age=%ld/%ld/%ld",
R
Roman Zippel 已提交
4391 4392 4393
				l->min_time,
				(long)div_u64(l->sum_time, l->count),
				l->max_time);
4394
		} else
4395
			len += sprintf(buf + len, " age=%ld",
4396 4397 4398
				l->min_time);

		if (l->min_pid != l->max_pid)
4399
			len += sprintf(buf + len, " pid=%ld-%ld",
4400 4401
				l->min_pid, l->max_pid);
		else
4402
			len += sprintf(buf + len, " pid=%ld",
4403 4404
				l->min_pid);

R
Rusty Russell 已提交
4405 4406
		if (num_online_cpus() > 1 &&
				!cpumask_empty(to_cpumask(l->cpus)) &&
4407 4408 4409 4410
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " cpus=%*pbl",
					 cpumask_pr_args(to_cpumask(l->cpus)));
4411

4412
		if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4413 4414 4415 4416
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " nodes=%*pbl",
					 nodemask_pr_args(&l->nodes));
4417

4418
		len += sprintf(buf + len, "\n");
4419 4420 4421
	}

	free_loc_track(&t);
E
Eric Dumazet 已提交
4422
	kfree(map);
4423
	if (!t.count)
4424 4425
		len += sprintf(buf, "No data\n");
	return len;
4426
}
4427
#endif
4428

4429
#ifdef SLUB_RESILIENCY_TEST
4430
static void __init resiliency_test(void)
4431 4432 4433
{
	u8 *p;

4434
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
4435

4436 4437 4438
	pr_err("SLUB resiliency testing\n");
	pr_err("-----------------------\n");
	pr_err("A. Corruption after allocation\n");
4439 4440 4441

	p = kzalloc(16, GFP_KERNEL);
	p[16] = 0x12;
4442 4443
	pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
	       p + 16);
4444 4445 4446 4447 4448 4449

	validate_slab_cache(kmalloc_caches[4]);

	/* Hmmm... The next two are dangerous */
	p = kzalloc(32, GFP_KERNEL);
	p[32 + sizeof(void *)] = 0x34;
4450 4451 4452
	pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");
4453 4454 4455 4456 4457

	validate_slab_cache(kmalloc_caches[5]);
	p = kzalloc(64, GFP_KERNEL);
	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
	*p = 0x56;
4458 4459 4460
	pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");
4461 4462
	validate_slab_cache(kmalloc_caches[6]);

4463
	pr_err("\nB. Corruption after free\n");
4464 4465 4466
	p = kzalloc(128, GFP_KERNEL);
	kfree(p);
	*p = 0x78;
4467
	pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
4468 4469 4470 4471 4472
	validate_slab_cache(kmalloc_caches[7]);

	p = kzalloc(256, GFP_KERNEL);
	kfree(p);
	p[50] = 0x9a;
4473
	pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
4474 4475 4476 4477 4478
	validate_slab_cache(kmalloc_caches[8]);

	p = kzalloc(512, GFP_KERNEL);
	kfree(p);
	p[512] = 0xab;
4479
	pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
4480 4481 4482 4483 4484 4485 4486 4487
	validate_slab_cache(kmalloc_caches[9]);
}
#else
#ifdef CONFIG_SYSFS
static void resiliency_test(void) {};
#endif
#endif

4488
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
4489
enum slab_stat_type {
4490 4491 4492 4493 4494
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
C
Christoph Lameter 已提交
4495 4496
};

4497
#define SO_ALL		(1 << SL_ALL)
C
Christoph Lameter 已提交
4498 4499 4500
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
4501
#define SO_TOTAL	(1 << SL_TOTAL)
C
Christoph Lameter 已提交
4502

4503 4504
static ssize_t show_slab_objects(struct kmem_cache *s,
			    char *buf, unsigned long flags)
C
Christoph Lameter 已提交
4505 4506 4507 4508 4509 4510
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;

4511
	nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4512 4513
	if (!nodes)
		return -ENOMEM;
C
Christoph Lameter 已提交
4514

4515 4516
	if (flags & SO_CPU) {
		int cpu;
C
Christoph Lameter 已提交
4517

4518
		for_each_possible_cpu(cpu) {
4519 4520
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
							       cpu);
4521
			int node;
4522
			struct page *page;
4523

4524
			page = READ_ONCE(c->page);
4525 4526
			if (!page)
				continue;
4527

4528 4529 4530 4531 4532 4533 4534
			node = page_to_nid(page);
			if (flags & SO_TOTAL)
				x = page->objects;
			else if (flags & SO_OBJECTS)
				x = page->inuse;
			else
				x = 1;
4535

4536 4537 4538
			total += x;
			nodes[node] += x;

4539
			page = READ_ONCE(c->partial);
4540
			if (page) {
L
Li Zefan 已提交
4541 4542 4543 4544 4545 4546 4547
				node = page_to_nid(page);
				if (flags & SO_TOTAL)
					WARN_ON_ONCE(1);
				else if (flags & SO_OBJECTS)
					WARN_ON_ONCE(1);
				else
					x = page->pages;
4548 4549
				total += x;
				nodes[node] += x;
4550
			}
C
Christoph Lameter 已提交
4551 4552 4553
		}
	}

4554
	get_online_mems();
4555
#ifdef CONFIG_SLUB_DEBUG
4556
	if (flags & SO_ALL) {
C
Christoph Lameter 已提交
4557 4558 4559
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {
4560

4561 4562 4563 4564 4565
			if (flags & SO_TOTAL)
				x = atomic_long_read(&n->total_objects);
			else if (flags & SO_OBJECTS)
				x = atomic_long_read(&n->total_objects) -
					count_partial(n, count_free);
C
Christoph Lameter 已提交
4566
			else
4567
				x = atomic_long_read(&n->nr_slabs);
C
Christoph Lameter 已提交
4568 4569 4570 4571
			total += x;
			nodes[node] += x;
		}

4572 4573 4574
	} else
#endif
	if (flags & SO_PARTIAL) {
C
Christoph Lameter 已提交
4575
		struct kmem_cache_node *n;
C
Christoph Lameter 已提交
4576

C
Christoph Lameter 已提交
4577
		for_each_kmem_cache_node(s, node, n) {
4578 4579 4580 4581
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
C
Christoph Lameter 已提交
4582
			else
4583
				x = n->nr_partial;
C
Christoph Lameter 已提交
4584 4585 4586 4587 4588 4589
			total += x;
			nodes[node] += x;
		}
	}
	x = sprintf(buf, "%lu", total);
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4590
	for (node = 0; node < nr_node_ids; node++)
C
Christoph Lameter 已提交
4591 4592 4593 4594
		if (nodes[node])
			x += sprintf(buf + x, " N%d=%lu",
					node, nodes[node]);
#endif
4595
	put_online_mems();
C
Christoph Lameter 已提交
4596 4597 4598 4599
	kfree(nodes);
	return x + sprintf(buf + x, "\n");
}

4600
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4601 4602 4603
static int any_slab_objects(struct kmem_cache *s)
{
	int node;
C
Christoph Lameter 已提交
4604
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
4605

C
Christoph Lameter 已提交
4606
	for_each_kmem_cache_node(s, node, n)
4607
		if (atomic_long_read(&n->total_objects))
C
Christoph Lameter 已提交
4608
			return 1;
C
Christoph Lameter 已提交
4609

C
Christoph Lameter 已提交
4610 4611
	return 0;
}
4612
#endif
C
Christoph Lameter 已提交
4613 4614

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4615
#define to_slab(n) container_of(n, struct kmem_cache, kobj)
C
Christoph Lameter 已提交
4616 4617 4618 4619 4620 4621 4622 4623

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
4624 4625
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)
C
Christoph Lameter 已提交
4626 4627 4628

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
4629
	__ATTR(_name, 0600, _name##_show, _name##_store)
C
Christoph Lameter 已提交
4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->size);
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->align);
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
4645
	return sprintf(buf, "%d\n", s->object_size);
C
Christoph Lameter 已提交
4646 4647 4648 4649 4650
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
4651
	return sprintf(buf, "%d\n", oo_objects(s->oo));
C
Christoph Lameter 已提交
4652 4653 4654
}
SLAB_ATTR_RO(objs_per_slab);

4655 4656 4657
static ssize_t order_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
4658 4659 4660
	unsigned long order;
	int err;

4661
	err = kstrtoul(buf, 10, &order);
4662 4663
	if (err)
		return err;
4664 4665 4666 4667 4668 4669 4670 4671

	if (order > slub_max_order || order < slub_min_order)
		return -EINVAL;

	calculate_sizes(s, order);
	return length;
}

C
Christoph Lameter 已提交
4672 4673
static ssize_t order_show(struct kmem_cache *s, char *buf)
{
4674
	return sprintf(buf, "%d\n", oo_order(s->oo));
C
Christoph Lameter 已提交
4675
}
4676
SLAB_ATTR(order);
C
Christoph Lameter 已提交
4677

4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%lu\n", s->min_partial);
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

4689
	err = kstrtoul(buf, 10, &min);
4690 4691 4692
	if (err)
		return err;

4693
	set_min_partial(s, min);
4694 4695 4696 4697
	return length;
}
SLAB_ATTR(min_partial);

4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->cpu_partial);
}

static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long objects;
	int err;

4709
	err = kstrtoul(buf, 10, &objects);
4710 4711
	if (err)
		return err;
4712
	if (objects && !kmem_cache_has_cpu_partial(s))
4713
		return -EINVAL;
4714 4715 4716 4717 4718 4719 4720

	s->cpu_partial = objects;
	flush_all(s);
	return length;
}
SLAB_ATTR(cpu_partial);

C
Christoph Lameter 已提交
4721 4722
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
J
Joe Perches 已提交
4723 4724 4725
	if (!s->ctor)
		return 0;
	return sprintf(buf, "%pS\n", s->ctor);
C
Christoph Lameter 已提交
4726 4727 4728 4729 4730
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
4731
	return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
C
Christoph Lameter 已提交
4732 4733 4734 4735 4736
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
4737
	return show_slab_objects(s, buf, SO_PARTIAL);
C
Christoph Lameter 已提交
4738 4739 4740 4741 4742
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
4743
	return show_slab_objects(s, buf, SO_CPU);
C
Christoph Lameter 已提交
4744 4745 4746 4747 4748
}
SLAB_ATTR_RO(cpu_slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
4749
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
C
Christoph Lameter 已提交
4750 4751 4752
}
SLAB_ATTR_RO(objects);

4753 4754 4755 4756 4757 4758
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
	int pages = 0;
	int cpu;
	int len;

	for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;

		if (page) {
			pages += page->pages;
			objects += page->pobjects;
		}
	}

	len = sprintf(buf, "%d(%d)", objects, pages);

#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
		struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;

		if (page && len < PAGE_SIZE - 20)
			len += sprintf(buf + len, " C%d=%d(%d)", cpu,
				page->pobjects, page->pages);
	}
#endif
	return len + sprintf(buf + len, "\n");
}
SLAB_ATTR_RO(slabs_cpu_partial);

4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}

static ssize_t reclaim_account_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
	if (buf[0] == '1')
		s->flags |= SLAB_RECLAIM_ACCOUNT;
	return length;
}
SLAB_ATTR(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
}
SLAB_ATTR_RO(destroy_by_rcu);

4825 4826 4827 4828 4829 4830
static ssize_t reserved_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->reserved);
}
SLAB_ATTR_RO(reserved);

4831
#ifdef CONFIG_SLUB_DEBUG
4832 4833 4834 4835 4836 4837
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

4838 4839 4840 4841 4842 4843
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

C
Christoph Lameter 已提交
4844 4845
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
4846
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
C
Christoph Lameter 已提交
4847 4848 4849 4850 4851
}

static ssize_t sanity_checks_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
4852
	s->flags &= ~SLAB_CONSISTENCY_CHECKS;
4853 4854
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
4855
		s->flags |= SLAB_CONSISTENCY_CHECKS;
4856
	}
C
Christoph Lameter 已提交
4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868
	return length;
}
SLAB_ATTR(sanity_checks);

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
}

static ssize_t trace_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
4869 4870 4871 4872 4873 4874 4875 4876
	/*
	 * Tracing a merged cache is going to give confusing results
	 * as well as cause other issues like converting a mergeable
	 * cache into an umergeable one.
	 */
	if (s->refcount > 1)
		return -EINVAL;

C
Christoph Lameter 已提交
4877
	s->flags &= ~SLAB_TRACE;
4878 4879
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4880
		s->flags |= SLAB_TRACE;
4881
	}
C
Christoph Lameter 已提交
4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897
	return length;
}
SLAB_ATTR(trace);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}

static ssize_t red_zone_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_RED_ZONE;
4898
	if (buf[0] == '1') {
C
Christoph Lameter 已提交
4899
		s->flags |= SLAB_RED_ZONE;
4900
	}
4901
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917
	return length;
}
SLAB_ATTR(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}

static ssize_t poison_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_POISON;
4918
	if (buf[0] == '1') {
C
Christoph Lameter 已提交
4919
		s->flags |= SLAB_POISON;
4920
	}
4921
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937
	return length;
}
SLAB_ATTR(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}

static ssize_t store_user_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_STORE_USER;
4938 4939
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
4940
		s->flags |= SLAB_STORE_USER;
4941
	}
4942
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
4943 4944 4945 4946
	return length;
}
SLAB_ATTR(store_user);

4947 4948 4949 4950 4951 4952 4953 4954
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
4955 4956 4957 4958 4959 4960 4961 4962
	int ret = -EINVAL;

	if (buf[0] == '1') {
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
4963 4964
}
SLAB_ATTR(validate);
4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991

static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_ALLOC);
}
SLAB_ATTR_RO(alloc_calls);

static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_FREE);
}
SLAB_ATTR_RO(free_calls);
#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
}

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
4992 4993 4994
	if (s->refcount > 1)
		return -EINVAL;

4995 4996 4997 4998 4999 5000
	s->flags &= ~SLAB_FAILSLAB;
	if (buf[0] == '1')
		s->flags |= SLAB_FAILSLAB;
	return length;
}
SLAB_ATTR(failslab);
5001
#endif
5002

5003 5004 5005 5006 5007 5008 5009 5010
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
5011 5012 5013
	if (buf[0] == '1')
		kmem_cache_shrink(s);
	else
5014 5015 5016 5017 5018
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

C
Christoph Lameter 已提交
5019
#ifdef CONFIG_NUMA
5020
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
C
Christoph Lameter 已提交
5021
{
5022
	return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
C
Christoph Lameter 已提交
5023 5024
}

5025
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
C
Christoph Lameter 已提交
5026 5027
				const char *buf, size_t length)
{
5028 5029 5030
	unsigned long ratio;
	int err;

5031
	err = kstrtoul(buf, 10, &ratio);
5032 5033 5034
	if (err)
		return err;

5035
	if (ratio <= 100)
5036
		s->remote_node_defrag_ratio = ratio * 10;
C
Christoph Lameter 已提交
5037 5038 5039

	return length;
}
5040
SLAB_ATTR(remote_node_defrag_ratio);
C
Christoph Lameter 已提交
5041 5042
#endif

5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
	int len;
	int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
5055
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5056 5057 5058 5059 5060 5061 5062

		data[cpu] = x;
		sum += x;
	}

	len = sprintf(buf, "%lu", sum);

5063
#ifdef CONFIG_SMP
5064 5065
	for_each_online_cpu(cpu) {
		if (data[cpu] && len < PAGE_SIZE - 20)
5066
			len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5067
	}
5068
#endif
5069 5070 5071 5072
	kfree(data);
	return len + sprintf(buf + len, "\n");
}

D
David Rientjes 已提交
5073 5074 5075 5076 5077
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
5078
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
D
David Rientjes 已提交
5079 5080
}

5081 5082 5083 5084 5085
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
D
David Rientjes 已提交
5086 5087 5088 5089 5090 5091 5092 5093 5094
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
5106
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5107 5108 5109 5110 5111 5112 5113
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5114
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5115
STAT_ATTR(ORDER_FALLBACK, order_fallback);
5116 5117
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5118 5119
STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5120 5121
STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5122 5123
#endif

P
Pekka Enberg 已提交
5124
static struct attribute *slab_attrs[] = {
C
Christoph Lameter 已提交
5125 5126 5127 5128
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
5129
	&min_partial_attr.attr,
5130
	&cpu_partial_attr.attr,
C
Christoph Lameter 已提交
5131
	&objects_attr.attr,
5132
	&objects_partial_attr.attr,
C
Christoph Lameter 已提交
5133 5134 5135 5136 5137 5138 5139 5140
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
5141
	&shrink_attr.attr,
5142
	&reserved_attr.attr,
5143
	&slabs_cpu_partial_attr.attr,
5144
#ifdef CONFIG_SLUB_DEBUG
5145 5146 5147 5148
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
C
Christoph Lameter 已提交
5149 5150 5151
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
5152
	&validate_attr.attr,
5153 5154
	&alloc_calls_attr.attr,
	&free_calls_attr.attr,
5155
#endif
C
Christoph Lameter 已提交
5156 5157 5158 5159
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
5160
	&remote_node_defrag_ratio_attr.attr,
5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
5173
	&alloc_node_mismatch_attr.attr,
5174 5175 5176 5177 5178 5179 5180
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
5181
	&deactivate_bypass_attr.attr,
5182
	&order_fallback_attr.attr,
5183 5184
	&cmpxchg_double_fail_attr.attr,
	&cmpxchg_double_cpu_fail_attr.attr,
5185 5186
	&cpu_partial_alloc_attr.attr,
	&cpu_partial_free_attr.attr,
5187 5188
	&cpu_partial_node_attr.attr,
	&cpu_partial_drain_attr.attr,
C
Christoph Lameter 已提交
5189
#endif
5190 5191 5192 5193
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif

C
Christoph Lameter 已提交
5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234
	NULL
};

static struct attribute_group slab_attr_group = {
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);
5235
#ifdef CONFIG_MEMCG
5236
	if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
5237
		struct kmem_cache *c;
C
Christoph Lameter 已提交
5238

5239 5240 5241 5242
		mutex_lock(&slab_mutex);
		if (s->max_attr_size < len)
			s->max_attr_size = len;

5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259
		/*
		 * This is a best effort propagation, so this function's return
		 * value will be determined by the parent cache only. This is
		 * basically because not all attributes will have a well
		 * defined semantics for rollbacks - most of the actions will
		 * have permanent effects.
		 *
		 * Returning the error value of any of the children that fail
		 * is not 100 % defined, in the sense that users seeing the
		 * error code won't be able to know anything about the state of
		 * the cache.
		 *
		 * Only returning the error code for the parent cache at least
		 * has well defined semantics. The cache being written to
		 * directly either failed or succeeded, in which case we loop
		 * through the descendants with best-effort propagation.
		 */
5260 5261
		for_each_memcg_cache(c, s)
			attribute->store(c, buf, len);
5262 5263 5264
		mutex_unlock(&slab_mutex);
	}
#endif
C
Christoph Lameter 已提交
5265 5266 5267
	return err;
}

5268 5269
static void memcg_propagate_slab_attrs(struct kmem_cache *s)
{
5270
#ifdef CONFIG_MEMCG
5271 5272
	int i;
	char *buffer = NULL;
5273
	struct kmem_cache *root_cache;
5274

5275
	if (is_root_cache(s))
5276 5277
		return;

5278
	root_cache = s->memcg_params.root_cache;
5279

5280 5281 5282 5283
	/*
	 * This mean this cache had no attribute written. Therefore, no point
	 * in copying default values around
	 */
5284
	if (!root_cache->max_attr_size)
5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305
		return;

	for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
		char mbuf[64];
		char *buf;
		struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);

		if (!attr || !attr->store || !attr->show)
			continue;

		/*
		 * It is really bad that we have to allocate here, so we will
		 * do it only as a fallback. If we actually allocate, though,
		 * we can just use the allocated buffer until the end.
		 *
		 * Most of the slub attributes will tend to be very small in
		 * size, but sysfs allows buffers up to a page, so they can
		 * theoretically happen.
		 */
		if (buffer)
			buf = buffer;
5306
		else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
5307 5308 5309 5310 5311 5312 5313 5314
			buf = mbuf;
		else {
			buffer = (char *) get_zeroed_page(GFP_KERNEL);
			if (WARN_ON(!buffer))
				continue;
			buf = buffer;
		}

5315
		attr->show(root_cache, buf);
5316 5317 5318 5319 5320 5321 5322 5323
		attr->store(s, buf, strlen(buf));
	}

	if (buffer)
		free_page((unsigned long)buffer);
#endif
}

5324 5325 5326 5327 5328
static void kmem_cache_release(struct kobject *k)
{
	slab_kmem_cache_release(to_slab(k));
}

5329
static const struct sysfs_ops slab_sysfs_ops = {
C
Christoph Lameter 已提交
5330 5331 5332 5333 5334 5335
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
5336
	.release = kmem_cache_release,
C
Christoph Lameter 已提交
5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;
}

5348
static const struct kset_uevent_ops slab_uevent_ops = {
C
Christoph Lameter 已提交
5349 5350 5351
	.filter = uevent_filter,
};

5352
static struct kset *slab_kset;
C
Christoph Lameter 已提交
5353

5354 5355
static inline struct kset *cache_kset(struct kmem_cache *s)
{
5356
#ifdef CONFIG_MEMCG
5357
	if (!is_root_cache(s))
5358
		return s->memcg_params.root_cache->memcg_kset;
5359 5360 5361 5362
#endif
	return slab_kset;
}

C
Christoph Lameter 已提交
5363 5364 5365
#define ID_STR_LENGTH 64

/* Create a unique string id for a slab cache:
C
Christoph Lameter 已提交
5366 5367
 *
 * Format	:[flags-]size
C
Christoph Lameter 已提交
5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

	BUG_ON(!name);

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
5388
	if (s->flags & SLAB_CONSISTENCY_CHECKS)
C
Christoph Lameter 已提交
5389
		*p++ = 'F';
V
Vegard Nossum 已提交
5390 5391
	if (!(s->flags & SLAB_NOTRACK))
		*p++ = 't';
V
Vladimir Davydov 已提交
5392 5393
	if (s->flags & SLAB_ACCOUNT)
		*p++ = 'A';
C
Christoph Lameter 已提交
5394 5395 5396
	if (p != name + 1)
		*p++ = '-';
	p += sprintf(p, "%07d", s->size);
5397

C
Christoph Lameter 已提交
5398 5399 5400 5401 5402 5403 5404 5405
	BUG_ON(p > name + ID_STR_LENGTH - 1);
	return name;
}

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
5406
	int unmergeable = slab_unmergeable(s);
C
Christoph Lameter 已提交
5407 5408 5409 5410 5411 5412 5413

	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
5414
		sysfs_remove_link(&slab_kset->kobj, s->name);
C
Christoph Lameter 已提交
5415 5416 5417 5418 5419 5420 5421 5422 5423
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
	}

5424
	s->kobj.kset = cache_kset(s);
5425
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
5426
	if (err)
5427
		goto out;
C
Christoph Lameter 已提交
5428 5429

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
5430 5431
	if (err)
		goto out_del_kobj;
5432

5433
#ifdef CONFIG_MEMCG
5434 5435 5436
	if (is_root_cache(s)) {
		s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
		if (!s->memcg_kset) {
5437 5438
			err = -ENOMEM;
			goto out_del_kobj;
5439 5440 5441 5442
		}
	}
#endif

C
Christoph Lameter 已提交
5443 5444 5445 5446 5447
	kobject_uevent(&s->kobj, KOBJ_ADD);
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
	}
5448 5449 5450 5451 5452 5453 5454
out:
	if (!unmergeable)
		kfree(name);
	return err;
out_del_kobj:
	kobject_del(&s->kobj);
	goto out;
C
Christoph Lameter 已提交
5455 5456
}

5457
void sysfs_slab_remove(struct kmem_cache *s)
C
Christoph Lameter 已提交
5458
{
5459
	if (slab_state < FULL)
5460 5461 5462 5463 5464 5465
		/*
		 * Sysfs has not been setup yet so no need to remove the
		 * cache from sysfs.
		 */
		return;

5466
#ifdef CONFIG_MEMCG
5467 5468
	kset_unregister(s->memcg_kset);
#endif
C
Christoph Lameter 已提交
5469 5470
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
	kobject_del(&s->kobj);
C
Christoph Lameter 已提交
5471
	kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5472 5473 5474 5475
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
N
Nick Andrew 已提交
5476
 * available lest we lose that information.
C
Christoph Lameter 已提交
5477 5478 5479 5480 5481 5482 5483
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

A
Adrian Bunk 已提交
5484
static struct saved_alias *alias_list;
C
Christoph Lameter 已提交
5485 5486 5487 5488 5489

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

5490
	if (slab_state == FULL) {
C
Christoph Lameter 已提交
5491 5492 5493
		/*
		 * If we have a leftover link then remove it.
		 */
5494 5495
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
C
Christoph Lameter 已提交
5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
	return 0;
}

static int __init slab_sysfs_init(void)
{
5511
	struct kmem_cache *s;
C
Christoph Lameter 已提交
5512 5513
	int err;

5514
	mutex_lock(&slab_mutex);
5515

5516
	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5517
	if (!slab_kset) {
5518
		mutex_unlock(&slab_mutex);
5519
		pr_err("Cannot register slab subsystem.\n");
C
Christoph Lameter 已提交
5520 5521 5522
		return -ENOSYS;
	}

5523
	slab_state = FULL;
5524

5525
	list_for_each_entry(s, &slab_caches, list) {
5526
		err = sysfs_slab_add(s);
5527
		if (err)
5528 5529
			pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
			       s->name);
5530
	}
C
Christoph Lameter 已提交
5531 5532 5533 5534 5535 5536

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
5537
		if (err)
5538 5539
			pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
			       al->name);
C
Christoph Lameter 已提交
5540 5541 5542
		kfree(al);
	}

5543
	mutex_unlock(&slab_mutex);
C
Christoph Lameter 已提交
5544 5545 5546 5547 5548
	resiliency_test();
	return 0;
}

__initcall(slab_sysfs_init);
5549
#endif /* CONFIG_SYSFS */
P
Pekka J Enberg 已提交
5550 5551 5552 5553

/*
 * The /proc/slabinfo ABI
 */
5554
#ifdef CONFIG_SLABINFO
5555
void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
P
Pekka J Enberg 已提交
5556 5557
{
	unsigned long nr_slabs = 0;
5558 5559
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
P
Pekka J Enberg 已提交
5560
	int node;
C
Christoph Lameter 已提交
5561
	struct kmem_cache_node *n;
P
Pekka J Enberg 已提交
5562

C
Christoph Lameter 已提交
5563
	for_each_kmem_cache_node(s, node, n) {
5564 5565
		nr_slabs += node_nr_slabs(n);
		nr_objs += node_nr_objs(n);
5566
		nr_free += count_partial(n, count_free);
P
Pekka J Enberg 已提交
5567 5568
	}

5569 5570 5571 5572 5573 5574
	sinfo->active_objs = nr_objs - nr_free;
	sinfo->num_objs = nr_objs;
	sinfo->active_slabs = nr_slabs;
	sinfo->num_slabs = nr_slabs;
	sinfo->objects_per_slab = oo_objects(s->oo);
	sinfo->cache_order = oo_order(s->oo);
P
Pekka J Enberg 已提交
5575 5576
}

5577
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
5578 5579 5580
{
}

5581 5582
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos)
5583
{
5584
	return -EIO;
5585
}
5586
#endif /* CONFIG_SLABINFO */