slub.c 142.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
C
Christoph Lameter 已提交
2 3 4 5
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
6 7
 * The allocator synchronizes using per slab locks or atomic operatios
 * and only uses a centralized lock to manage a pool of partial slabs.
C
Christoph Lameter 已提交
8
 *
C
Christoph Lameter 已提交
9
 * (C) 2007 SGI, Christoph Lameter
10
 * (C) 2011 Linux Foundation, Christoph Lameter
C
Christoph Lameter 已提交
11 12 13
 */

#include <linux/mm.h>
N
Nick Piggin 已提交
14
#include <linux/swap.h> /* struct reclaim_state */
C
Christoph Lameter 已提交
15 16 17 18 19
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
#include <linux/bitops.h>
#include <linux/slab.h>
20
#include "slab.h"
21
#include <linux/proc_fs.h>
C
Christoph Lameter 已提交
22
#include <linux/seq_file.h>
23
#include <linux/kasan.h>
C
Christoph Lameter 已提交
24 25 26 27
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
28
#include <linux/debugobjects.h>
C
Christoph Lameter 已提交
29
#include <linux/kallsyms.h>
30
#include <linux/memory.h>
R
Roman Zippel 已提交
31
#include <linux/math64.h>
A
Akinobu Mita 已提交
32
#include <linux/fault-inject.h>
33
#include <linux/stacktrace.h>
34
#include <linux/prefetch.h>
35
#include <linux/memcontrol.h>
36
#include <linux/random.h>
C
Christoph Lameter 已提交
37

38 39
#include <trace/events/kmem.h>

40 41
#include "internal.h"

C
Christoph Lameter 已提交
42 43
/*
 * Lock order:
44
 *   1. slab_mutex (Global Mutex)
45 46
 *   2. node->list_lock
 *   3. slab_lock(page) (Only on some arches and for debugging)
C
Christoph Lameter 已提交
47
 *
48
 *   slab_mutex
49
 *
50
 *   The role of the slab_mutex is to protect the list of all the slabs
51 52 53
 *   and to synchronize major metadata changes to slab cache structures.
 *
 *   The slab_lock is only used for debugging and on arches that do not
54
 *   have the ability to do a cmpxchg_double. It only protects:
55
 *	A. page->freelist	-> List of object free in a page
56 57 58
 *	B. page->inuse		-> Number of objects in use
 *	C. page->objects	-> Number of objects in page
 *	D. page->frozen		-> frozen state
59 60 61 62 63 64
 *
 *   If a slab is frozen then it is exempt from list management. It is not
 *   on any list. The processor that froze the slab is the one who can
 *   perform list operations on the page. Other processors may put objects
 *   onto the freelist but the processor that froze the slab is the only
 *   one that can retrieve the objects from the page's freelist.
C
Christoph Lameter 已提交
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
 *
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
 *   Interrupts are disabled during allocation and deallocation in order to
 *   make the slab allocator safe to use in the context of an irq. In addition
 *   interrupts are disabled to ensure that the processor does not change
 *   while handling per_cpu slabs, due to kernel preemption.
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
C
Christoph Lameter 已提交
85 86
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
C
Christoph Lameter 已提交
87
 * freed then the slab will show up again on the partial lists.
C
Christoph Lameter 已提交
88 89
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
C
Christoph Lameter 已提交
90 91 92 93 94 95 96
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
 * Overloading of page flags that are otherwise used for LRU management.
 *
97 98 99 100 101 102 103 104 105 106 107 108
 * PageActive 		The slab is frozen and exempt from list processing.
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
109
 * 			freelist that allows lockless access to
110 111
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
C
Christoph Lameter 已提交
112 113 114
 *
 * PageError		Slab requires special handling due to debug
 * 			options set. This moves	slab handling out of
115
 * 			the fast path and disables lockless freelists.
C
Christoph Lameter 已提交
116 117
 */

118 119
static inline int kmem_cache_debug(struct kmem_cache *s)
{
120
#ifdef CONFIG_SLUB_DEBUG
121
	return unlikely(s->flags & SLAB_DEBUG_FLAGS);
122
#else
123
	return 0;
124
#endif
125
}
126

127
void *fixup_red_left(struct kmem_cache *s, void *p)
128 129 130 131 132 133 134
{
	if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE)
		p += s->red_left_pad;

	return p;
}

135 136 137 138 139 140 141 142 143
static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	return !kmem_cache_debug(s);
#else
	return false;
#endif
}

C
Christoph Lameter 已提交
144 145 146 147 148 149 150 151 152 153 154
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

/* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST

155 156 157
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG

158 159 160 161
/*
 * Mininum number of partial slabs. These will be left on the partial
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
162
#define MIN_PARTIAL 5
C
Christoph Lameter 已提交
163

164 165 166
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
167
 * sort the partial list by the number of objects in use.
168 169 170
 */
#define MAX_PARTIAL 10

171
#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
C
Christoph Lameter 已提交
172
				SLAB_POISON | SLAB_STORE_USER)
C
Christoph Lameter 已提交
173

174 175 176 177 178 179 180 181
/*
 * These debug flags cannot use CMPXCHG because there might be consistency
 * issues when checking or reading debug information
 */
#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)


182
/*
183 184 185
 * Debugging flags that require metadata to be stored in the slab.  These get
 * disabled when slub_debug=O is used and a cache's min order increases with
 * metadata.
186
 */
187
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
188

189 190
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
191
#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */
192

C
Christoph Lameter 已提交
193
/* Internal SLUB flags */
194
/* Poison object */
195
#define __OBJECT_POISON		((slab_flags_t __force)0x80000000U)
196
/* Use cmpxchg_double */
197
#define __CMPXCHG_DOUBLE	((slab_flags_t __force)0x40000000U)
C
Christoph Lameter 已提交
198

199 200 201
/*
 * Tracking user of a slab.
 */
202
#define TRACK_ADDRS_COUNT 16
203
struct track {
204
	unsigned long addr;	/* Called from address */
205 206 207
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
208 209 210 211 212 213 214
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

215
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
216 217
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
218
static void memcg_propagate_slab_attrs(struct kmem_cache *s);
219
static void sysfs_slab_remove(struct kmem_cache *s);
C
Christoph Lameter 已提交
220
#else
221 222 223
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
224
static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
225
static inline void sysfs_slab_remove(struct kmem_cache *s) { }
C
Christoph Lameter 已提交
226 227
#endif

228
static inline void stat(const struct kmem_cache *s, enum stat_item si)
229 230
{
#ifdef CONFIG_SLUB_STATS
231 232 233 234 235
	/*
	 * The rmw is racy on a preemptible kernel but this is acceptable, so
	 * avoid this_cpu_add()'s irq-disable overhead.
	 */
	raw_cpu_inc(s->cpu_slab->stat[si]);
236 237 238
#endif
}

C
Christoph Lameter 已提交
239 240 241 242
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
/*
 * Returns freelist pointer (ptr). With hardening, this is obfuscated
 * with an XOR of the address where the pointer is held and a per-cache
 * random number.
 */
static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
				 unsigned long ptr_addr)
{
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	return (void *)((unsigned long)ptr ^ s->random ^ ptr_addr);
#else
	return ptr;
#endif
}

/* Returns the freelist pointer recorded at location ptr_addr. */
static inline void *freelist_dereference(const struct kmem_cache *s,
					 void *ptr_addr)
{
	return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
			    (unsigned long)ptr_addr);
}

266 267
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
268
	return freelist_dereference(s, object + s->offset);
269 270
}

271 272
static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{
273
	prefetch(object + s->offset);
274 275
}

276 277
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
278
	unsigned long freepointer_addr;
279 280
	void *p;

281 282 283
	if (!debug_pagealloc_enabled())
		return get_freepointer(s, object);

284 285 286
	freepointer_addr = (unsigned long)object + s->offset;
	probe_kernel_read(&p, (void **)freepointer_addr, sizeof(p));
	return freelist_ptr(s, p, freepointer_addr);
287 288
}

289 290
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
291 292
	unsigned long freeptr_addr = (unsigned long)object + s->offset;

293 294 295 296
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	BUG_ON(object == fp); /* naive detection of double free or corruption */
#endif

297
	*(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
298 299 300
}

/* Loop over all objects in a slab */
301
#define for_each_object(__p, __s, __addr, __objects) \
302 303 304
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)
305

306
#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
307 308 309
	for (__p = fixup_red_left(__s, __addr), __idx = 1; \
		__idx <= __objects; \
		__p += (__s)->size, __idx++)
310

311
/* Determine object index from a given position */
312
static inline unsigned int slab_index(void *p, struct kmem_cache *s, void *addr)
313 314 315 316
{
	return (p - addr) / s->size;
}

317
static inline unsigned int order_objects(unsigned int order, unsigned int size)
318
{
319
	return ((unsigned int)PAGE_SIZE << order) / size;
320 321
}

322
static inline struct kmem_cache_order_objects oo_make(unsigned int order,
323
		unsigned int size)
324 325
{
	struct kmem_cache_order_objects x = {
326
		(order << OO_SHIFT) + order_objects(order, size)
327 328 329 330 331
	};

	return x;
}

332
static inline unsigned int oo_order(struct kmem_cache_order_objects x)
333
{
334
	return x.x >> OO_SHIFT;
335 336
}

337
static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
338
{
339
	return x.x & OO_MASK;
340 341
}

342 343 344 345 346
/*
 * Per slab locking using the pagelock
 */
static __always_inline void slab_lock(struct page *page)
{
347
	VM_BUG_ON_PAGE(PageTail(page), page);
348 349 350 351 352
	bit_spin_lock(PG_locked, &page->flags);
}

static __always_inline void slab_unlock(struct page *page)
{
353
	VM_BUG_ON_PAGE(PageTail(page), page);
354 355 356
	__bit_spin_unlock(PG_locked, &page->flags);
}

357 358 359 360 361 362 363
/* Interrupts must be disabled (for the fallback code to work right) */
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
	VM_BUG_ON(!irqs_disabled());
364 365
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
366
	if (s->flags & __CMPXCHG_DOUBLE) {
367
		if (cmpxchg_double(&page->freelist, &page->counters,
368 369
				   freelist_old, counters_old,
				   freelist_new, counters_new))
370
			return true;
371 372 373 374
	} else
#endif
	{
		slab_lock(page);
375 376
		if (page->freelist == freelist_old &&
					page->counters == counters_old) {
377
			page->freelist = freelist_new;
378
			page->counters = counters_new;
379
			slab_unlock(page);
380
			return true;
381 382 383 384 385 386 387 388
		}
		slab_unlock(page);
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
389
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
390 391
#endif

392
	return false;
393 394
}

395 396 397 398 399
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
400 401
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
402
	if (s->flags & __CMPXCHG_DOUBLE) {
403
		if (cmpxchg_double(&page->freelist, &page->counters,
404 405
				   freelist_old, counters_old,
				   freelist_new, counters_new))
406
			return true;
407 408 409
	} else
#endif
	{
410 411 412
		unsigned long flags;

		local_irq_save(flags);
413
		slab_lock(page);
414 415
		if (page->freelist == freelist_old &&
					page->counters == counters_old) {
416
			page->freelist = freelist_new;
417
			page->counters = counters_new;
418
			slab_unlock(page);
419
			local_irq_restore(flags);
420
			return true;
421
		}
422
		slab_unlock(page);
423
		local_irq_restore(flags);
424 425 426 427 428 429
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
430
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
431 432
#endif

433
	return false;
434 435
}

436
#ifdef CONFIG_SLUB_DEBUG
437 438 439
/*
 * Determine a map of object in use on a page.
 *
440
 * Node listlock must be held to guarantee that the page does
441 442 443 444 445 446 447 448 449 450 451
 * not vanish from under us.
 */
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
{
	void *p;
	void *addr = page_address(page);

	for (p = page->freelist; p; p = get_freepointer(s, p))
		set_bit(slab_index(p, s, addr), map);
}

452
static inline unsigned int size_from_object(struct kmem_cache *s)
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
{
	if (s->flags & SLAB_RED_ZONE)
		return s->size - s->red_left_pad;

	return s->size;
}

static inline void *restore_red_left(struct kmem_cache *s, void *p)
{
	if (s->flags & SLAB_RED_ZONE)
		p -= s->red_left_pad;

	return p;
}

468 469 470
/*
 * Debug settings:
 */
471
#if defined(CONFIG_SLUB_DEBUG_ON)
472
static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
473
#else
474
static slab_flags_t slub_debug;
475
#endif
476 477

static char *slub_debug_slabs;
478
static int disable_higher_order_debug;
479

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
/*
 * slub is about to manipulate internal object metadata.  This memory lies
 * outside the range of the allocated object, so accessing it would normally
 * be reported by kasan as a bounds error.  metadata_access_enable() is used
 * to tell kasan that these accesses are OK.
 */
static inline void metadata_access_enable(void)
{
	kasan_disable_current();
}

static inline void metadata_access_disable(void)
{
	kasan_enable_current();
}

C
Christoph Lameter 已提交
496 497 498
/*
 * Object debugging
 */
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518

/* Verify that a pointer has an address that is valid within a slab page */
static inline int check_valid_pointer(struct kmem_cache *s,
				struct page *page, void *object)
{
	void *base;

	if (!object)
		return 1;

	base = page_address(page);
	object = restore_red_left(s, object);
	if (object < base || object >= base + page->objects * s->size ||
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

519 520
static void print_section(char *level, char *text, u8 *addr,
			  unsigned int length)
C
Christoph Lameter 已提交
521
{
522
	metadata_access_enable();
523
	print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
524
			length, 1);
525
	metadata_access_disable();
C
Christoph Lameter 已提交
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
}

static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}

static void set_track(struct kmem_cache *s, void *object,
542
			enum track_item alloc, unsigned long addr)
C
Christoph Lameter 已提交
543
{
A
Akinobu Mita 已提交
544
	struct track *p = get_track(s, object, alloc);
C
Christoph Lameter 已提交
545 546

	if (addr) {
547 548 549 550 551 552 553 554
#ifdef CONFIG_STACKTRACE
		struct stack_trace trace;
		int i;

		trace.nr_entries = 0;
		trace.max_entries = TRACK_ADDRS_COUNT;
		trace.entries = p->addrs;
		trace.skip = 3;
555
		metadata_access_enable();
556
		save_stack_trace(&trace);
557
		metadata_access_disable();
558 559 560 561 562 563 564 565 566

		/* See rant in lockdep.c */
		if (trace.nr_entries != 0 &&
		    trace.entries[trace.nr_entries - 1] == ULONG_MAX)
			trace.nr_entries--;

		for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
			p->addrs[i] = 0;
#endif
C
Christoph Lameter 已提交
567 568
		p->addr = addr;
		p->cpu = smp_processor_id();
569
		p->pid = current->pid;
C
Christoph Lameter 已提交
570 571 572 573 574 575 576
		p->when = jiffies;
	} else
		memset(p, 0, sizeof(struct track));
}

static void init_tracking(struct kmem_cache *s, void *object)
{
577 578 579
	if (!(s->flags & SLAB_STORE_USER))
		return;

580 581
	set_track(s, object, TRACK_FREE, 0UL);
	set_track(s, object, TRACK_ALLOC, 0UL);
C
Christoph Lameter 已提交
582 583
}

584
static void print_track(const char *s, struct track *t, unsigned long pr_time)
C
Christoph Lameter 已提交
585 586 587 588
{
	if (!t->addr)
		return;

589
	pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
590
	       s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
591 592 593 594 595
#ifdef CONFIG_STACKTRACE
	{
		int i;
		for (i = 0; i < TRACK_ADDRS_COUNT; i++)
			if (t->addrs[i])
596
				pr_err("\t%pS\n", (void *)t->addrs[i]);
597 598 599 600
			else
				break;
	}
#endif
601 602 603 604
}

static void print_tracking(struct kmem_cache *s, void *object)
{
605
	unsigned long pr_time = jiffies;
606 607 608
	if (!(s->flags & SLAB_STORE_USER))
		return;

609 610
	print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
	print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
611 612 613 614
}

static void print_page_info(struct page *page)
{
615
	pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
616
	       page, page->objects, page->inuse, page->freelist, page->flags);
617 618 619 620 621

}

static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
622
	struct va_format vaf;
623 624 625
	va_list args;

	va_start(args, fmt);
626 627
	vaf.fmt = fmt;
	vaf.va = &args;
628
	pr_err("=============================================================================\n");
629
	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
630
	pr_err("-----------------------------------------------------------------------------\n\n");
631

632
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
633
	va_end(args);
C
Christoph Lameter 已提交
634 635
}

636 637
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
638
	struct va_format vaf;
639 640 641
	va_list args;

	va_start(args, fmt);
642 643 644
	vaf.fmt = fmt;
	vaf.va = &args;
	pr_err("FIX %s: %pV\n", s->name, &vaf);
645 646 647 648
	va_end(args);
}

static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
C
Christoph Lameter 已提交
649 650
{
	unsigned int off;	/* Offset of last byte */
651
	u8 *addr = page_address(page);
652 653 654 655 656

	print_tracking(s, p);

	print_page_info(page);

657 658
	pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
	       p, p - addr, get_freepointer(s, p));
659

660
	if (s->flags & SLAB_RED_ZONE)
661 662
		print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
			      s->red_left_pad);
663
	else if (p > addr + 16)
664
		print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
C
Christoph Lameter 已提交
665

666
	print_section(KERN_ERR, "Object ", p,
667
		      min_t(unsigned int, s->object_size, PAGE_SIZE));
C
Christoph Lameter 已提交
668
	if (s->flags & SLAB_RED_ZONE)
669
		print_section(KERN_ERR, "Redzone ", p + s->object_size,
670
			s->inuse - s->object_size);
C
Christoph Lameter 已提交
671 672 673 674 675 676

	if (s->offset)
		off = s->offset + sizeof(void *);
	else
		off = s->inuse;

677
	if (s->flags & SLAB_STORE_USER)
C
Christoph Lameter 已提交
678 679
		off += 2 * sizeof(struct track);

680 681
	off += kasan_metadata_size(s);

682
	if (off != size_from_object(s))
C
Christoph Lameter 已提交
683
		/* Beginning of the filler is the free pointer */
684 685
		print_section(KERN_ERR, "Padding ", p + off,
			      size_from_object(s) - off);
686 687

	dump_stack();
C
Christoph Lameter 已提交
688 689
}

690
void object_err(struct kmem_cache *s, struct page *page,
C
Christoph Lameter 已提交
691 692
			u8 *object, char *reason)
{
693
	slab_bug(s, "%s", reason);
694
	print_trailer(s, page, object);
C
Christoph Lameter 已提交
695 696
}

697
static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
698
			const char *fmt, ...)
C
Christoph Lameter 已提交
699 700 701 702
{
	va_list args;
	char buf[100];

703 704
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
C
Christoph Lameter 已提交
705
	va_end(args);
706
	slab_bug(s, "%s", buf);
707
	print_page_info(page);
C
Christoph Lameter 已提交
708 709 710
	dump_stack();
}

711
static void init_object(struct kmem_cache *s, void *object, u8 val)
C
Christoph Lameter 已提交
712 713 714
{
	u8 *p = object;

715 716 717
	if (s->flags & SLAB_RED_ZONE)
		memset(p - s->red_left_pad, val, s->red_left_pad);

C
Christoph Lameter 已提交
718
	if (s->flags & __OBJECT_POISON) {
719 720
		memset(p, POISON_FREE, s->object_size - 1);
		p[s->object_size - 1] = POISON_END;
C
Christoph Lameter 已提交
721 722 723
	}

	if (s->flags & SLAB_RED_ZONE)
724
		memset(p + s->object_size, val, s->inuse - s->object_size);
C
Christoph Lameter 已提交
725 726
}

727 728 729 730 731 732 733 734 735
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
	memset(from, data, to - from);
}

static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
			u8 *object, char *what,
736
			u8 *start, unsigned int value, unsigned int bytes)
737 738 739 740
{
	u8 *fault;
	u8 *end;

741
	metadata_access_enable();
742
	fault = memchr_inv(start, value, bytes);
743
	metadata_access_disable();
744 745 746 747 748 749 750 751
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

	slab_bug(s, "%s overwritten", what);
752
	pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
753 754 755 756 757
					fault, end - 1, fault[0], value);
	print_trailer(s, page, object);

	restore_bytes(s, what, value, fault, end);
	return 0;
C
Christoph Lameter 已提交
758 759 760 761 762 763 764 765 766
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
 * 	pointer is the first word of the object.
C
Christoph Lameter 已提交
767
 *
C
Christoph Lameter 已提交
768 769 770
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
771
 * object + s->object_size
C
Christoph Lameter 已提交
772
 * 	Padding to reach word boundary. This is also used for Redzoning.
C
Christoph Lameter 已提交
773
 * 	Padding is extended by another word if Redzoning is enabled and
774
 * 	object_size == inuse.
C
Christoph Lameter 已提交
775
 *
C
Christoph Lameter 已提交
776 777 778 779
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
C
Christoph Lameter 已提交
780 781
 * 	Meta data starts here.
 *
C
Christoph Lameter 已提交
782 783
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
C
Christoph Lameter 已提交
784
 * 	C. Padding to reach required alignment boundary or at mininum
C
Christoph Lameter 已提交
785
 * 		one word if debugging is on to be able to detect writes
C
Christoph Lameter 已提交
786 787 788
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
C
Christoph Lameter 已提交
789 790
 *
 * object + s->size
C
Christoph Lameter 已提交
791
 * 	Nothing is used beyond s->size.
C
Christoph Lameter 已提交
792
 *
793
 * If slabcaches are merged then the object_size and inuse boundaries are mostly
C
Christoph Lameter 已提交
794
 * ignored. And therefore no slab options that rely on these boundaries
C
Christoph Lameter 已提交
795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
 * may be used with merged slabcaches.
 */

static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
{
	unsigned long off = s->inuse;	/* The end of info */

	if (s->offset)
		/* Freepointer is placed after the object. */
		off += sizeof(void *);

	if (s->flags & SLAB_STORE_USER)
		/* We also have user information there */
		off += 2 * sizeof(struct track);

810 811
	off += kasan_metadata_size(s);

812
	if (size_from_object(s) == off)
C
Christoph Lameter 已提交
813 814
		return 1;

815
	return check_bytes_and_report(s, page, p, "Object padding",
816
			p + off, POISON_INUSE, size_from_object(s) - off);
C
Christoph Lameter 已提交
817 818
}

819
/* Check the pad bytes at the end of a slab page */
C
Christoph Lameter 已提交
820 821
static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
822 823 824
	u8 *start;
	u8 *fault;
	u8 *end;
825
	u8 *pad;
826 827
	int length;
	int remainder;
C
Christoph Lameter 已提交
828 829 830 831

	if (!(s->flags & SLAB_POISON))
		return 1;

832
	start = page_address(page);
833
	length = PAGE_SIZE << compound_order(page);
834 835
	end = start + length;
	remainder = length % s->size;
C
Christoph Lameter 已提交
836 837 838
	if (!remainder)
		return 1;

839
	pad = end - remainder;
840
	metadata_access_enable();
841
	fault = memchr_inv(pad, POISON_INUSE, remainder);
842
	metadata_access_disable();
843 844 845 846 847 848
	if (!fault)
		return 1;
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
849
	print_section(KERN_ERR, "Padding ", pad, remainder);
850

851
	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
852
	return 0;
C
Christoph Lameter 已提交
853 854 855
}

static int check_object(struct kmem_cache *s, struct page *page,
856
					void *object, u8 val)
C
Christoph Lameter 已提交
857 858
{
	u8 *p = object;
859
	u8 *endobject = object + s->object_size;
C
Christoph Lameter 已提交
860 861

	if (s->flags & SLAB_RED_ZONE) {
862 863 864 865
		if (!check_bytes_and_report(s, page, object, "Redzone",
			object - s->red_left_pad, val, s->red_left_pad))
			return 0;

866
		if (!check_bytes_and_report(s, page, object, "Redzone",
867
			endobject, val, s->inuse - s->object_size))
C
Christoph Lameter 已提交
868 869
			return 0;
	} else {
870
		if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
871
			check_bytes_and_report(s, page, p, "Alignment padding",
872 873
				endobject, POISON_INUSE,
				s->inuse - s->object_size);
874
		}
C
Christoph Lameter 已提交
875 876 877
	}

	if (s->flags & SLAB_POISON) {
878
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
879
			(!check_bytes_and_report(s, page, p, "Poison", p,
880
					POISON_FREE, s->object_size - 1) ||
881
			 !check_bytes_and_report(s, page, p, "Poison",
882
				p + s->object_size - 1, POISON_END, 1)))
C
Christoph Lameter 已提交
883 884 885 886 887 888 889
			return 0;
		/*
		 * check_pad_bytes cleans up on its own.
		 */
		check_pad_bytes(s, page, p);
	}

890
	if (!s->offset && val == SLUB_RED_ACTIVE)
C
Christoph Lameter 已提交
891 892 893 894 895 896 897 898 899 900
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
	if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
		object_err(s, page, p, "Freepointer corrupt");
		/*
901
		 * No choice but to zap it and thus lose the remainder
C
Christoph Lameter 已提交
902
		 * of the free objects in this slab. May cause
C
Christoph Lameter 已提交
903
		 * another error because the object count is now wrong.
C
Christoph Lameter 已提交
904
		 */
905
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
906 907 908 909 910 911 912
		return 0;
	}
	return 1;
}

static int check_slab(struct kmem_cache *s, struct page *page)
{
913 914
	int maxobj;

C
Christoph Lameter 已提交
915 916 917
	VM_BUG_ON(!irqs_disabled());

	if (!PageSlab(page)) {
918
		slab_err(s, page, "Not a valid slab page");
C
Christoph Lameter 已提交
919 920
		return 0;
	}
921

922
	maxobj = order_objects(compound_order(page), s->size);
923 924
	if (page->objects > maxobj) {
		slab_err(s, page, "objects %u > max %u",
925
			page->objects, maxobj);
926 927 928
		return 0;
	}
	if (page->inuse > page->objects) {
929
		slab_err(s, page, "inuse %u > max %u",
930
			page->inuse, page->objects);
C
Christoph Lameter 已提交
931 932 933 934 935 936 937 938
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
	slab_pad_check(s, page);
	return 1;
}

/*
C
Christoph Lameter 已提交
939 940
 * Determine if a certain object on a page is on the freelist. Must hold the
 * slab lock to guarantee that the chains are in a consistent state.
C
Christoph Lameter 已提交
941 942 943 944
 */
static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{
	int nr = 0;
945
	void *fp;
C
Christoph Lameter 已提交
946
	void *object = NULL;
947
	int max_objects;
C
Christoph Lameter 已提交
948

949
	fp = page->freelist;
950
	while (fp && nr <= page->objects) {
C
Christoph Lameter 已提交
951 952 953 954 955 956
		if (fp == search)
			return 1;
		if (!check_valid_pointer(s, page, fp)) {
			if (object) {
				object_err(s, page, object,
					"Freechain corrupt");
957
				set_freepointer(s, object, NULL);
C
Christoph Lameter 已提交
958
			} else {
959
				slab_err(s, page, "Freepointer corrupt");
960
				page->freelist = NULL;
961
				page->inuse = page->objects;
962
				slab_fix(s, "Freelist cleared");
C
Christoph Lameter 已提交
963 964 965 966 967 968 969 970 971
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

972
	max_objects = order_objects(compound_order(page), s->size);
973 974
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
975 976

	if (page->objects != max_objects) {
977 978
		slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
			 page->objects, max_objects);
979 980 981
		page->objects = max_objects;
		slab_fix(s, "Number of objects adjusted.");
	}
982
	if (page->inuse != page->objects - nr) {
983 984
		slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
			 page->inuse, page->objects - nr);
985
		page->inuse = page->objects - nr;
986
		slab_fix(s, "Object count adjusted.");
C
Christoph Lameter 已提交
987 988 989 990
	}
	return search == NULL;
}

991 992
static void trace(struct kmem_cache *s, struct page *page, void *object,
								int alloc)
993 994
{
	if (s->flags & SLAB_TRACE) {
995
		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
996 997 998 999 1000 1001
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
1002
			print_section(KERN_INFO, "Object ", (void *)object,
1003
					s->object_size);
1004 1005 1006 1007 1008

		dump_stack();
	}
}

1009
/*
C
Christoph Lameter 已提交
1010
 * Tracking of fully allocated slabs for debugging purposes.
1011
 */
1012 1013
static void add_full(struct kmem_cache *s,
	struct kmem_cache_node *n, struct page *page)
1014
{
1015 1016 1017
	if (!(s->flags & SLAB_STORE_USER))
		return;

1018
	lockdep_assert_held(&n->list_lock);
1019 1020 1021
	list_add(&page->lru, &n->full);
}

1022
static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
1023 1024 1025 1026
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

1027
	lockdep_assert_held(&n->list_lock);
1028 1029 1030
	list_del(&page->lru);
}

1031 1032 1033 1034 1035 1036 1037 1038
/* Tracking of the number of slabs for debugging purposes */
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
{
	struct kmem_cache_node *n = get_node(s, node);

	return atomic_long_read(&n->nr_slabs);
}

1039 1040 1041 1042 1043
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

1044
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1045 1046 1047 1048 1049 1050 1051 1052 1053
{
	struct kmem_cache_node *n = get_node(s, node);

	/*
	 * May be called early in order to allocate a slab for the
	 * kmem_cache_node structure. Solve the chicken-egg
	 * dilemma by deferring the increment of the count during
	 * bootstrap (see early_kmem_cache_node_alloc).
	 */
1054
	if (likely(n)) {
1055
		atomic_long_inc(&n->nr_slabs);
1056 1057
		atomic_long_add(objects, &n->total_objects);
	}
1058
}
1059
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1060 1061 1062 1063
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
1064
	atomic_long_sub(objects, &n->total_objects);
1065 1066 1067
}

/* Object debug checks for alloc/free paths */
1068 1069 1070 1071 1072 1073
static void setup_object_debug(struct kmem_cache *s, struct page *page,
								void *object)
{
	if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
		return;

1074
	init_object(s, object, SLUB_RED_INACTIVE);
1075 1076 1077
	init_tracking(s, object);
}

1078
static inline int alloc_consistency_checks(struct kmem_cache *s,
1079
					struct page *page,
1080
					void *object, unsigned long addr)
C
Christoph Lameter 已提交
1081 1082
{
	if (!check_slab(s, page))
1083
		return 0;
C
Christoph Lameter 已提交
1084 1085 1086

	if (!check_valid_pointer(s, page, object)) {
		object_err(s, page, object, "Freelist Pointer check fails");
1087
		return 0;
C
Christoph Lameter 已提交
1088 1089
	}

1090
	if (!check_object(s, page, object, SLUB_RED_INACTIVE))
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
		return 0;

	return 1;
}

static noinline int alloc_debug_processing(struct kmem_cache *s,
					struct page *page,
					void *object, unsigned long addr)
{
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!alloc_consistency_checks(s, page, object, addr))
			goto bad;
	}
C
Christoph Lameter 已提交
1104

1105 1106 1107 1108
	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
	trace(s, page, object, 1);
1109
	init_object(s, object, SLUB_RED_ACTIVE);
C
Christoph Lameter 已提交
1110
	return 1;
1111

C
Christoph Lameter 已提交
1112 1113 1114 1115 1116
bad:
	if (PageSlab(page)) {
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
C
Christoph Lameter 已提交
1117
		 * as used avoids touching the remaining objects.
C
Christoph Lameter 已提交
1118
		 */
1119
		slab_fix(s, "Marking all objects used");
1120
		page->inuse = page->objects;
1121
		page->freelist = NULL;
C
Christoph Lameter 已提交
1122 1123 1124 1125
	}
	return 0;
}

1126 1127
static inline int free_consistency_checks(struct kmem_cache *s,
		struct page *page, void *object, unsigned long addr)
C
Christoph Lameter 已提交
1128 1129
{
	if (!check_valid_pointer(s, page, object)) {
1130
		slab_err(s, page, "Invalid object pointer 0x%p", object);
1131
		return 0;
C
Christoph Lameter 已提交
1132 1133 1134
	}

	if (on_freelist(s, page, object)) {
1135
		object_err(s, page, object, "Object already free");
1136
		return 0;
C
Christoph Lameter 已提交
1137 1138
	}

1139
	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1140
		return 0;
C
Christoph Lameter 已提交
1141

1142
	if (unlikely(s != page->slab_cache)) {
1143
		if (!PageSlab(page)) {
1144 1145
			slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
				 object);
1146
		} else if (!page->slab_cache) {
1147 1148
			pr_err("SLUB <none>: no slab for object 0x%p.\n",
			       object);
1149
			dump_stack();
1150
		} else
1151 1152
			object_err(s, page, object,
					"page slab pointer corrupt.");
1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183
		return 0;
	}
	return 1;
}

/* Supports checking bulk free of a constructed freelist */
static noinline int free_debug_processing(
	struct kmem_cache *s, struct page *page,
	void *head, void *tail, int bulk_cnt,
	unsigned long addr)
{
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	void *object = head;
	int cnt = 0;
	unsigned long uninitialized_var(flags);
	int ret = 0;

	spin_lock_irqsave(&n->list_lock, flags);
	slab_lock(page);

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!check_slab(s, page))
			goto out;
	}

next_object:
	cnt++;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!free_consistency_checks(s, page, object, addr))
			goto out;
C
Christoph Lameter 已提交
1184
	}
1185 1186 1187 1188

	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
1189
	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
1190
	init_object(s, object, SLUB_RED_INACTIVE);
1191 1192 1193 1194 1195 1196

	/* Reached end of constructed freelist yet? */
	if (object != tail) {
		object = get_freepointer(s, object);
		goto next_object;
	}
1197 1198
	ret = 1;

1199
out:
1200 1201 1202 1203
	if (cnt != bulk_cnt)
		slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
			 bulk_cnt, cnt);

1204
	slab_unlock(page);
1205
	spin_unlock_irqrestore(&n->list_lock, flags);
1206 1207 1208
	if (!ret)
		slab_fix(s, "Object at 0x%p not freed", object);
	return ret;
C
Christoph Lameter 已提交
1209 1210
}

1211 1212
static int __init setup_slub_debug(char *str)
{
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
	slub_debug = DEBUG_DEFAULT_FLAGS;
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	if (*str == ',')
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
		goto check_slabs;

	slub_debug = 0;
	if (*str == '-')
		/*
		 * Switch off all debugging measures.
		 */
		goto out;

	/*
	 * Determine which debug features should be switched on
	 */
1237
	for (; *str && *str != ','; str++) {
1238 1239
		switch (tolower(*str)) {
		case 'f':
1240
			slub_debug |= SLAB_CONSISTENCY_CHECKS;
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
			break;
		case 'z':
			slub_debug |= SLAB_RED_ZONE;
			break;
		case 'p':
			slub_debug |= SLAB_POISON;
			break;
		case 'u':
			slub_debug |= SLAB_STORE_USER;
			break;
		case 't':
			slub_debug |= SLAB_TRACE;
			break;
1254 1255 1256
		case 'a':
			slub_debug |= SLAB_FAILSLAB;
			break;
1257 1258 1259 1260 1261 1262 1263
		case 'o':
			/*
			 * Avoid enabling debugging on caches if its minimum
			 * order would increase as a result.
			 */
			disable_higher_order_debug = 1;
			break;
1264
		default:
1265 1266
			pr_err("slub_debug option '%c' unknown. skipped\n",
			       *str);
1267
		}
1268 1269
	}

1270
check_slabs:
1271 1272
	if (*str == ',')
		slub_debug_slabs = str + 1;
1273
out:
1274 1275 1276 1277 1278
	return 1;
}

__setup("slub_debug", setup_slub_debug);

1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
/*
 * kmem_cache_flags - apply debugging options to the cache
 * @object_size:	the size of an object without meta data
 * @flags:		flags to set
 * @name:		name of the cache
 * @ctor:		constructor function
 *
 * Debug option(s) are applied to @flags. In addition to the debug
 * option(s), if a slab name (or multiple) is specified i.e.
 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
 * then only the select slabs will receive the debug option(s).
 */
1291
slab_flags_t kmem_cache_flags(unsigned int object_size,
1292
	slab_flags_t flags, const char *name,
1293
	void (*ctor)(void *))
1294
{
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	char *iter;
	size_t len;

	/* If slub_debug = 0, it folds into the if conditional. */
	if (!slub_debug_slabs)
		return flags | slub_debug;

	len = strlen(name);
	iter = slub_debug_slabs;
	while (*iter) {
		char *end, *glob;
		size_t cmplen;

		end = strchr(iter, ',');
		if (!end)
			end = iter + strlen(iter);

		glob = strnchr(iter, end - iter, '*');
		if (glob)
			cmplen = glob - iter;
		else
			cmplen = max_t(size_t, len, (end - iter));

		if (!strncmp(name, iter, cmplen)) {
			flags |= slub_debug;
			break;
		}

		if (!*end)
			break;
		iter = end + 1;
	}
1327 1328

	return flags;
1329
}
1330
#else /* !CONFIG_SLUB_DEBUG */
1331 1332
static inline void setup_object_debug(struct kmem_cache *s,
			struct page *page, void *object) {}
1333

1334
static inline int alloc_debug_processing(struct kmem_cache *s,
1335
	struct page *page, void *object, unsigned long addr) { return 0; }
1336

1337
static inline int free_debug_processing(
1338 1339
	struct kmem_cache *s, struct page *page,
	void *head, void *tail, int bulk_cnt,
1340
	unsigned long addr) { return 0; }
1341 1342 1343 1344

static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
			{ return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page,
1345
			void *object, u8 val) { return 1; }
1346 1347
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
					struct page *page) {}
1348 1349
static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
					struct page *page) {}
1350
slab_flags_t kmem_cache_flags(unsigned int object_size,
1351
	slab_flags_t flags, const char *name,
1352
	void (*ctor)(void *))
1353 1354 1355
{
	return flags;
}
1356
#define slub_debug 0
1357

1358 1359
#define disable_higher_order_debug 0

1360 1361
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }
1362 1363
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1364 1365 1366 1367
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1368

1369 1370 1371 1372 1373 1374
#endif /* CONFIG_SLUB_DEBUG */

/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
 */
1375
static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
1376 1377
{
	kmemleak_alloc(ptr, size, 1, flags);
1378
	return kasan_kmalloc_large(ptr, size, flags);
1379 1380
}

1381
static __always_inline void kfree_hook(void *x)
1382 1383
{
	kmemleak_free(x);
1384
	kasan_kfree_large(x, _RET_IP_);
1385 1386
}

1387
static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x)
1388 1389
{
	kmemleak_free_recursive(x, s->flags);
1390

1391 1392 1393 1394 1395
	/*
	 * Trouble is that we may no longer disable interrupts in the fast path
	 * So in order to make the debug calls that expect irqs to be
	 * disabled we need to disable interrupts temporarily.
	 */
1396
#ifdef CONFIG_LOCKDEP
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
	{
		unsigned long flags;

		local_irq_save(flags);
		debug_check_no_locks_freed(x, s->object_size);
		local_irq_restore(flags);
	}
#endif
	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->object_size);
1407

1408 1409
	/* KASAN might put x into memory quarantine, delaying its reuse */
	return kasan_slab_free(s, x, _RET_IP_);
1410
}
1411

1412 1413
static inline bool slab_free_freelist_hook(struct kmem_cache *s,
					   void **head, void **tail)
1414 1415 1416 1417 1418
{
/*
 * Compiler cannot detect this function can be removed if slab_free_hook()
 * evaluates to nothing.  Thus, catch all relevant config debug options here.
 */
1419
#if defined(CONFIG_LOCKDEP)	||		\
1420 1421 1422 1423
	defined(CONFIG_DEBUG_KMEMLEAK) ||	\
	defined(CONFIG_DEBUG_OBJECTS_FREE) ||	\
	defined(CONFIG_KASAN)

1424 1425 1426 1427 1428 1429 1430
	void *object;
	void *next = *head;
	void *old_tail = *tail ? *tail : *head;

	/* Head and tail of the reconstructed freelist */
	*head = NULL;
	*tail = NULL;
1431 1432

	do {
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
		object = next;
		next = get_freepointer(s, object);
		/* If object's reuse doesn't have to be delayed */
		if (!slab_free_hook(s, object)) {
			/* Move object to the new freelist */
			set_freepointer(s, object, *head);
			*head = object;
			if (!*tail)
				*tail = object;
		}
	} while (object != old_tail);

	if (*head == *tail)
		*tail = NULL;

	return *head != NULL;
#else
	return true;
1451 1452 1453
#endif
}

1454
static void *setup_object(struct kmem_cache *s, struct page *page,
1455 1456 1457
				void *object)
{
	setup_object_debug(s, page, object);
1458
	object = kasan_init_slab_obj(s, object);
1459 1460 1461 1462 1463
	if (unlikely(s->ctor)) {
		kasan_unpoison_object_data(s, object);
		s->ctor(object);
		kasan_poison_object_data(s, object);
	}
1464
	return object;
1465 1466
}

C
Christoph Lameter 已提交
1467 1468 1469
/*
 * Slab allocation and freeing
 */
1470 1471
static inline struct page *alloc_slab_page(struct kmem_cache *s,
		gfp_t flags, int node, struct kmem_cache_order_objects oo)
1472
{
1473
	struct page *page;
1474
	unsigned int order = oo_order(oo);
1475

1476
	if (node == NUMA_NO_NODE)
1477
		page = alloc_pages(flags, order);
1478
	else
1479
		page = __alloc_pages_node(node, flags, order);
1480

1481 1482 1483 1484
	if (page && memcg_charge_slab(page, flags, order, s)) {
		__free_pages(page, order);
		page = NULL;
	}
1485 1486

	return page;
1487 1488
}

1489 1490 1491 1492
#ifdef CONFIG_SLAB_FREELIST_RANDOM
/* Pre-initialize the random sequence cache */
static int init_cache_random_seq(struct kmem_cache *s)
{
1493
	unsigned int count = oo_objects(s->oo);
1494 1495
	int err;

1496 1497 1498 1499
	/* Bailout if already initialised */
	if (s->random_seq)
		return 0;

1500 1501 1502 1503 1504 1505 1506 1507 1508
	err = cache_random_seq_create(s, count, GFP_KERNEL);
	if (err) {
		pr_err("SLUB: Unable to initialize free list for %s\n",
			s->name);
		return err;
	}

	/* Transform to an offset on the set of pages */
	if (s->random_seq) {
1509 1510
		unsigned int i;

1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
		for (i = 0; i < count; i++)
			s->random_seq[i] *= s->size;
	}
	return 0;
}

/* Initialize each random sequence freelist per cache */
static void __init init_freelist_randomization(void)
{
	struct kmem_cache *s;

	mutex_lock(&slab_mutex);

	list_for_each_entry(s, &slab_caches, list)
		init_cache_random_seq(s);

	mutex_unlock(&slab_mutex);
}

/* Get the next entry on the pre-computed freelist randomized */
static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
				unsigned long *pos, void *start,
				unsigned long page_limit,
				unsigned long freelist_count)
{
	unsigned int idx;

	/*
	 * If the target page allocation failed, the number of objects on the
	 * page might be smaller than the usual size defined by the cache.
	 */
	do {
		idx = s->random_seq[*pos];
		*pos += 1;
		if (*pos >= freelist_count)
			*pos = 0;
	} while (unlikely(idx >= page_limit));

	return (char *)start + idx;
}

/* Shuffle the single linked freelist based on a random pre-computed sequence */
static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
{
	void *start;
	void *cur;
	void *next;
	unsigned long idx, pos, page_limit, freelist_count;

	if (page->objects < 2 || !s->random_seq)
		return false;

	freelist_count = oo_objects(s->oo);
	pos = get_random_int() % freelist_count;

	page_limit = page->objects * s->size;
	start = fixup_red_left(s, page_address(page));

	/* First entry is used as the base of the freelist */
	cur = next_freelist_entry(s, page, &pos, start, page_limit,
				freelist_count);
1572
	cur = setup_object(s, page, cur);
1573 1574 1575 1576 1577
	page->freelist = cur;

	for (idx = 1; idx < page->objects; idx++) {
		next = next_freelist_entry(s, page, &pos, start, page_limit,
			freelist_count);
1578
		next = setup_object(s, page, next);
1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
		set_freepointer(s, cur, next);
		cur = next;
	}
	set_freepointer(s, cur, NULL);

	return true;
}
#else
static inline int init_cache_random_seq(struct kmem_cache *s)
{
	return 0;
}
static inline void init_freelist_randomization(void) { }
static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
{
	return false;
}
#endif /* CONFIG_SLAB_FREELIST_RANDOM */

C
Christoph Lameter 已提交
1598 1599
static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
{
1600
	struct page *page;
1601
	struct kmem_cache_order_objects oo = s->oo;
1602
	gfp_t alloc_gfp;
1603
	void *start, *p, *next;
1604
	int idx, order;
1605
	bool shuffle;
C
Christoph Lameter 已提交
1606

1607 1608
	flags &= gfp_allowed_mask;

1609
	if (gfpflags_allow_blocking(flags))
1610 1611
		local_irq_enable();

1612
	flags |= s->allocflags;
1613

1614 1615 1616 1617 1618
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1619
	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1620
		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
1621

1622
	page = alloc_slab_page(s, alloc_gfp, node, oo);
1623 1624
	if (unlikely(!page)) {
		oo = s->min;
1625
		alloc_gfp = flags;
1626 1627 1628 1629
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
1630
		page = alloc_slab_page(s, alloc_gfp, node, oo);
1631 1632 1633
		if (unlikely(!page))
			goto out;
		stat(s, ORDER_FALLBACK);
1634
	}
1635

1636
	page->objects = oo_objects(oo);
C
Christoph Lameter 已提交
1637

1638
	order = compound_order(page);
1639
	page->slab_cache = s;
1640
	__SetPageSlab(page);
1641
	if (page_is_pfmemalloc(page))
1642
		SetPageSlabPfmemalloc(page);
C
Christoph Lameter 已提交
1643 1644 1645 1646

	start = page_address(page);

	if (unlikely(s->flags & SLAB_POISON))
1647
		memset(start, POISON_INUSE, PAGE_SIZE << order);
C
Christoph Lameter 已提交
1648

1649 1650
	kasan_poison_slab(page);

1651 1652 1653 1654
	shuffle = shuffle_freelist(s, page);

	if (!shuffle) {
		for_each_object_idx(p, idx, s, start, page->objects) {
1655 1656 1657 1658 1659
			if (likely(idx < page->objects)) {
				next = p + s->size;
				next = setup_object(s, page, next);
				set_freepointer(s, p, next);
			} else
1660 1661
				set_freepointer(s, p, NULL);
		}
1662 1663 1664
		start = fixup_red_left(s, start);
		start = setup_object(s, page, start);
		page->freelist = start;
C
Christoph Lameter 已提交
1665 1666
	}

1667
	page->inuse = page->objects;
1668
	page->frozen = 1;
1669

C
Christoph Lameter 已提交
1670
out:
1671
	if (gfpflags_allow_blocking(flags))
1672 1673 1674 1675
		local_irq_disable();
	if (!page)
		return NULL;

1676
	mod_lruvec_page_state(page,
1677 1678 1679 1680 1681 1682
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
		1 << oo_order(oo));

	inc_slabs_node(s, page_to_nid(page), page->objects);

C
Christoph Lameter 已提交
1683 1684 1685
	return page;
}

1686 1687 1688
static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
{
	if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
1689
		gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
1690 1691 1692
		flags &= ~GFP_SLAB_BUG_MASK;
		pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
				invalid_mask, &invalid_mask, flags, &flags);
1693
		dump_stack();
1694 1695 1696 1697 1698 1699
	}

	return allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
}

C
Christoph Lameter 已提交
1700 1701
static void __free_slab(struct kmem_cache *s, struct page *page)
{
1702 1703
	int order = compound_order(page);
	int pages = 1 << order;
C
Christoph Lameter 已提交
1704

1705
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
C
Christoph Lameter 已提交
1706 1707 1708
		void *p;

		slab_pad_check(s, page);
1709 1710
		for_each_object(p, s, page_address(page),
						page->objects)
1711
			check_object(s, page, p, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
1712 1713
	}

1714
	mod_lruvec_page_state(page,
C
Christoph Lameter 已提交
1715 1716
		(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1717
		-pages);
C
Christoph Lameter 已提交
1718

1719
	__ClearPageSlabPfmemalloc(page);
1720
	__ClearPageSlab(page);
1721

1722
	page->mapping = NULL;
N
Nick Piggin 已提交
1723 1724
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
1725 1726
	memcg_uncharge_slab(page, order, s);
	__free_pages(page, order);
C
Christoph Lameter 已提交
1727 1728 1729 1730
}

static void rcu_free_slab(struct rcu_head *h)
{
1731
	struct page *page = container_of(h, struct page, rcu_head);
1732

1733
	__free_slab(page->slab_cache, page);
C
Christoph Lameter 已提交
1734 1735 1736 1737
}

static void free_slab(struct kmem_cache *s, struct page *page)
{
1738
	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
1739
		call_rcu(&page->rcu_head, rcu_free_slab);
C
Christoph Lameter 已提交
1740 1741 1742 1743 1744 1745
	} else
		__free_slab(s, page);
}

static void discard_slab(struct kmem_cache *s, struct page *page)
{
1746
	dec_slabs_node(s, page_to_nid(page), page->objects);
C
Christoph Lameter 已提交
1747 1748 1749 1750
	free_slab(s, page);
}

/*
1751
 * Management of partially allocated slabs.
C
Christoph Lameter 已提交
1752
 */
1753 1754
static inline void
__add_partial(struct kmem_cache_node *n, struct page *page, int tail)
C
Christoph Lameter 已提交
1755
{
C
Christoph Lameter 已提交
1756
	n->nr_partial++;
1757
	if (tail == DEACTIVATE_TO_TAIL)
1758 1759 1760
		list_add_tail(&page->lru, &n->partial);
	else
		list_add(&page->lru, &n->partial);
C
Christoph Lameter 已提交
1761 1762
}

1763 1764
static inline void add_partial(struct kmem_cache_node *n,
				struct page *page, int tail)
1765
{
1766
	lockdep_assert_held(&n->list_lock);
1767 1768
	__add_partial(n, page, tail);
}
1769

1770 1771 1772 1773
static inline void remove_partial(struct kmem_cache_node *n,
					struct page *page)
{
	lockdep_assert_held(&n->list_lock);
1774 1775
	list_del(&page->lru);
	n->nr_partial--;
1776 1777
}

C
Christoph Lameter 已提交
1778
/*
1779 1780
 * Remove slab from the partial list, freeze it and
 * return the pointer to the freelist.
C
Christoph Lameter 已提交
1781
 *
1782
 * Returns a list of objects or NULL if it fails.
C
Christoph Lameter 已提交
1783
 */
1784
static inline void *acquire_slab(struct kmem_cache *s,
1785
		struct kmem_cache_node *n, struct page *page,
1786
		int mode, int *objects)
C
Christoph Lameter 已提交
1787
{
1788 1789 1790 1791
	void *freelist;
	unsigned long counters;
	struct page new;

1792 1793
	lockdep_assert_held(&n->list_lock);

1794 1795 1796 1797 1798
	/*
	 * Zap the freelist and set the frozen bit.
	 * The old freelist is the list of objects for the
	 * per cpu allocation list.
	 */
1799 1800 1801
	freelist = page->freelist;
	counters = page->counters;
	new.counters = counters;
1802
	*objects = new.objects - new.inuse;
1803
	if (mode) {
1804
		new.inuse = page->objects;
1805 1806 1807 1808
		new.freelist = NULL;
	} else {
		new.freelist = freelist;
	}
1809

1810
	VM_BUG_ON(new.frozen);
1811
	new.frozen = 1;
1812

1813
	if (!__cmpxchg_double_slab(s, page,
1814
			freelist, counters,
1815
			new.freelist, new.counters,
1816 1817
			"acquire_slab"))
		return NULL;
1818 1819

	remove_partial(n, page);
1820
	WARN_ON(!freelist);
1821
	return freelist;
C
Christoph Lameter 已提交
1822 1823
}

1824
static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
1825
static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
1826

C
Christoph Lameter 已提交
1827
/*
C
Christoph Lameter 已提交
1828
 * Try to allocate a partial slab from a specific node.
C
Christoph Lameter 已提交
1829
 */
1830 1831
static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
				struct kmem_cache_cpu *c, gfp_t flags)
C
Christoph Lameter 已提交
1832
{
1833 1834
	struct page *page, *page2;
	void *object = NULL;
1835
	unsigned int available = 0;
1836
	int objects;
C
Christoph Lameter 已提交
1837 1838 1839 1840

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
C
Christoph Lameter 已提交
1841 1842
	 * partial slab and there is none available then get_partials()
	 * will return NULL.
C
Christoph Lameter 已提交
1843 1844 1845 1846 1847
	 */
	if (!n || !n->nr_partial)
		return NULL;

	spin_lock(&n->list_lock);
1848
	list_for_each_entry_safe(page, page2, &n->partial, lru) {
1849
		void *t;
1850

1851 1852 1853
		if (!pfmemalloc_match(page, flags))
			continue;

1854
		t = acquire_slab(s, n, page, object == NULL, &objects);
1855 1856 1857
		if (!t)
			break;

1858
		available += objects;
1859
		if (!object) {
1860 1861 1862 1863
			c->page = page;
			stat(s, ALLOC_FROM_PARTIAL);
			object = t;
		} else {
1864
			put_cpu_partial(s, page, 0);
1865
			stat(s, CPU_PARTIAL_NODE);
1866
		}
1867
		if (!kmem_cache_has_cpu_partial(s)
1868
			|| available > slub_cpu_partial(s) / 2)
1869 1870
			break;

1871
	}
C
Christoph Lameter 已提交
1872
	spin_unlock(&n->list_lock);
1873
	return object;
C
Christoph Lameter 已提交
1874 1875 1876
}

/*
C
Christoph Lameter 已提交
1877
 * Get a page from somewhere. Search in increasing NUMA distances.
C
Christoph Lameter 已提交
1878
 */
1879
static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
1880
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1881 1882 1883
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
1884
	struct zoneref *z;
1885 1886
	struct zone *zone;
	enum zone_type high_zoneidx = gfp_zone(flags);
1887
	void *object;
1888
	unsigned int cpuset_mems_cookie;
C
Christoph Lameter 已提交
1889 1890

	/*
C
Christoph Lameter 已提交
1891 1892 1893 1894
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
C
Christoph Lameter 已提交
1895
	 *
C
Christoph Lameter 已提交
1896 1897 1898 1899
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
C
Christoph Lameter 已提交
1900
	 *
1901 1902 1903 1904 1905
	 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
	 * (which makes defrag_ratio = 1000) then every (well almost)
	 * allocation will first attempt to defrag slab caches on other nodes.
	 * This means scanning over all nodes to look for partial slabs which
	 * may be expensive if we do it every time we are trying to find a slab
C
Christoph Lameter 已提交
1906
	 * with available objects.
C
Christoph Lameter 已提交
1907
	 */
1908 1909
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
C
Christoph Lameter 已提交
1910 1911
		return NULL;

1912
	do {
1913
		cpuset_mems_cookie = read_mems_allowed_begin();
1914
		zonelist = node_zonelist(mempolicy_slab_node(), flags);
1915 1916 1917 1918 1919
		for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
			struct kmem_cache_node *n;

			n = get_node(s, zone_to_nid(zone));

1920
			if (n && cpuset_zone_allowed(zone, flags) &&
1921
					n->nr_partial > s->min_partial) {
1922
				object = get_partial_node(s, n, c, flags);
1923 1924
				if (object) {
					/*
1925 1926 1927 1928 1929
					 * Don't check read_mems_allowed_retry()
					 * here - if mems_allowed was updated in
					 * parallel, that was a harmless race
					 * between allocation and the cpuset
					 * update
1930 1931 1932
					 */
					return object;
				}
1933
			}
C
Christoph Lameter 已提交
1934
		}
1935
	} while (read_mems_allowed_retry(cpuset_mems_cookie));
C
Christoph Lameter 已提交
1936 1937 1938 1939 1940 1941 1942
#endif
	return NULL;
}

/*
 * Get a partial page, lock it and return it.
 */
1943
static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
1944
		struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
1945
{
1946
	void *object;
1947 1948 1949 1950 1951 1952
	int searchnode = node;

	if (node == NUMA_NO_NODE)
		searchnode = numa_mem_id();
	else if (!node_present_pages(node))
		searchnode = node_to_mem_node(node);
C
Christoph Lameter 已提交
1953

1954
	object = get_partial_node(s, get_node(s, searchnode), c, flags);
1955 1956
	if (object || node != NUMA_NO_NODE)
		return object;
C
Christoph Lameter 已提交
1957

1958
	return get_any_partial(s, flags, c);
C
Christoph Lameter 已提交
1959 1960
}

1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
#ifdef CONFIG_PREEMPT
/*
 * Calculate the next globally unique transaction for disambiguiation
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
#endif

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

2002
	pr_info("%s %s: cmpxchg redo ", n, s->name);
2003 2004 2005

#ifdef CONFIG_PREEMPT
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
2006
		pr_warn("due to cpu change %d -> %d\n",
2007 2008 2009 2010
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
2011
		pr_warn("due to cpu running other code. Event %ld->%ld\n",
2012 2013
			tid_to_event(tid), tid_to_event(actual_tid));
	else
2014
		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
2015 2016
			actual_tid, tid, next_tid(tid));
#endif
2017
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
2018 2019
}

2020
static void init_kmem_cache_cpus(struct kmem_cache *s)
2021 2022 2023 2024 2025 2026
{
	int cpu;

	for_each_possible_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
}
2027

C
Christoph Lameter 已提交
2028 2029 2030
/*
 * Remove the cpu slab
 */
2031
static void deactivate_slab(struct kmem_cache *s, struct page *page,
2032
				void *freelist, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2033
{
2034 2035 2036 2037 2038
	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	int lock = 0;
	enum slab_modes l = M_NONE, m = M_NONE;
	void *nextfree;
2039
	int tail = DEACTIVATE_TO_HEAD;
2040 2041 2042 2043
	struct page new;
	struct page old;

	if (page->freelist) {
2044
		stat(s, DEACTIVATE_REMOTE_FREES);
2045
		tail = DEACTIVATE_TO_TAIL;
2046 2047
	}

2048
	/*
2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
	 * Stage one: Free all available per cpu objects back
	 * to the page freelist while it is still frozen. Leave the
	 * last one.
	 *
	 * There is no need to take the list->lock because the page
	 * is still frozen.
	 */
	while (freelist && (nextfree = get_freepointer(s, freelist))) {
		void *prior;
		unsigned long counters;

		do {
			prior = page->freelist;
			counters = page->counters;
			set_freepointer(s, freelist, prior);
			new.counters = counters;
			new.inuse--;
2066
			VM_BUG_ON(!new.frozen);
2067

2068
		} while (!__cmpxchg_double_slab(s, page,
2069 2070 2071 2072 2073 2074 2075
			prior, counters,
			freelist, new.counters,
			"drain percpu freelist"));

		freelist = nextfree;
	}

2076
	/*
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
	 * Stage two: Ensure that the page is unfrozen while the
	 * list presence reflects the actual number of objects
	 * during unfreeze.
	 *
	 * We setup the list membership and then perform a cmpxchg
	 * with the count. If there is a mismatch then the page
	 * is not unfrozen but the page is on the wrong list.
	 *
	 * Then we restart the process which may have to remove
	 * the page from the list that we just put it on again
	 * because the number of objects in the slab may have
	 * changed.
2089
	 */
2090
redo:
2091

2092 2093
	old.freelist = page->freelist;
	old.counters = page->counters;
2094
	VM_BUG_ON(!old.frozen);
2095

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
	/* Determine target state of the slab */
	new.counters = old.counters;
	if (freelist) {
		new.inuse--;
		set_freepointer(s, freelist, old.freelist);
		new.freelist = freelist;
	} else
		new.freelist = old.freelist;

	new.frozen = 0;

2107
	if (!new.inuse && n->nr_partial >= s->min_partial)
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
		m = M_FREE;
	else if (new.freelist) {
		m = M_PARTIAL;
		if (!lock) {
			lock = 1;
			/*
			 * Taking the spinlock removes the possiblity
			 * that acquire_slab() will see a slab page that
			 * is frozen
			 */
			spin_lock(&n->list_lock);
		}
	} else {
		m = M_FULL;
		if (kmem_cache_debug(s) && !lock) {
			lock = 1;
			/*
			 * This also ensures that the scanning of full
			 * slabs from diagnostic functions will not see
			 * any frozen slabs.
			 */
			spin_lock(&n->list_lock);
		}
	}

	if (l != m) {
		if (l == M_PARTIAL)
			remove_partial(n, page);
		else if (l == M_FULL)
2137
			remove_full(s, n, page);
2138

2139
		if (m == M_PARTIAL)
2140
			add_partial(n, page, tail);
2141
		else if (m == M_FULL)
2142 2143 2144 2145
			add_full(s, n, page);
	}

	l = m;
2146
	if (!__cmpxchg_double_slab(s, page,
2147 2148 2149 2150 2151 2152 2153 2154
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"))
		goto redo;

	if (lock)
		spin_unlock(&n->list_lock);

2155 2156 2157 2158 2159
	if (m == M_PARTIAL)
		stat(s, tail);
	else if (m == M_FULL)
		stat(s, DEACTIVATE_FULL);
	else if (m == M_FREE) {
2160 2161 2162
		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
2163
	}
2164 2165 2166

	c->page = NULL;
	c->freelist = NULL;
C
Christoph Lameter 已提交
2167 2168
}

2169 2170 2171
/*
 * Unfreeze all the cpu partial slabs.
 *
2172 2173 2174
 * This function must be called with interrupts disabled
 * for the cpu using c (or some other guarantee must be there
 * to guarantee no concurrent accesses).
2175
 */
2176 2177
static void unfreeze_partials(struct kmem_cache *s,
		struct kmem_cache_cpu *c)
2178
{
2179
#ifdef CONFIG_SLUB_CPU_PARTIAL
2180
	struct kmem_cache_node *n = NULL, *n2 = NULL;
2181
	struct page *page, *discard_page = NULL;
2182 2183 2184 2185 2186 2187

	while ((page = c->partial)) {
		struct page new;
		struct page old;

		c->partial = page->next;
2188 2189 2190 2191 2192 2193 2194 2195 2196

		n2 = get_node(s, page_to_nid(page));
		if (n != n2) {
			if (n)
				spin_unlock(&n->list_lock);

			n = n2;
			spin_lock(&n->list_lock);
		}
2197 2198 2199 2200 2201

		do {

			old.freelist = page->freelist;
			old.counters = page->counters;
2202
			VM_BUG_ON(!old.frozen);
2203 2204 2205 2206 2207 2208

			new.counters = old.counters;
			new.freelist = old.freelist;

			new.frozen = 0;

2209
		} while (!__cmpxchg_double_slab(s, page,
2210 2211 2212 2213
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"));

2214
		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
2215 2216
			page->next = discard_page;
			discard_page = page;
2217 2218 2219
		} else {
			add_partial(n, page, DEACTIVATE_TO_TAIL);
			stat(s, FREE_ADD_PARTIAL);
2220 2221 2222 2223 2224
		}
	}

	if (n)
		spin_unlock(&n->list_lock);
2225 2226 2227 2228 2229 2230 2231 2232 2233

	while (discard_page) {
		page = discard_page;
		discard_page = discard_page->next;

		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
	}
2234
#endif
2235 2236 2237 2238
}

/*
 * Put a page that was just frozen (in __slab_free) into a partial page
2239
 * slot if available.
2240 2241 2242 2243
 *
 * If we did not find a slot then simply move all the partials to the
 * per node partial list.
 */
2244
static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
2245
{
2246
#ifdef CONFIG_SLUB_CPU_PARTIAL
2247 2248 2249 2250
	struct page *oldpage;
	int pages;
	int pobjects;

2251
	preempt_disable();
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266
	do {
		pages = 0;
		pobjects = 0;
		oldpage = this_cpu_read(s->cpu_slab->partial);

		if (oldpage) {
			pobjects = oldpage->pobjects;
			pages = oldpage->pages;
			if (drain && pobjects > s->cpu_partial) {
				unsigned long flags;
				/*
				 * partial array is full. Move the existing
				 * set to the per node partial list.
				 */
				local_irq_save(flags);
2267
				unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
2268
				local_irq_restore(flags);
2269
				oldpage = NULL;
2270 2271
				pobjects = 0;
				pages = 0;
2272
				stat(s, CPU_PARTIAL_DRAIN);
2273 2274 2275 2276 2277 2278 2279 2280 2281 2282
			}
		}

		pages++;
		pobjects += page->objects - page->inuse;

		page->pages = pages;
		page->pobjects = pobjects;
		page->next = oldpage;

2283 2284
	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
								!= oldpage);
2285 2286 2287 2288 2289 2290 2291 2292
	if (unlikely(!s->cpu_partial)) {
		unsigned long flags;

		local_irq_save(flags);
		unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
		local_irq_restore(flags);
	}
	preempt_enable();
2293
#endif
2294 2295
}

2296
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2297
{
2298
	stat(s, CPUSLAB_FLUSH);
2299
	deactivate_slab(s, c->page, c->freelist, c);
2300 2301

	c->tid = next_tid(c->tid);
C
Christoph Lameter 已提交
2302 2303 2304 2305
}

/*
 * Flush cpu slab.
C
Christoph Lameter 已提交
2306
 *
C
Christoph Lameter 已提交
2307 2308
 * Called from IPI handler with interrupts disabled.
 */
2309
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
C
Christoph Lameter 已提交
2310
{
2311
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
C
Christoph Lameter 已提交
2312

2313 2314
	if (c->page)
		flush_slab(s, c);
2315

2316
	unfreeze_partials(s, c);
C
Christoph Lameter 已提交
2317 2318 2319 2320 2321 2322
}

static void flush_cpu_slab(void *d)
{
	struct kmem_cache *s = d;

2323
	__flush_cpu_slab(s, smp_processor_id());
C
Christoph Lameter 已提交
2324 2325
}

2326 2327 2328 2329 2330
static bool has_cpu_slab(int cpu, void *info)
{
	struct kmem_cache *s = info;
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

2331
	return c->page || slub_percpu_partial(c);
2332 2333
}

C
Christoph Lameter 已提交
2334 2335
static void flush_all(struct kmem_cache *s)
{
2336
	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
C
Christoph Lameter 已提交
2337 2338
}

2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
/*
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
 */
static int slub_cpu_dead(unsigned int cpu)
{
	struct kmem_cache *s;
	unsigned long flags;

	mutex_lock(&slab_mutex);
	list_for_each_entry(s, &slab_caches, list) {
		local_irq_save(flags);
		__flush_cpu_slab(s, cpu);
		local_irq_restore(flags);
	}
	mutex_unlock(&slab_mutex);
	return 0;
}

2358 2359 2360 2361
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
2362
static inline int node_match(struct page *page, int node)
2363 2364
{
#ifdef CONFIG_NUMA
2365
	if (node != NUMA_NO_NODE && page_to_nid(page) != node)
2366 2367 2368 2369 2370
		return 0;
#endif
	return 1;
}

2371
#ifdef CONFIG_SLUB_DEBUG
2372 2373 2374 2375 2376
static int count_free(struct page *page)
{
	return page->objects - page->inuse;
}

2377 2378 2379 2380 2381 2382 2383
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->total_objects);
}
#endif /* CONFIG_SLUB_DEBUG */

#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396
static unsigned long count_partial(struct kmem_cache_node *n,
					int (*get_count)(struct page *))
{
	unsigned long flags;
	unsigned long x = 0;
	struct page *page;

	spin_lock_irqsave(&n->list_lock, flags);
	list_for_each_entry(page, &n->partial, lru)
		x += get_count(page);
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}
2397
#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
2398

2399 2400 2401
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
2402 2403 2404
#ifdef CONFIG_SLUB_DEBUG
	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);
2405
	int node;
2406
	struct kmem_cache_node *n;
2407

2408 2409 2410
	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
		return;

2411 2412
	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
		nid, gfpflags, &gfpflags);
2413
	pr_warn("  cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
2414 2415
		s->name, s->object_size, s->size, oo_order(s->oo),
		oo_order(s->min));
2416

2417
	if (oo_order(s->min) > get_order(s->object_size))
2418 2419
		pr_warn("  %s debugging increased min order, use slub_debug=O to disable.\n",
			s->name);
2420

2421
	for_each_kmem_cache_node(s, node, n) {
2422 2423 2424 2425
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

2426 2427 2428
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
2429

2430
		pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
2431 2432
			node, nr_slabs, nr_objs, nr_free);
	}
2433
#endif
2434 2435
}

2436 2437 2438
static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
			int node, struct kmem_cache_cpu **pc)
{
2439
	void *freelist;
2440 2441
	struct kmem_cache_cpu *c = *pc;
	struct page *page;
2442

2443 2444
	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));

2445
	freelist = get_partial(s, flags, node, c);
2446

2447 2448 2449 2450
	if (freelist)
		return freelist;

	page = new_slab(s, flags, node);
2451
	if (page) {
2452
		c = raw_cpu_ptr(s->cpu_slab);
2453 2454 2455 2456 2457 2458 2459
		if (c->page)
			flush_slab(s, c);

		/*
		 * No other reference to the page yet so we can
		 * muck around with it freely without cmpxchg
		 */
2460
		freelist = page->freelist;
2461 2462 2463 2464 2465 2466
		page->freelist = NULL;

		stat(s, ALLOC_SLAB);
		c->page = page;
		*pc = c;
	} else
2467
		freelist = NULL;
2468

2469
	return freelist;
2470 2471
}

2472 2473 2474 2475 2476 2477 2478 2479
static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
{
	if (unlikely(PageSlabPfmemalloc(page)))
		return gfp_pfmemalloc_allowed(gfpflags);

	return true;
}

2480
/*
2481 2482
 * Check the page->freelist of a page and either transfer the freelist to the
 * per cpu freelist or deactivate the page.
2483 2484 2485 2486
 *
 * The page is still frozen if the return value is not NULL.
 *
 * If this function returns NULL then the page has been unfrozen.
2487 2488
 *
 * This function must be called with interrupt disabled.
2489 2490 2491 2492 2493 2494 2495 2496 2497 2498
 */
static inline void *get_freelist(struct kmem_cache *s, struct page *page)
{
	struct page new;
	unsigned long counters;
	void *freelist;

	do {
		freelist = page->freelist;
		counters = page->counters;
2499

2500
		new.counters = counters;
2501
		VM_BUG_ON(!new.frozen);
2502 2503 2504 2505

		new.inuse = page->objects;
		new.frozen = freelist != NULL;

2506
	} while (!__cmpxchg_double_slab(s, page,
2507 2508 2509 2510 2511 2512 2513
		freelist, counters,
		NULL, new.counters,
		"get_freelist"));

	return freelist;
}

C
Christoph Lameter 已提交
2514
/*
2515 2516 2517 2518 2519 2520
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
C
Christoph Lameter 已提交
2521
 *
2522 2523 2524
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
C
Christoph Lameter 已提交
2525
 *
2526
 * And if we were unable to get a new slab from the partial slab lists then
C
Christoph Lameter 已提交
2527 2528
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
2529 2530 2531
 *
 * Version of __slab_alloc to use when we know that interrupts are
 * already disabled (which is the case for bulk allocation).
C
Christoph Lameter 已提交
2532
 */
2533
static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2534
			  unsigned long addr, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2535
{
2536
	void *freelist;
2537
	struct page *page;
C
Christoph Lameter 已提交
2538

2539 2540
	page = c->page;
	if (!page)
C
Christoph Lameter 已提交
2541
		goto new_slab;
2542
redo:
2543

2544
	if (unlikely(!node_match(page, node))) {
2545 2546 2547 2548 2549 2550 2551
		int searchnode = node;

		if (node != NUMA_NO_NODE && !node_present_pages(node))
			searchnode = node_to_mem_node(node);

		if (unlikely(!node_match(page, searchnode))) {
			stat(s, ALLOC_NODE_MISMATCH);
2552
			deactivate_slab(s, page, c->freelist, c);
2553 2554
			goto new_slab;
		}
2555
	}
C
Christoph Lameter 已提交
2556

2557 2558 2559 2560 2561 2562
	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
	if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2563
		deactivate_slab(s, page, c->freelist, c);
2564 2565 2566
		goto new_slab;
	}

2567
	/* must check again c->freelist in case of cpu migration or IRQ */
2568 2569
	freelist = c->freelist;
	if (freelist)
2570
		goto load_freelist;
2571

2572
	freelist = get_freelist(s, page);
C
Christoph Lameter 已提交
2573

2574
	if (!freelist) {
2575 2576
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
2577
		goto new_slab;
2578
	}
C
Christoph Lameter 已提交
2579

2580
	stat(s, ALLOC_REFILL);
C
Christoph Lameter 已提交
2581

2582
load_freelist:
2583 2584 2585 2586 2587
	/*
	 * freelist is pointing to the list of objects to be used.
	 * page is pointing to the page from which the objects are obtained.
	 * That page must be frozen for per cpu allocations to work.
	 */
2588
	VM_BUG_ON(!c->page->frozen);
2589
	c->freelist = get_freepointer(s, freelist);
2590
	c->tid = next_tid(c->tid);
2591
	return freelist;
C
Christoph Lameter 已提交
2592 2593

new_slab:
2594

2595 2596 2597
	if (slub_percpu_partial(c)) {
		page = c->page = slub_percpu_partial(c);
		slub_set_percpu_partial(c, page);
2598 2599
		stat(s, CPU_PARTIAL_ALLOC);
		goto redo;
C
Christoph Lameter 已提交
2600 2601
	}

2602
	freelist = new_slab_objects(s, gfpflags, node, &c);
2603

2604
	if (unlikely(!freelist)) {
2605
		slab_out_of_memory(s, gfpflags, node);
2606
		return NULL;
C
Christoph Lameter 已提交
2607
	}
2608

2609
	page = c->page;
2610
	if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
2611
		goto load_freelist;
2612

2613
	/* Only entered in the debug case */
2614 2615
	if (kmem_cache_debug(s) &&
			!alloc_debug_processing(s, page, freelist, addr))
2616
		goto new_slab;	/* Slab failed checks. Next slab needed */
2617

2618
	deactivate_slab(s, page, get_freepointer(s, freelist), c);
2619
	return freelist;
2620 2621
}

2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646
/*
 * Another one that disabled interrupt and compensates for possible
 * cpu changes by refetching the per cpu area pointer.
 */
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *p;
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPT
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif

	p = ___slab_alloc(s, gfpflags, node, addr, c);
	local_irq_restore(flags);
	return p;
}

2647 2648 2649 2650 2651 2652 2653 2654 2655 2656
/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
2657
static __always_inline void *slab_alloc_node(struct kmem_cache *s,
2658
		gfp_t gfpflags, int node, unsigned long addr)
2659
{
2660
	void *object;
2661
	struct kmem_cache_cpu *c;
2662
	struct page *page;
2663
	unsigned long tid;
2664

2665 2666
	s = slab_pre_alloc_hook(s, gfpflags);
	if (!s)
A
Akinobu Mita 已提交
2667
		return NULL;
2668 2669 2670 2671 2672 2673
redo:
	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
2674
	 *
2675 2676 2677
	 * We should guarantee that tid and kmem_cache are retrieved on
	 * the same cpu. It could be different if CONFIG_PREEMPT so we need
	 * to check if it is matched or not.
2678
	 */
2679 2680 2681
	do {
		tid = this_cpu_read(s->cpu_slab->tid);
		c = raw_cpu_ptr(s->cpu_slab);
2682 2683
	} while (IS_ENABLED(CONFIG_PREEMPT) &&
		 unlikely(tid != READ_ONCE(c->tid)));
2684 2685 2686 2687 2688 2689 2690 2691 2692 2693

	/*
	 * Irqless object alloc/free algorithm used here depends on sequence
	 * of fetching cpu_slab's data. tid should be fetched before anything
	 * on c to guarantee that object and page associated with previous tid
	 * won't be used with current tid. If we fetch tid first, object and
	 * page could be one associated with next tid and our alloc/free
	 * request will be failed. In this case, we will retry. So, no problem.
	 */
	barrier();
2694 2695 2696 2697 2698 2699 2700 2701

	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */

2702
	object = c->freelist;
2703
	page = c->page;
2704
	if (unlikely(!object || !node_match(page, node))) {
2705
		object = __slab_alloc(s, gfpflags, node, addr, c);
2706 2707
		stat(s, ALLOC_SLOWPATH);
	} else {
2708 2709
		void *next_object = get_freepointer_safe(s, object);

2710
		/*
L
Lucas De Marchi 已提交
2711
		 * The cmpxchg will only match if there was no additional
2712 2713
		 * operation and if we are on the right processor.
		 *
2714 2715
		 * The cmpxchg does the following atomically (without lock
		 * semantics!)
2716 2717 2718 2719
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
2720 2721 2722
		 * Since this is without lock semantics the protection is only
		 * against code executing on this cpu *not* from access by
		 * other cpus.
2723
		 */
2724
		if (unlikely(!this_cpu_cmpxchg_double(
2725 2726
				s->cpu_slab->freelist, s->cpu_slab->tid,
				object, tid,
2727
				next_object, next_tid(tid)))) {
2728 2729 2730 2731

			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
2732
		prefetch_freepointer(s, next_object);
2733
		stat(s, ALLOC_FASTPATH);
2734
	}
2735

2736
	if (unlikely(gfpflags & __GFP_ZERO) && object)
2737
		memset(object, 0, s->object_size);
2738

2739
	slab_post_alloc_hook(s, gfpflags, 1, &object);
2740

2741
	return object;
C
Christoph Lameter 已提交
2742 2743
}

2744 2745 2746 2747 2748 2749
static __always_inline void *slab_alloc(struct kmem_cache *s,
		gfp_t gfpflags, unsigned long addr)
{
	return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
}

C
Christoph Lameter 已提交
2750 2751
void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
2752
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);
2753

2754 2755
	trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
				s->size, gfpflags);
2756 2757

	return ret;
C
Christoph Lameter 已提交
2758 2759 2760
}
EXPORT_SYMBOL(kmem_cache_alloc);

2761
#ifdef CONFIG_TRACING
2762 2763
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
2764
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);
2765
	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2766
	ret = kasan_kmalloc(s, ret, size, gfpflags);
2767 2768 2769
	return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);
2770 2771
#endif

C
Christoph Lameter 已提交
2772 2773 2774
#ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
2775
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
2776

2777
	trace_kmem_cache_alloc_node(_RET_IP_, ret,
2778
				    s->object_size, s->size, gfpflags, node);
2779 2780

	return ret;
C
Christoph Lameter 已提交
2781 2782 2783
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

2784
#ifdef CONFIG_TRACING
2785
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
2786
				    gfp_t gfpflags,
2787
				    int node, size_t size)
2788
{
2789
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
2790 2791 2792

	trace_kmalloc_node(_RET_IP_, ret,
			   size, s->size, gfpflags, node);
2793

2794
	ret = kasan_kmalloc(s, ret, size, gfpflags);
2795
	return ret;
2796
}
2797
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
2798
#endif
2799
#endif
2800

C
Christoph Lameter 已提交
2801
/*
2802
 * Slow path handling. This may still be called frequently since objects
2803
 * have a longer lifetime than the cpu slabs in most processing loads.
C
Christoph Lameter 已提交
2804
 *
2805 2806 2807
 * So we still attempt to reduce cache line usage. Just take the slab
 * lock and free the item. If there is no additional partial page
 * handling required then we can return immediately.
C
Christoph Lameter 已提交
2808
 */
2809
static void __slab_free(struct kmem_cache *s, struct page *page,
2810 2811 2812
			void *head, void *tail, int cnt,
			unsigned long addr)

C
Christoph Lameter 已提交
2813 2814
{
	void *prior;
2815 2816 2817 2818
	int was_frozen;
	struct page new;
	unsigned long counters;
	struct kmem_cache_node *n = NULL;
2819
	unsigned long uninitialized_var(flags);
C
Christoph Lameter 已提交
2820

2821
	stat(s, FREE_SLOWPATH);
C
Christoph Lameter 已提交
2822

2823
	if (kmem_cache_debug(s) &&
2824
	    !free_debug_processing(s, page, head, tail, cnt, addr))
2825
		return;
C
Christoph Lameter 已提交
2826

2827
	do {
2828 2829 2830 2831
		if (unlikely(n)) {
			spin_unlock_irqrestore(&n->list_lock, flags);
			n = NULL;
		}
2832 2833
		prior = page->freelist;
		counters = page->counters;
2834
		set_freepointer(s, tail, prior);
2835 2836
		new.counters = counters;
		was_frozen = new.frozen;
2837
		new.inuse -= cnt;
2838
		if ((!new.inuse || !prior) && !was_frozen) {
2839

2840
			if (kmem_cache_has_cpu_partial(s) && !prior) {
2841 2842

				/*
2843 2844 2845 2846
				 * Slab was on no list before and will be
				 * partially empty
				 * We can defer the list move and instead
				 * freeze it.
2847 2848 2849
				 */
				new.frozen = 1;

2850
			} else { /* Needs to be taken off a list */
2851

2852
				n = get_node(s, page_to_nid(page));
2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
				/*
				 * Speculatively acquire the list_lock.
				 * If the cmpxchg does not succeed then we may
				 * drop the list_lock without any processing.
				 *
				 * Otherwise the list_lock will synchronize with
				 * other processors updating the list of slabs.
				 */
				spin_lock_irqsave(&n->list_lock, flags);

			}
2864
		}
C
Christoph Lameter 已提交
2865

2866 2867
	} while (!cmpxchg_double_slab(s, page,
		prior, counters,
2868
		head, new.counters,
2869
		"__slab_free"));
C
Christoph Lameter 已提交
2870

2871
	if (likely(!n)) {
2872 2873 2874 2875 2876

		/*
		 * If we just froze the page then put it onto the
		 * per cpu partial list.
		 */
2877
		if (new.frozen && !was_frozen) {
2878
			put_cpu_partial(s, page, 1);
2879 2880
			stat(s, CPU_PARTIAL_FREE);
		}
2881
		/*
2882 2883 2884
		 * The list lock was not taken therefore no list
		 * activity can be necessary.
		 */
2885 2886 2887 2888
		if (was_frozen)
			stat(s, FREE_FROZEN);
		return;
	}
C
Christoph Lameter 已提交
2889

2890
	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
2891 2892
		goto slab_empty;

C
Christoph Lameter 已提交
2893
	/*
2894 2895
	 * Objects left in the slab. If it was not on the partial list before
	 * then add it.
C
Christoph Lameter 已提交
2896
	 */
2897 2898
	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
		if (kmem_cache_debug(s))
2899
			remove_full(s, n, page);
2900 2901
		add_partial(n, page, DEACTIVATE_TO_TAIL);
		stat(s, FREE_ADD_PARTIAL);
2902
	}
2903
	spin_unlock_irqrestore(&n->list_lock, flags);
C
Christoph Lameter 已提交
2904 2905 2906
	return;

slab_empty:
2907
	if (prior) {
C
Christoph Lameter 已提交
2908
		/*
2909
		 * Slab on the partial list.
C
Christoph Lameter 已提交
2910
		 */
2911
		remove_partial(n, page);
2912
		stat(s, FREE_REMOVE_PARTIAL);
2913
	} else {
2914
		/* Slab must be on the full list */
2915 2916
		remove_full(s, n, page);
	}
2917

2918
	spin_unlock_irqrestore(&n->list_lock, flags);
2919
	stat(s, FREE_SLAB);
C
Christoph Lameter 已提交
2920 2921 2922
	discard_slab(s, page);
}

2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
2933 2934 2935 2936
 *
 * Bulk free of a freelist with several objects (all pointing to the
 * same page) possible by specifying head and tail ptr, plus objects
 * count (cnt). Bulk free indicated by tail pointer being set.
2937
 */
2938 2939 2940
static __always_inline void do_slab_free(struct kmem_cache *s,
				struct page *page, void *head, void *tail,
				int cnt, unsigned long addr)
2941
{
2942
	void *tail_obj = tail ? : head;
2943
	struct kmem_cache_cpu *c;
2944 2945 2946 2947 2948 2949
	unsigned long tid;
redo:
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
2950
	 * during the cmpxchg then the free will succeed.
2951
	 */
2952 2953 2954
	do {
		tid = this_cpu_read(s->cpu_slab->tid);
		c = raw_cpu_ptr(s->cpu_slab);
2955 2956
	} while (IS_ENABLED(CONFIG_PREEMPT) &&
		 unlikely(tid != READ_ONCE(c->tid)));
2957

2958 2959
	/* Same with comment on barrier() in slab_alloc_node() */
	barrier();
2960

2961
	if (likely(page == c->page)) {
2962
		set_freepointer(s, tail_obj, c->freelist);
2963

2964
		if (unlikely(!this_cpu_cmpxchg_double(
2965 2966
				s->cpu_slab->freelist, s->cpu_slab->tid,
				c->freelist, tid,
2967
				head, next_tid(tid)))) {
2968 2969 2970 2971

			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
2972
		stat(s, FREE_FASTPATH);
2973
	} else
2974
		__slab_free(s, page, head, tail_obj, cnt, addr);
2975 2976 2977

}

2978 2979 2980 2981 2982
static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
				      void *head, void *tail, int cnt,
				      unsigned long addr)
{
	/*
2983 2984
	 * With KASAN enabled slab_free_freelist_hook modifies the freelist
	 * to remove objects, whose reuse must be delayed.
2985
	 */
2986 2987
	if (slab_free_freelist_hook(s, &head, &tail))
		do_slab_free(s, page, head, tail, cnt, addr);
2988 2989
}

2990
#ifdef CONFIG_KASAN_GENERIC
2991 2992 2993 2994 2995 2996
void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
{
	do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
}
#endif

C
Christoph Lameter 已提交
2997 2998
void kmem_cache_free(struct kmem_cache *s, void *x)
{
2999 3000
	s = cache_from_obj(s, x);
	if (!s)
3001
		return;
3002
	slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
3003
	trace_kmem_cache_free(_RET_IP_, x);
C
Christoph Lameter 已提交
3004 3005 3006
}
EXPORT_SYMBOL(kmem_cache_free);

3007
struct detached_freelist {
3008
	struct page *page;
3009 3010 3011
	void *tail;
	void *freelist;
	int cnt;
3012
	struct kmem_cache *s;
3013
};
3014

3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026
/*
 * This function progressively scans the array with free objects (with
 * a limited look ahead) and extract objects belonging to the same
 * page.  It builds a detached freelist directly within the given
 * page/objects.  This can happen without any need for
 * synchronization, because the objects are owned by running process.
 * The freelist is build up as a single linked list in the objects.
 * The idea is, that this detached freelist can then be bulk
 * transferred to the real freelist(s), but only requiring a single
 * synchronization primitive.  Look ahead in the array is limited due
 * to performance reasons.
 */
3027 3028 3029
static inline
int build_detached_freelist(struct kmem_cache *s, size_t size,
			    void **p, struct detached_freelist *df)
3030 3031 3032 3033
{
	size_t first_skipped_index = 0;
	int lookahead = 3;
	void *object;
3034
	struct page *page;
3035

3036 3037
	/* Always re-init detached_freelist */
	df->page = NULL;
3038

3039 3040
	do {
		object = p[--size];
3041
		/* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
3042
	} while (!object && size);
3043

3044 3045
	if (!object)
		return 0;
3046

3047 3048 3049 3050 3051 3052
	page = virt_to_head_page(object);
	if (!s) {
		/* Handle kalloc'ed objects */
		if (unlikely(!PageSlab(page))) {
			BUG_ON(!PageCompound(page));
			kfree_hook(object);
3053
			__free_pages(page, compound_order(page));
3054 3055 3056 3057 3058 3059 3060 3061
			p[size] = NULL; /* mark object processed */
			return size;
		}
		/* Derive kmem_cache from object */
		df->s = page->slab_cache;
	} else {
		df->s = cache_from_obj(s, object); /* Support for memcg */
	}
3062

3063
	/* Start new detached freelist */
3064
	df->page = page;
3065
	set_freepointer(df->s, object, NULL);
3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078
	df->tail = object;
	df->freelist = object;
	p[size] = NULL; /* mark object processed */
	df->cnt = 1;

	while (size) {
		object = p[--size];
		if (!object)
			continue; /* Skip processed objects */

		/* df->page is always set at this point */
		if (df->page == virt_to_head_page(object)) {
			/* Opportunity build freelist */
3079
			set_freepointer(df->s, object, df->freelist);
3080 3081 3082 3083 3084
			df->freelist = object;
			df->cnt++;
			p[size] = NULL; /* mark object processed */

			continue;
3085
		}
3086 3087 3088 3089 3090 3091 3092

		/* Limit look ahead search */
		if (!--lookahead)
			break;

		if (!first_skipped_index)
			first_skipped_index = size + 1;
3093
	}
3094 3095 3096 3097 3098

	return first_skipped_index;
}

/* Note that interrupts must be enabled when calling this function. */
3099
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
3100 3101 3102 3103 3104 3105 3106 3107
{
	if (WARN_ON(!size))
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
3108
		if (!df.page)
3109 3110
			continue;

3111
		slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
3112
	} while (likely(size));
3113 3114 3115
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

3116
/* Note that interrupts must be enabled when calling this function. */
3117 3118
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
			  void **p)
3119
{
3120 3121 3122
	struct kmem_cache_cpu *c;
	int i;

3123 3124 3125 3126
	/* memcg and kmem_cache debug support */
	s = slab_pre_alloc_hook(s, flags);
	if (unlikely(!s))
		return false;
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137
	/*
	 * Drain objects in the per cpu slab, while disabling local
	 * IRQs, which protects against PREEMPT and interrupts
	 * handlers invoking normal fastpath.
	 */
	local_irq_disable();
	c = this_cpu_ptr(s->cpu_slab);

	for (i = 0; i < size; i++) {
		void *object = c->freelist;

3138 3139 3140 3141 3142
		if (unlikely(!object)) {
			/*
			 * Invoking slow path likely have side-effect
			 * of re-populating per CPU c->freelist
			 */
3143
			p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
3144
					    _RET_IP_, c);
3145 3146 3147
			if (unlikely(!p[i]))
				goto error;

3148 3149 3150
			c = this_cpu_ptr(s->cpu_slab);
			continue; /* goto for-loop */
		}
3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164
		c->freelist = get_freepointer(s, object);
		p[i] = object;
	}
	c->tid = next_tid(c->tid);
	local_irq_enable();

	/* Clear memory outside IRQ disabled fastpath loop */
	if (unlikely(flags & __GFP_ZERO)) {
		int j;

		for (j = 0; j < i; j++)
			memset(p[j], 0, s->object_size);
	}

3165 3166
	/* memcg and kmem_cache debug support */
	slab_post_alloc_hook(s, flags, size, p);
3167
	return i;
3168 3169
error:
	local_irq_enable();
3170 3171
	slab_post_alloc_hook(s, flags, i, p);
	__kmem_cache_free_bulk(s, i, p);
3172
	return 0;
3173 3174 3175 3176
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk);


C
Christoph Lameter 已提交
3177
/*
C
Christoph Lameter 已提交
3178 3179 3180 3181
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
C
Christoph Lameter 已提交
3182 3183 3184 3185
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
C
Christoph Lameter 已提交
3186
 * must be moved on and off the partial lists and is therefore a factor in
C
Christoph Lameter 已提交
3187 3188 3189 3190 3191 3192 3193 3194 3195
 * locking overhead.
 */

/*
 * Mininum / Maximum order of slab pages. This influences locking overhead
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
3196 3197 3198
static unsigned int slub_min_order;
static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
static unsigned int slub_min_objects;
C
Christoph Lameter 已提交
3199 3200 3201 3202

/*
 * Calculate the order of allocation given an slab object size.
 *
C
Christoph Lameter 已提交
3203 3204 3205 3206
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
3207
 * unused space left. We go to a higher order if more than 1/16th of the slab
C
Christoph Lameter 已提交
3208 3209 3210 3211 3212 3213
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
C
Christoph Lameter 已提交
3214
 *
C
Christoph Lameter 已提交
3215 3216 3217 3218
 * slub_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slub_max_order then
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
C
Christoph Lameter 已提交
3219
 *
C
Christoph Lameter 已提交
3220 3221 3222 3223
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
 * requested a higher mininum order then we start with that one instead of
 * the smallest order which will fit the object.
C
Christoph Lameter 已提交
3224
 */
3225 3226
static inline unsigned int slab_order(unsigned int size,
		unsigned int min_objects, unsigned int max_order,
3227
		unsigned int fract_leftover)
C
Christoph Lameter 已提交
3228
{
3229 3230
	unsigned int min_order = slub_min_order;
	unsigned int order;
C
Christoph Lameter 已提交
3231

3232
	if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
3233
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
3234

3235
	for (order = max(min_order, (unsigned int)get_order(min_objects * size));
3236
			order <= max_order; order++) {
C
Christoph Lameter 已提交
3237

3238 3239
		unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
		unsigned int rem;
C
Christoph Lameter 已提交
3240

3241
		rem = slab_size % size;
C
Christoph Lameter 已提交
3242

3243
		if (rem <= slab_size / fract_leftover)
C
Christoph Lameter 已提交
3244 3245
			break;
	}
C
Christoph Lameter 已提交
3246

C
Christoph Lameter 已提交
3247 3248 3249
	return order;
}

3250
static inline int calculate_order(unsigned int size)
3251
{
3252 3253 3254
	unsigned int order;
	unsigned int min_objects;
	unsigned int max_objects;
3255 3256 3257 3258 3259 3260

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 *
3261
	 * First we increase the acceptable waste in a slab. Then
3262 3263 3264
	 * we reduce the minimum objects required in a slab.
	 */
	min_objects = slub_min_objects;
3265 3266
	if (!min_objects)
		min_objects = 4 * (fls(nr_cpu_ids) + 1);
3267
	max_objects = order_objects(slub_max_order, size);
3268 3269
	min_objects = min(min_objects, max_objects);

3270
	while (min_objects > 1) {
3271 3272
		unsigned int fraction;

3273
		fraction = 16;
3274 3275
		while (fraction >= 4) {
			order = slab_order(size, min_objects,
3276
					slub_max_order, fraction);
3277 3278 3279 3280
			if (order <= slub_max_order)
				return order;
			fraction /= 2;
		}
3281
		min_objects--;
3282 3283 3284 3285 3286 3287
	}

	/*
	 * We were unable to place multiple objects in a slab. Now
	 * lets see if we can place a single object there.
	 */
3288
	order = slab_order(size, 1, slub_max_order, 1);
3289 3290 3291 3292 3293 3294
	if (order <= slub_max_order)
		return order;

	/*
	 * Doh this slab cannot be placed using slub_max_order.
	 */
3295
	order = slab_order(size, 1, MAX_ORDER, 1);
D
David Rientjes 已提交
3296
	if (order < MAX_ORDER)
3297 3298 3299 3300
		return order;
	return -ENOSYS;
}

3301
static void
3302
init_kmem_cache_node(struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3303 3304 3305 3306
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
3307
#ifdef CONFIG_SLUB_DEBUG
3308
	atomic_long_set(&n->nr_slabs, 0);
3309
	atomic_long_set(&n->total_objects, 0);
3310
	INIT_LIST_HEAD(&n->full);
3311
#endif
C
Christoph Lameter 已提交
3312 3313
}

3314
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
3315
{
3316
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
3317
			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
3318

3319
	/*
3320 3321
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
3322
	 */
3323 3324
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));
3325 3326 3327 3328 3329

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
3330

3331
	return 1;
3332 3333
}

3334 3335
static struct kmem_cache *kmem_cache_node;

C
Christoph Lameter 已提交
3336 3337 3338 3339 3340
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
3341 3342
 * Note that this function only works on the kmem_cache_node
 * when allocating for the kmem_cache_node. This is used for bootstrapping
3343
 * memory on a fresh node that has no slab structures yet.
C
Christoph Lameter 已提交
3344
 */
3345
static void early_kmem_cache_node_alloc(int node)
C
Christoph Lameter 已提交
3346 3347 3348 3349
{
	struct page *page;
	struct kmem_cache_node *n;

3350
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
C
Christoph Lameter 已提交
3351

3352
	page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
C
Christoph Lameter 已提交
3353 3354

	BUG_ON(!page);
3355
	if (page_to_nid(page) != node) {
3356 3357
		pr_err("SLUB: Unable to allocate memory from node %d\n", node);
		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
3358 3359
	}

C
Christoph Lameter 已提交
3360 3361
	n = page->freelist;
	BUG_ON(!n);
3362
#ifdef CONFIG_SLUB_DEBUG
3363
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
3364
	init_tracking(kmem_cache_node, n);
3365
#endif
3366
	n = kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
3367
		      GFP_KERNEL);
3368 3369 3370 3371
	page->freelist = get_freepointer(kmem_cache_node, n);
	page->inuse = 1;
	page->frozen = 0;
	kmem_cache_node->node[node] = n;
3372
	init_kmem_cache_node(n);
3373
	inc_slabs_node(kmem_cache_node, node, page->objects);
C
Christoph Lameter 已提交
3374

3375
	/*
3376 3377
	 * No locks need to be taken here as it has just been
	 * initialized and there is no concurrent access.
3378
	 */
3379
	__add_partial(n, page, DEACTIVATE_TO_HEAD);
C
Christoph Lameter 已提交
3380 3381 3382 3383 3384
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;
3385
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
3386

3387
	for_each_kmem_cache_node(s, node, n) {
C
Christoph Lameter 已提交
3388
		s->node[node] = NULL;
3389
		kmem_cache_free(kmem_cache_node, n);
C
Christoph Lameter 已提交
3390 3391 3392
	}
}

3393 3394
void __kmem_cache_release(struct kmem_cache *s)
{
3395
	cache_random_seq_destroy(s);
3396 3397 3398 3399
	free_percpu(s->cpu_slab);
	free_kmem_cache_nodes(s);
}

3400
static int init_kmem_cache_nodes(struct kmem_cache *s)
C
Christoph Lameter 已提交
3401 3402 3403
{
	int node;

3404
	for_each_node_state(node, N_NORMAL_MEMORY) {
C
Christoph Lameter 已提交
3405 3406
		struct kmem_cache_node *n;

3407
		if (slab_state == DOWN) {
3408
			early_kmem_cache_node_alloc(node);
3409 3410
			continue;
		}
3411
		n = kmem_cache_alloc_node(kmem_cache_node,
3412
						GFP_KERNEL, node);
C
Christoph Lameter 已提交
3413

3414 3415 3416
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
C
Christoph Lameter 已提交
3417
		}
3418

3419
		init_kmem_cache_node(n);
3420
		s->node[node] = n;
C
Christoph Lameter 已提交
3421 3422 3423 3424
	}
	return 1;
}

3425
static void set_min_partial(struct kmem_cache *s, unsigned long min)
3426 3427 3428 3429 3430 3431 3432 3433
{
	if (min < MIN_PARTIAL)
		min = MIN_PARTIAL;
	else if (min > MAX_PARTIAL)
		min = MAX_PARTIAL;
	s->min_partial = min;
}

3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466
static void set_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
	 * This setting also determines
	 *
	 * A) The number of objects from per cpu partial slabs dumped to the
	 *    per node list when we reach the limit.
	 * B) The number of objects in cpu partial slabs to extract from the
	 *    per node list when we run out of per cpu objects. We only fetch
	 *    50% to keep some capacity around for frees.
	 */
	if (!kmem_cache_has_cpu_partial(s))
		s->cpu_partial = 0;
	else if (s->size >= PAGE_SIZE)
		s->cpu_partial = 2;
	else if (s->size >= 1024)
		s->cpu_partial = 6;
	else if (s->size >= 256)
		s->cpu_partial = 13;
	else
		s->cpu_partial = 30;
#endif
}

C
Christoph Lameter 已提交
3467 3468 3469 3470
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
3471
static int calculate_sizes(struct kmem_cache *s, int forced_order)
C
Christoph Lameter 已提交
3472
{
3473
	slab_flags_t flags = s->flags;
3474
	unsigned int size = s->object_size;
3475
	unsigned int order;
C
Christoph Lameter 已提交
3476

3477 3478 3479 3480 3481 3482 3483 3484
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
3485 3486 3487 3488 3489
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
3490
	if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
3491
			!s->ctor)
C
Christoph Lameter 已提交
3492 3493 3494 3495 3496 3497
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
C
Christoph Lameter 已提交
3498
	 * If we are Redzoning then check if there is some space between the
C
Christoph Lameter 已提交
3499
	 * end of the object and the free pointer. If not then add an
C
Christoph Lameter 已提交
3500
	 * additional word to have some bytes to store Redzone information.
C
Christoph Lameter 已提交
3501
	 */
3502
	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
C
Christoph Lameter 已提交
3503
		size += sizeof(void *);
3504
#endif
C
Christoph Lameter 已提交
3505 3506

	/*
C
Christoph Lameter 已提交
3507 3508
	 * With that we have determined the number of bytes in actual use
	 * by the object. This is the potential offset to the free pointer.
C
Christoph Lameter 已提交
3509 3510 3511
	 */
	s->inuse = size;

3512
	if (((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
3513
		s->ctor)) {
C
Christoph Lameter 已提交
3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
		 * destructor or are poisoning the objects.
		 */
		s->offset = size;
		size += sizeof(void *);
	}

3526
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
3527 3528 3529 3530 3531 3532
	if (flags & SLAB_STORE_USER)
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);
3533
#endif
C
Christoph Lameter 已提交
3534

3535 3536
	kasan_cache_create(s, &size, &s->flags);
#ifdef CONFIG_SLUB_DEBUG
3537
	if (flags & SLAB_RED_ZONE) {
C
Christoph Lameter 已提交
3538 3539 3540 3541
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
3542
		 * corrupted if a user writes before the start
C
Christoph Lameter 已提交
3543 3544 3545
		 * of the object.
		 */
		size += sizeof(void *);
3546 3547 3548 3549 3550

		s->red_left_pad = sizeof(void *);
		s->red_left_pad = ALIGN(s->red_left_pad, s->align);
		size += s->red_left_pad;
	}
3551
#endif
C
Christoph Lameter 已提交
3552

C
Christoph Lameter 已提交
3553 3554 3555 3556 3557
	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
3558
	size = ALIGN(size, s->align);
C
Christoph Lameter 已提交
3559
	s->size = size;
3560 3561 3562
	if (forced_order >= 0)
		order = forced_order;
	else
3563
		order = calculate_order(size);
C
Christoph Lameter 已提交
3564

3565
	if ((int)order < 0)
C
Christoph Lameter 已提交
3566 3567
		return 0;

3568
	s->allocflags = 0;
3569
	if (order)
3570 3571 3572
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
3573
		s->allocflags |= GFP_DMA;
3574 3575 3576 3577

	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

C
Christoph Lameter 已提交
3578 3579 3580
	/*
	 * Determine the number of objects per slab
	 */
3581 3582
	s->oo = oo_make(order, size);
	s->min = oo_make(get_order(size), size);
3583 3584
	if (oo_objects(s->oo) > oo_objects(s->max))
		s->max = s->oo;
C
Christoph Lameter 已提交
3585

3586
	return !!oo_objects(s->oo);
C
Christoph Lameter 已提交
3587 3588
}

3589
static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
C
Christoph Lameter 已提交
3590
{
3591
	s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor);
3592 3593 3594
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	s->random = get_random_long();
#endif
C
Christoph Lameter 已提交
3595

3596
	if (!calculate_sizes(s, -1))
C
Christoph Lameter 已提交
3597
		goto error;
3598 3599 3600 3601 3602
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
3603
		if (get_order(s->size) > get_order(s->object_size)) {
3604 3605 3606 3607 3608 3609
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
			if (!calculate_sizes(s, -1))
				goto error;
		}
	}
C
Christoph Lameter 已提交
3610

3611 3612
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3613
	if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
3614 3615 3616 3617
		/* Enable fast mode */
		s->flags |= __CMPXCHG_DOUBLE;
#endif

3618 3619 3620 3621
	/*
	 * The larger the object size is, the more pages we want on the partial
	 * list to avoid pounding the page allocator excessively.
	 */
3622 3623
	set_min_partial(s, ilog2(s->size) / 2);

3624
	set_cpu_partial(s);
3625

C
Christoph Lameter 已提交
3626
#ifdef CONFIG_NUMA
3627
	s->remote_node_defrag_ratio = 1000;
C
Christoph Lameter 已提交
3628
#endif
3629 3630 3631 3632 3633 3634 3635

	/* Initialize the pre-computed randomized freelist if slab is up */
	if (slab_state >= UP) {
		if (init_cache_random_seq(s))
			goto error;
	}

3636
	if (!init_kmem_cache_nodes(s))
3637
		goto error;
C
Christoph Lameter 已提交
3638

3639
	if (alloc_kmem_cache_cpus(s))
3640
		return 0;
3641

3642
	free_kmem_cache_nodes(s);
C
Christoph Lameter 已提交
3643 3644
error:
	if (flags & SLAB_PANIC)
3645 3646
		panic("Cannot create slab %s size=%u realsize=%u order=%u offset=%u flags=%lx\n",
		      s->name, s->size, s->size,
3647
		      oo_order(s->oo), s->offset, (unsigned long)flags);
3648
	return -EINVAL;
C
Christoph Lameter 已提交
3649 3650
}

3651 3652 3653 3654 3655 3656
static void list_slab_objects(struct kmem_cache *s, struct page *page,
							const char *text)
{
#ifdef CONFIG_SLUB_DEBUG
	void *addr = page_address(page);
	void *p;
3657
	unsigned long *map = bitmap_zalloc(page->objects, GFP_ATOMIC);
3658 3659
	if (!map)
		return;
3660
	slab_err(s, page, text, s->name);
3661 3662
	slab_lock(page);

3663
	get_map(s, page, map);
3664 3665 3666
	for_each_object(p, s, addr, page->objects) {

		if (!test_bit(slab_index(p, s, addr), map)) {
3667
			pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
3668 3669 3670 3671
			print_tracking(s, p);
		}
	}
	slab_unlock(page);
3672
	bitmap_free(map);
3673 3674 3675
#endif
}

C
Christoph Lameter 已提交
3676
/*
3677
 * Attempt to free all partial slabs on a node.
3678 3679
 * This is called from __kmem_cache_shutdown(). We must take list_lock
 * because sysfs file might still access partial list after the shutdowning.
C
Christoph Lameter 已提交
3680
 */
3681
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3682
{
3683
	LIST_HEAD(discard);
C
Christoph Lameter 已提交
3684 3685
	struct page *page, *h;

3686 3687
	BUG_ON(irqs_disabled());
	spin_lock_irq(&n->list_lock);
3688
	list_for_each_entry_safe(page, h, &n->partial, lru) {
C
Christoph Lameter 已提交
3689
		if (!page->inuse) {
3690
			remove_partial(n, page);
3691
			list_add(&page->lru, &discard);
3692 3693
		} else {
			list_slab_objects(s, page,
3694
			"Objects remaining in %s on __kmem_cache_shutdown()");
3695
		}
3696
	}
3697
	spin_unlock_irq(&n->list_lock);
3698 3699 3700

	list_for_each_entry_safe(page, h, &discard, lru)
		discard_slab(s, page);
C
Christoph Lameter 已提交
3701 3702
}

3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713
bool __kmem_cache_empty(struct kmem_cache *s)
{
	int node;
	struct kmem_cache_node *n;

	for_each_kmem_cache_node(s, node, n)
		if (n->nr_partial || slabs_node(s, node))
			return false;
	return true;
}

C
Christoph Lameter 已提交
3714
/*
C
Christoph Lameter 已提交
3715
 * Release all resources used by a slab cache.
C
Christoph Lameter 已提交
3716
 */
3717
int __kmem_cache_shutdown(struct kmem_cache *s)
C
Christoph Lameter 已提交
3718 3719
{
	int node;
3720
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
3721 3722 3723

	flush_all(s);
	/* Attempt to free all objects */
3724
	for_each_kmem_cache_node(s, node, n) {
3725 3726
		free_partial(s, n);
		if (n->nr_partial || slabs_node(s, node))
C
Christoph Lameter 已提交
3727 3728
			return 1;
	}
3729
	sysfs_slab_remove(s);
C
Christoph Lameter 已提交
3730 3731 3732 3733 3734 3735 3736 3737 3738
	return 0;
}

/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

static int __init setup_slub_min_order(char *str)
{
3739
	get_option(&str, (int *)&slub_min_order);
C
Christoph Lameter 已提交
3740 3741 3742 3743 3744 3745 3746 3747

	return 1;
}

__setup("slub_min_order=", setup_slub_min_order);

static int __init setup_slub_max_order(char *str)
{
3748 3749
	get_option(&str, (int *)&slub_max_order);
	slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
C
Christoph Lameter 已提交
3750 3751 3752 3753 3754 3755 3756 3757

	return 1;
}

__setup("slub_max_order=", setup_slub_max_order);

static int __init setup_slub_min_objects(char *str)
{
3758
	get_option(&str, (int *)&slub_min_objects);
C
Christoph Lameter 已提交
3759 3760 3761 3762 3763 3764 3765 3766

	return 1;
}

__setup("slub_min_objects=", setup_slub_min_objects);

void *__kmalloc(size_t size, gfp_t flags)
{
3767
	struct kmem_cache *s;
3768
	void *ret;
C
Christoph Lameter 已提交
3769

3770
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3771
		return kmalloc_large(size, flags);
3772

3773
	s = kmalloc_slab(size, flags);
3774 3775

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3776 3777
		return s;

3778
	ret = slab_alloc(s, flags, _RET_IP_);
3779

3780
	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
3781

3782
	ret = kasan_kmalloc(s, ret, size, flags);
3783

3784
	return ret;
C
Christoph Lameter 已提交
3785 3786 3787
}
EXPORT_SYMBOL(__kmalloc);

3788
#ifdef CONFIG_NUMA
3789 3790
static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
3791
	struct page *page;
3792
	void *ptr = NULL;
3793

3794
	flags |= __GFP_COMP;
3795
	page = alloc_pages_node(node, flags, get_order(size));
3796
	if (page)
3797 3798
		ptr = page_address(page);

3799
	return kmalloc_large_node_hook(ptr, size, flags);
3800 3801
}

C
Christoph Lameter 已提交
3802 3803
void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
3804
	struct kmem_cache *s;
3805
	void *ret;
C
Christoph Lameter 已提交
3806

3807
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
3808 3809
		ret = kmalloc_large_node(size, flags, node);

3810 3811 3812
		trace_kmalloc_node(_RET_IP_, ret,
				   size, PAGE_SIZE << get_order(size),
				   flags, node);
3813 3814 3815

		return ret;
	}
3816

3817
	s = kmalloc_slab(size, flags);
3818 3819

	if (unlikely(ZERO_OR_NULL_PTR(s)))
3820 3821
		return s;

3822
	ret = slab_alloc_node(s, flags, node, _RET_IP_);
3823

3824
	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
3825

3826
	ret = kasan_kmalloc(s, ret, size, flags);
3827

3828
	return ret;
C
Christoph Lameter 已提交
3829 3830 3831 3832
}
EXPORT_SYMBOL(__kmalloc_node);
#endif

3833 3834
#ifdef CONFIG_HARDENED_USERCOPY
/*
3835 3836 3837
 * Rejects incorrectly sized objects and objects that are to be copied
 * to/from userspace but do not fall entirely within the containing slab
 * cache's usercopy region.
3838 3839 3840 3841
 *
 * Returns NULL if check passes, otherwise const char * to name of cache
 * to indicate an error.
 */
3842 3843
void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
			 bool to_user)
3844 3845
{
	struct kmem_cache *s;
3846
	unsigned int offset;
3847 3848 3849 3850 3851 3852 3853
	size_t object_size;

	/* Find object and usable object size. */
	s = page->slab_cache;

	/* Reject impossible pointers. */
	if (ptr < page_address(page))
3854 3855
		usercopy_abort("SLUB object not in SLUB page?!", NULL,
			       to_user, 0, n);
3856 3857 3858 3859 3860 3861 3862

	/* Find offset within object. */
	offset = (ptr - page_address(page)) % s->size;

	/* Adjust for redzone and reject if within the redzone. */
	if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE) {
		if (offset < s->red_left_pad)
3863 3864
			usercopy_abort("SLUB object in left red zone",
				       s->name, to_user, offset, n);
3865 3866 3867
		offset -= s->red_left_pad;
	}

3868 3869 3870 3871
	/* Allow address range falling entirely within usercopy region. */
	if (offset >= s->useroffset &&
	    offset - s->useroffset <= s->usersize &&
	    n <= s->useroffset - offset + s->usersize)
3872
		return;
3873

3874 3875 3876 3877 3878 3879 3880
	/*
	 * If the copy is still within the allocated object, produce
	 * a warning instead of rejecting the copy. This is intended
	 * to be a temporary method to find any missing usercopy
	 * whitelists.
	 */
	object_size = slab_ksize(s);
3881 3882
	if (usercopy_fallback &&
	    offset <= object_size && n <= object_size - offset) {
3883 3884 3885
		usercopy_warn("SLUB object", s->name, to_user, offset, n);
		return;
	}
3886

3887
	usercopy_abort("SLUB object", s->name, to_user, offset, n);
3888 3889 3890
}
#endif /* CONFIG_HARDENED_USERCOPY */

3891
static size_t __ksize(const void *object)
C
Christoph Lameter 已提交
3892
{
3893
	struct page *page;
C
Christoph Lameter 已提交
3894

3895
	if (unlikely(object == ZERO_SIZE_PTR))
3896 3897
		return 0;

3898 3899
	page = virt_to_head_page(object);

P
Pekka Enberg 已提交
3900 3901
	if (unlikely(!PageSlab(page))) {
		WARN_ON(!PageCompound(page));
3902
		return PAGE_SIZE << compound_order(page);
P
Pekka Enberg 已提交
3903
	}
C
Christoph Lameter 已提交
3904

3905
	return slab_ksize(page->slab_cache);
C
Christoph Lameter 已提交
3906
}
3907 3908 3909 3910 3911

size_t ksize(const void *object)
{
	size_t size = __ksize(object);
	/* We assume that ksize callers could use whole allocated area,
3912 3913 3914
	 * so we need to unpoison this area.
	 */
	kasan_unpoison_shadow(object, size);
3915 3916
	return size;
}
3917
EXPORT_SYMBOL(ksize);
C
Christoph Lameter 已提交
3918 3919 3920 3921

void kfree(const void *x)
{
	struct page *page;
3922
	void *object = (void *)x;
C
Christoph Lameter 已提交
3923

3924 3925
	trace_kfree(_RET_IP_, x);

3926
	if (unlikely(ZERO_OR_NULL_PTR(x)))
C
Christoph Lameter 已提交
3927 3928
		return;

3929
	page = virt_to_head_page(x);
3930
	if (unlikely(!PageSlab(page))) {
3931
		BUG_ON(!PageCompound(page));
3932
		kfree_hook(object);
3933
		__free_pages(page, compound_order(page));
3934 3935
		return;
	}
3936
	slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
C
Christoph Lameter 已提交
3937 3938 3939
}
EXPORT_SYMBOL(kfree);

3940 3941
#define SHRINK_PROMOTE_MAX 32

3942
/*
3943 3944 3945
 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
 * up most to the head of the partial lists. New allocations will then
 * fill those up and thus they can be removed from the partial lists.
C
Christoph Lameter 已提交
3946 3947 3948 3949
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
3950
 */
3951
int __kmem_cache_shrink(struct kmem_cache *s)
3952 3953 3954 3955 3956 3957
{
	int node;
	int i;
	struct kmem_cache_node *n;
	struct page *page;
	struct page *t;
3958 3959
	struct list_head discard;
	struct list_head promote[SHRINK_PROMOTE_MAX];
3960
	unsigned long flags;
3961
	int ret = 0;
3962 3963

	flush_all(s);
3964
	for_each_kmem_cache_node(s, node, n) {
3965 3966 3967
		INIT_LIST_HEAD(&discard);
		for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
			INIT_LIST_HEAD(promote + i);
3968 3969 3970 3971

		spin_lock_irqsave(&n->list_lock, flags);

		/*
3972
		 * Build lists of slabs to discard or promote.
3973
		 *
C
Christoph Lameter 已提交
3974 3975
		 * Note that concurrent frees may occur while we hold the
		 * list_lock. page->inuse here is the upper limit.
3976 3977
		 */
		list_for_each_entry_safe(page, t, &n->partial, lru) {
3978 3979 3980 3981 3982 3983 3984 3985 3986 3987
			int free = page->objects - page->inuse;

			/* Do not reread page->inuse */
			barrier();

			/* We do not keep full slabs on the list */
			BUG_ON(free <= 0);

			if (free == page->objects) {
				list_move(&page->lru, &discard);
3988
				n->nr_partial--;
3989 3990
			} else if (free <= SHRINK_PROMOTE_MAX)
				list_move(&page->lru, promote + free - 1);
3991 3992 3993
		}

		/*
3994 3995
		 * Promote the slabs filled up most to the head of the
		 * partial list.
3996
		 */
3997 3998
		for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
			list_splice(promote + i, &n->partial);
3999 4000

		spin_unlock_irqrestore(&n->list_lock, flags);
4001 4002

		/* Release empty slabs */
4003
		list_for_each_entry_safe(page, t, &discard, lru)
4004
			discard_slab(s, page);
4005 4006 4007

		if (slabs_node(s, node))
			ret = 1;
4008 4009
	}

4010
	return ret;
4011 4012
}

4013
#ifdef CONFIG_MEMCG
4014 4015
static void kmemcg_cache_deact_after_rcu(struct kmem_cache *s)
{
4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029
	/*
	 * Called with all the locks held after a sched RCU grace period.
	 * Even if @s becomes empty after shrinking, we can't know that @s
	 * doesn't have allocations already in-flight and thus can't
	 * destroy @s until the associated memcg is released.
	 *
	 * However, let's remove the sysfs files for empty caches here.
	 * Each cache has a lot of interface files which aren't
	 * particularly useful for empty draining caches; otherwise, we can
	 * easily end up with millions of unnecessary sysfs files on
	 * systems which have a lot of memory and transient cgroups.
	 */
	if (!__kmem_cache_shrink(s))
		sysfs_slab_remove(s);
4030 4031
}

4032 4033 4034 4035 4036 4037
void __kmemcg_cache_deactivate(struct kmem_cache *s)
{
	/*
	 * Disable empty slabs caching. Used to avoid pinning offline
	 * memory cgroups by kmem pages that can be freed.
	 */
4038
	slub_set_cpu_partial(s, 0);
4039 4040 4041 4042
	s->min_partial = 0;

	/*
	 * s->cpu_partial is checked locklessly (see put_cpu_partial), so
4043
	 * we have to make sure the change is visible before shrinking.
4044
	 */
4045
	slab_deactivate_memcg_cache_rcu_sched(s, kmemcg_cache_deact_after_rcu);
4046 4047 4048
}
#endif

4049 4050 4051 4052
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

4053
	mutex_lock(&slab_mutex);
4054
	list_for_each_entry(s, &slab_caches, list)
4055
		__kmem_cache_shrink(s);
4056
	mutex_unlock(&slab_mutex);
4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
	int offline_node;

4068
	offline_node = marg->status_change_nid_normal;
4069 4070 4071 4072 4073 4074 4075 4076

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

4077
	mutex_lock(&slab_mutex);
4078 4079 4080 4081 4082 4083
	list_for_each_entry(s, &slab_caches, list) {
		n = get_node(s, offline_node);
		if (n) {
			/*
			 * if n->nr_slabs > 0, slabs still exist on the node
			 * that is going down. We were unable to free them,
4084
			 * and offline_pages() function shouldn't call this
4085 4086
			 * callback. So, we must fail.
			 */
4087
			BUG_ON(slabs_node(s, offline_node));
4088 4089

			s->node[offline_node] = NULL;
4090
			kmem_cache_free(kmem_cache_node, n);
4091 4092
		}
	}
4093
	mutex_unlock(&slab_mutex);
4094 4095 4096 4097 4098 4099 4100
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
4101
	int nid = marg->status_change_nid_normal;
4102 4103 4104 4105 4106 4107 4108 4109 4110 4111
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
4112
	 * We are bringing a node online. No memory is available yet. We must
4113 4114 4115
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
4116
	mutex_lock(&slab_mutex);
4117 4118 4119 4120 4121 4122
	list_for_each_entry(s, &slab_caches, list) {
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
4123
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
4124 4125 4126 4127
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
4128
		init_kmem_cache_node(n);
4129 4130 4131
		s->node[nid] = n;
	}
out:
4132
	mutex_unlock(&slab_mutex);
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
4156 4157 4158 4159
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
4160 4161 4162
	return ret;
}

4163 4164 4165 4166
static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};
4167

C
Christoph Lameter 已提交
4168 4169 4170 4171
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

4172 4173
/*
 * Used for early kmem_cache structures that were allocated using
4174 4175
 * the page allocator. Allocate them properly then fix up the pointers
 * that may be pointing to the wrong kmem_cache structure.
4176 4177
 */

4178
static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
4179 4180
{
	int node;
4181
	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
4182
	struct kmem_cache_node *n;
4183

4184
	memcpy(s, static_cache, kmem_cache->object_size);
4185

4186 4187 4188 4189 4190 4191
	/*
	 * This runs very early, and only the boot processor is supposed to be
	 * up.  Even if it weren't true, IRQs are not up so we couldn't fire
	 * IPIs around.
	 */
	__flush_cpu_slab(s, smp_processor_id());
4192
	for_each_kmem_cache_node(s, node, n) {
4193 4194
		struct page *p;

4195 4196
		list_for_each_entry(p, &n->partial, lru)
			p->slab_cache = s;
4197

4198
#ifdef CONFIG_SLUB_DEBUG
4199 4200
		list_for_each_entry(p, &n->full, lru)
			p->slab_cache = s;
4201 4202
#endif
	}
4203
	slab_init_memcg_params(s);
4204
	list_add(&s->list, &slab_caches);
4205
	memcg_link_cache(s);
4206
	return s;
4207 4208
}

C
Christoph Lameter 已提交
4209 4210
void __init kmem_cache_init(void)
{
4211 4212
	static __initdata struct kmem_cache boot_kmem_cache,
		boot_kmem_cache_node;
4213

4214 4215 4216
	if (debug_guardpage_minorder())
		slub_max_order = 0;

4217 4218
	kmem_cache_node = &boot_kmem_cache_node;
	kmem_cache = &boot_kmem_cache;
4219

4220
	create_boot_cache(kmem_cache_node, "kmem_cache_node",
4221
		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
4222

4223
	register_hotmemory_notifier(&slab_memory_callback_nb);
C
Christoph Lameter 已提交
4224 4225 4226 4227

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

4228 4229 4230
	create_boot_cache(kmem_cache, "kmem_cache",
			offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *),
4231
		       SLAB_HWCACHE_ALIGN, 0, 0);
4232

4233 4234
	kmem_cache = bootstrap(&boot_kmem_cache);
	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
4235 4236

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
4237
	setup_kmalloc_cache_index_table();
4238
	create_kmalloc_caches(0);
C
Christoph Lameter 已提交
4239

4240 4241 4242
	/* Setup random freelists for each cache */
	init_freelist_randomization();

4243 4244
	cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
				  slub_cpu_dead);
C
Christoph Lameter 已提交
4245

4246
	pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%d\n",
4247
		cache_line_size(),
C
Christoph Lameter 已提交
4248 4249 4250 4251
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

4252 4253 4254 4255
void __init kmem_cache_init_late(void)
{
}

4256
struct kmem_cache *
4257
__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
4258
		   slab_flags_t flags, void (*ctor)(void *))
C
Christoph Lameter 已提交
4259
{
4260
	struct kmem_cache *s, *c;
C
Christoph Lameter 已提交
4261

4262
	s = find_mergeable(size, align, flags, name, ctor);
C
Christoph Lameter 已提交
4263 4264
	if (s) {
		s->refcount++;
4265

C
Christoph Lameter 已提交
4266 4267 4268 4269
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
4270
		s->object_size = max(s->object_size, size);
4271
		s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
C
Christoph Lameter 已提交
4272

4273
		for_each_memcg_cache(c, s) {
4274
			c->object_size = s->object_size;
4275
			c->inuse = max(c->inuse, ALIGN(size, sizeof(void *)));
4276 4277
		}

4278 4279
		if (sysfs_slab_alias(s, name)) {
			s->refcount--;
4280
			s = NULL;
4281
		}
4282
	}
C
Christoph Lameter 已提交
4283

4284 4285
	return s;
}
4286

4287
int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
4288
{
4289 4290 4291 4292 4293
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;
4294

4295 4296 4297 4298
	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

4299
	memcg_propagate_slab_attrs(s);
4300 4301
	err = sysfs_slab_add(s);
	if (err)
4302
		__kmem_cache_release(s);
4303

4304
	return err;
C
Christoph Lameter 已提交
4305 4306
}

4307
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
C
Christoph Lameter 已提交
4308
{
4309
	struct kmem_cache *s;
4310
	void *ret;
4311

4312
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
4313 4314
		return kmalloc_large(size, gfpflags);

4315
	s = kmalloc_slab(size, gfpflags);
C
Christoph Lameter 已提交
4316

4317
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4318
		return s;
C
Christoph Lameter 已提交
4319

4320
	ret = slab_alloc(s, gfpflags, caller);
4321

L
Lucas De Marchi 已提交
4322
	/* Honor the call site pointer we received. */
4323
	trace_kmalloc(caller, ret, size, s->size, gfpflags);
4324 4325

	return ret;
C
Christoph Lameter 已提交
4326 4327
}

4328
#ifdef CONFIG_NUMA
C
Christoph Lameter 已提交
4329
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4330
					int node, unsigned long caller)
C
Christoph Lameter 已提交
4331
{
4332
	struct kmem_cache *s;
4333
	void *ret;
4334

4335
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
4336 4337 4338 4339 4340 4341 4342 4343
		ret = kmalloc_large_node(size, gfpflags, node);

		trace_kmalloc_node(caller, ret,
				   size, PAGE_SIZE << get_order(size),
				   gfpflags, node);

		return ret;
	}
4344

4345
	s = kmalloc_slab(size, gfpflags);
C
Christoph Lameter 已提交
4346

4347
	if (unlikely(ZERO_OR_NULL_PTR(s)))
4348
		return s;
C
Christoph Lameter 已提交
4349

4350
	ret = slab_alloc_node(s, gfpflags, node, caller);
4351

L
Lucas De Marchi 已提交
4352
	/* Honor the call site pointer we received. */
4353
	trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4354 4355

	return ret;
C
Christoph Lameter 已提交
4356
}
4357
#endif
C
Christoph Lameter 已提交
4358

4359
#ifdef CONFIG_SYSFS
4360 4361 4362 4363 4364 4365 4366 4367 4368
static int count_inuse(struct page *page)
{
	return page->inuse;
}

static int count_total(struct page *page)
{
	return page->objects;
}
4369
#endif
4370

4371
#ifdef CONFIG_SLUB_DEBUG
4372 4373
static int validate_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4374 4375
{
	void *p;
4376
	void *addr = page_address(page);
4377 4378 4379 4380 4381 4382

	if (!check_slab(s, page) ||
			!on_freelist(s, page, NULL))
		return 0;

	/* Now we know that a valid freelist exists */
4383
	bitmap_zero(map, page->objects);
4384

4385 4386 4387 4388 4389
	get_map(s, page, map);
	for_each_object(p, s, addr, page->objects) {
		if (test_bit(slab_index(p, s, addr), map))
			if (!check_object(s, page, p, SLUB_RED_INACTIVE))
				return 0;
4390 4391
	}

4392
	for_each_object(p, s, addr, page->objects)
4393
		if (!test_bit(slab_index(p, s, addr), map))
4394
			if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4395 4396 4397 4398
				return 0;
	return 1;
}

4399 4400
static void validate_slab_slab(struct kmem_cache *s, struct page *page,
						unsigned long *map)
4401
{
4402 4403 4404
	slab_lock(page);
	validate_slab(s, page, map);
	slab_unlock(page);
4405 4406
}

4407 4408
static int validate_slab_node(struct kmem_cache *s,
		struct kmem_cache_node *n, unsigned long *map)
4409 4410 4411 4412 4413 4414 4415 4416
{
	unsigned long count = 0;
	struct page *page;
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

	list_for_each_entry(page, &n->partial, lru) {
4417
		validate_slab_slab(s, page, map);
4418 4419 4420
		count++;
	}
	if (count != n->nr_partial)
4421 4422
		pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
		       s->name, count, n->nr_partial);
4423 4424 4425 4426 4427

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

	list_for_each_entry(page, &n->full, lru) {
4428
		validate_slab_slab(s, page, map);
4429 4430 4431
		count++;
	}
	if (count != atomic_long_read(&n->nr_slabs))
4432 4433
		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
		       s->name, count, atomic_long_read(&n->nr_slabs));
4434 4435 4436 4437 4438 4439

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

4440
static long validate_slab_cache(struct kmem_cache *s)
4441 4442 4443
{
	int node;
	unsigned long count = 0;
4444
	struct kmem_cache_node *n;
4445
	unsigned long *map = bitmap_alloc(oo_objects(s->max), GFP_KERNEL);
4446 4447 4448

	if (!map)
		return -ENOMEM;
4449 4450

	flush_all(s);
4451
	for_each_kmem_cache_node(s, node, n)
4452
		count += validate_slab_node(s, n, map);
4453
	bitmap_free(map);
4454 4455
	return count;
}
4456
/*
C
Christoph Lameter 已提交
4457
 * Generate lists of code addresses where slabcache objects are allocated
4458 4459 4460 4461 4462
 * and freed.
 */

struct location {
	unsigned long count;
4463
	unsigned long addr;
4464 4465 4466 4467 4468
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
R
Rusty Russell 已提交
4469
	DECLARE_BITMAP(cpus, NR_CPUS);
4470
	nodemask_t nodes;
4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
};

static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

4486
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4487 4488 4489 4490 4491 4492
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

4493
	l = (void *)__get_free_pages(flags, order);
4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
4507
				const struct track *track)
4508 4509 4510
{
	long start, end, pos;
	struct location *l;
4511
	unsigned long caddr;
4512
	unsigned long age = jiffies - track->when;
4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527

	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

		caddr = t->loc[pos].addr;
4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543
		if (track->addr == caddr) {

			l = &t->loc[pos];
			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

R
Rusty Russell 已提交
4544 4545
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
4546 4547
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
4548 4549 4550
			return 1;
		}

4551
		if (track->addr < caddr)
4552 4553 4554 4555 4556 4557
			end = pos;
		else
			start = pos;
	}

	/*
C
Christoph Lameter 已提交
4558
	 * Not found. Insert new tracking element.
4559
	 */
4560
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4561 4562 4563 4564 4565 4566 4567 4568
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
4569 4570 4571 4572 4573 4574
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
R
Rusty Russell 已提交
4575 4576
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4577 4578
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
4579 4580 4581 4582
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
4583
		struct page *page, enum track_item alloc,
4584
		unsigned long *map)
4585
{
4586
	void *addr = page_address(page);
4587 4588
	void *p;

4589
	bitmap_zero(map, page->objects);
4590
	get_map(s, page, map);
4591

4592
	for_each_object(p, s, addr, page->objects)
4593 4594
		if (!test_bit(slab_index(p, s, addr), map))
			add_location(t, s, get_track(s, p, alloc));
4595 4596 4597 4598 4599
}

static int list_locations(struct kmem_cache *s, char *buf,
					enum track_item alloc)
{
4600
	int len = 0;
4601
	unsigned long i;
4602
	struct loc_track t = { 0, 0, NULL };
4603
	int node;
4604
	struct kmem_cache_node *n;
4605
	unsigned long *map = bitmap_alloc(oo_objects(s->max), GFP_KERNEL);
4606

4607
	if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4608
				     GFP_KERNEL)) {
4609
		bitmap_free(map);
4610
		return sprintf(buf, "Out of memory\n");
4611
	}
4612 4613 4614
	/* Push back cpu slabs */
	flush_all(s);

4615
	for_each_kmem_cache_node(s, node, n) {
4616 4617 4618
		unsigned long flags;
		struct page *page;

4619
		if (!atomic_long_read(&n->nr_slabs))
4620 4621 4622 4623
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry(page, &n->partial, lru)
4624
			process_slab(&t, s, page, alloc, map);
4625
		list_for_each_entry(page, &n->full, lru)
4626
			process_slab(&t, s, page, alloc, map);
4627 4628 4629 4630
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	for (i = 0; i < t.count; i++) {
4631
		struct location *l = &t.loc[i];
4632

H
Hugh Dickins 已提交
4633
		if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4634
			break;
4635
		len += sprintf(buf + len, "%7ld ", l->count);
4636 4637

		if (l->addr)
4638
			len += sprintf(buf + len, "%pS", (void *)l->addr);
4639
		else
4640
			len += sprintf(buf + len, "<not-available>");
4641 4642

		if (l->sum_time != l->min_time) {
4643
			len += sprintf(buf + len, " age=%ld/%ld/%ld",
R
Roman Zippel 已提交
4644 4645 4646
				l->min_time,
				(long)div_u64(l->sum_time, l->count),
				l->max_time);
4647
		} else
4648
			len += sprintf(buf + len, " age=%ld",
4649 4650 4651
				l->min_time);

		if (l->min_pid != l->max_pid)
4652
			len += sprintf(buf + len, " pid=%ld-%ld",
4653 4654
				l->min_pid, l->max_pid);
		else
4655
			len += sprintf(buf + len, " pid=%ld",
4656 4657
				l->min_pid);

R
Rusty Russell 已提交
4658 4659
		if (num_online_cpus() > 1 &&
				!cpumask_empty(to_cpumask(l->cpus)) &&
4660 4661 4662 4663
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " cpus=%*pbl",
					 cpumask_pr_args(to_cpumask(l->cpus)));
4664

4665
		if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4666 4667 4668 4669
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " nodes=%*pbl",
					 nodemask_pr_args(&l->nodes));
4670

4671
		len += sprintf(buf + len, "\n");
4672 4673 4674
	}

	free_loc_track(&t);
4675
	bitmap_free(map);
4676
	if (!t.count)
4677 4678
		len += sprintf(buf, "No data\n");
	return len;
4679
}
4680
#endif
4681

4682
#ifdef SLUB_RESILIENCY_TEST
4683
static void __init resiliency_test(void)
4684 4685
{
	u8 *p;
4686
	int type = KMALLOC_NORMAL;
4687

4688
	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
4689

4690 4691 4692
	pr_err("SLUB resiliency testing\n");
	pr_err("-----------------------\n");
	pr_err("A. Corruption after allocation\n");
4693 4694 4695

	p = kzalloc(16, GFP_KERNEL);
	p[16] = 0x12;
4696 4697
	pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
	       p + 16);
4698

4699
	validate_slab_cache(kmalloc_caches[type][4]);
4700 4701 4702 4703

	/* Hmmm... The next two are dangerous */
	p = kzalloc(32, GFP_KERNEL);
	p[32 + sizeof(void *)] = 0x34;
4704 4705 4706
	pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");
4707

4708
	validate_slab_cache(kmalloc_caches[type][5]);
4709 4710 4711
	p = kzalloc(64, GFP_KERNEL);
	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
	*p = 0x56;
4712 4713 4714
	pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
	       p);
	pr_err("If allocated object is overwritten then not detectable\n\n");
4715
	validate_slab_cache(kmalloc_caches[type][6]);
4716

4717
	pr_err("\nB. Corruption after free\n");
4718 4719 4720
	p = kzalloc(128, GFP_KERNEL);
	kfree(p);
	*p = 0x78;
4721
	pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
4722
	validate_slab_cache(kmalloc_caches[type][7]);
4723 4724 4725 4726

	p = kzalloc(256, GFP_KERNEL);
	kfree(p);
	p[50] = 0x9a;
4727
	pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
4728
	validate_slab_cache(kmalloc_caches[type][8]);
4729 4730 4731 4732

	p = kzalloc(512, GFP_KERNEL);
	kfree(p);
	p[512] = 0xab;
4733
	pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
4734
	validate_slab_cache(kmalloc_caches[type][9]);
4735 4736 4737 4738 4739 4740 4741
}
#else
#ifdef CONFIG_SYSFS
static void resiliency_test(void) {};
#endif
#endif

4742
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
4743
enum slab_stat_type {
4744 4745 4746 4747 4748
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
C
Christoph Lameter 已提交
4749 4750
};

4751
#define SO_ALL		(1 << SL_ALL)
C
Christoph Lameter 已提交
4752 4753 4754
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
4755
#define SO_TOTAL	(1 << SL_TOTAL)
C
Christoph Lameter 已提交
4756

4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772
#ifdef CONFIG_MEMCG
static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

static int __init setup_slub_memcg_sysfs(char *str)
{
	int v;

	if (get_option(&str, &v) > 0)
		memcg_sysfs_enabled = v;

	return 1;
}

__setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs);
#endif

4773 4774
static ssize_t show_slab_objects(struct kmem_cache *s,
			    char *buf, unsigned long flags)
C
Christoph Lameter 已提交
4775 4776 4777 4778 4779 4780
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;

4781
	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
4782 4783
	if (!nodes)
		return -ENOMEM;
C
Christoph Lameter 已提交
4784

4785 4786
	if (flags & SO_CPU) {
		int cpu;
C
Christoph Lameter 已提交
4787

4788
		for_each_possible_cpu(cpu) {
4789 4790
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
							       cpu);
4791
			int node;
4792
			struct page *page;
4793

4794
			page = READ_ONCE(c->page);
4795 4796
			if (!page)
				continue;
4797

4798 4799 4800 4801 4802 4803 4804
			node = page_to_nid(page);
			if (flags & SO_TOTAL)
				x = page->objects;
			else if (flags & SO_OBJECTS)
				x = page->inuse;
			else
				x = 1;
4805

4806 4807 4808
			total += x;
			nodes[node] += x;

4809
			page = slub_percpu_partial_read_once(c);
4810
			if (page) {
4811 4812 4813 4814 4815 4816 4817
				node = page_to_nid(page);
				if (flags & SO_TOTAL)
					WARN_ON_ONCE(1);
				else if (flags & SO_OBJECTS)
					WARN_ON_ONCE(1);
				else
					x = page->pages;
4818 4819
				total += x;
				nodes[node] += x;
4820
			}
C
Christoph Lameter 已提交
4821 4822 4823
		}
	}

4824
	get_online_mems();
4825
#ifdef CONFIG_SLUB_DEBUG
4826
	if (flags & SO_ALL) {
4827 4828 4829
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {
4830

4831 4832 4833 4834 4835
			if (flags & SO_TOTAL)
				x = atomic_long_read(&n->total_objects);
			else if (flags & SO_OBJECTS)
				x = atomic_long_read(&n->total_objects) -
					count_partial(n, count_free);
C
Christoph Lameter 已提交
4836
			else
4837
				x = atomic_long_read(&n->nr_slabs);
C
Christoph Lameter 已提交
4838 4839 4840 4841
			total += x;
			nodes[node] += x;
		}

4842 4843 4844
	} else
#endif
	if (flags & SO_PARTIAL) {
4845
		struct kmem_cache_node *n;
C
Christoph Lameter 已提交
4846

4847
		for_each_kmem_cache_node(s, node, n) {
4848 4849 4850 4851
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
C
Christoph Lameter 已提交
4852
			else
4853
				x = n->nr_partial;
C
Christoph Lameter 已提交
4854 4855 4856 4857 4858 4859
			total += x;
			nodes[node] += x;
		}
	}
	x = sprintf(buf, "%lu", total);
#ifdef CONFIG_NUMA
4860
	for (node = 0; node < nr_node_ids; node++)
C
Christoph Lameter 已提交
4861 4862 4863 4864
		if (nodes[node])
			x += sprintf(buf + x, " N%d=%lu",
					node, nodes[node]);
#endif
4865
	put_online_mems();
C
Christoph Lameter 已提交
4866 4867 4868 4869
	kfree(nodes);
	return x + sprintf(buf + x, "\n");
}

4870
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4871 4872 4873
static int any_slab_objects(struct kmem_cache *s)
{
	int node;
4874
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
4875

4876
	for_each_kmem_cache_node(s, node, n)
4877
		if (atomic_long_read(&n->total_objects))
C
Christoph Lameter 已提交
4878
			return 1;
4879

C
Christoph Lameter 已提交
4880 4881
	return 0;
}
4882
#endif
C
Christoph Lameter 已提交
4883 4884

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4885
#define to_slab(n) container_of(n, struct kmem_cache, kobj)
C
Christoph Lameter 已提交
4886 4887 4888 4889 4890 4891 4892 4893

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
4894 4895
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)
C
Christoph Lameter 已提交
4896 4897 4898

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
4899
	__ATTR(_name, 0600, _name##_show, _name##_store)
C
Christoph Lameter 已提交
4900 4901 4902

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
4903
	return sprintf(buf, "%u\n", s->size);
C
Christoph Lameter 已提交
4904 4905 4906 4907 4908
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
4909
	return sprintf(buf, "%u\n", s->align);
C
Christoph Lameter 已提交
4910 4911 4912 4913 4914
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
4915
	return sprintf(buf, "%u\n", s->object_size);
C
Christoph Lameter 已提交
4916 4917 4918 4919 4920
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
4921
	return sprintf(buf, "%u\n", oo_objects(s->oo));
C
Christoph Lameter 已提交
4922 4923 4924
}
SLAB_ATTR_RO(objs_per_slab);

4925 4926 4927
static ssize_t order_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
4928
	unsigned int order;
4929 4930
	int err;

4931
	err = kstrtouint(buf, 10, &order);
4932 4933
	if (err)
		return err;
4934 4935 4936 4937 4938 4939 4940 4941

	if (order > slub_max_order || order < slub_min_order)
		return -EINVAL;

	calculate_sizes(s, order);
	return length;
}

C
Christoph Lameter 已提交
4942 4943
static ssize_t order_show(struct kmem_cache *s, char *buf)
{
4944
	return sprintf(buf, "%u\n", oo_order(s->oo));
C
Christoph Lameter 已提交
4945
}
4946
SLAB_ATTR(order);
C
Christoph Lameter 已提交
4947

4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%lu\n", s->min_partial);
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

4959
	err = kstrtoul(buf, 10, &min);
4960 4961 4962
	if (err)
		return err;

4963
	set_min_partial(s, min);
4964 4965 4966 4967
	return length;
}
SLAB_ATTR(min_partial);

4968 4969
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
4970
	return sprintf(buf, "%u\n", slub_cpu_partial(s));
4971 4972 4973 4974 4975
}

static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
4976
	unsigned int objects;
4977 4978
	int err;

4979
	err = kstrtouint(buf, 10, &objects);
4980 4981
	if (err)
		return err;
4982
	if (objects && !kmem_cache_has_cpu_partial(s))
4983
		return -EINVAL;
4984

4985
	slub_set_cpu_partial(s, objects);
4986 4987 4988 4989 4990
	flush_all(s);
	return length;
}
SLAB_ATTR(cpu_partial);

C
Christoph Lameter 已提交
4991 4992
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
4993 4994 4995
	if (!s->ctor)
		return 0;
	return sprintf(buf, "%pS\n", s->ctor);
C
Christoph Lameter 已提交
4996 4997 4998 4999 5000
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
5001
	return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
C
Christoph Lameter 已提交
5002 5003 5004 5005 5006
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
5007
	return show_slab_objects(s, buf, SO_PARTIAL);
C
Christoph Lameter 已提交
5008 5009 5010 5011 5012
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
5013
	return show_slab_objects(s, buf, SO_CPU);
C
Christoph Lameter 已提交
5014 5015 5016 5017 5018
}
SLAB_ATTR_RO(cpu_slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
5019
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
C
Christoph Lameter 已提交
5020 5021 5022
}
SLAB_ATTR_RO(objects);

5023 5024 5025 5026 5027 5028
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

5029 5030 5031 5032 5033 5034 5035 5036
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
	int pages = 0;
	int cpu;
	int len;

	for_each_online_cpu(cpu) {
5037 5038 5039
		struct page *page;

		page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050

		if (page) {
			pages += page->pages;
			objects += page->pobjects;
		}
	}

	len = sprintf(buf, "%d(%d)", objects, pages);

#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
5051 5052 5053
		struct page *page;

		page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
5054 5055 5056 5057 5058 5059 5060 5061 5062 5063

		if (page && len < PAGE_SIZE - 20)
			len += sprintf(buf + len, " C%d=%d(%d)", cpu,
				page->pobjects, page->pages);
	}
#endif
	return len + sprintf(buf + len, "\n");
}
SLAB_ATTR_RO(slabs_cpu_partial);

5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}

static ssize_t reclaim_account_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
	if (buf[0] == '1')
		s->flags |= SLAB_RECLAIM_ACCOUNT;
	return length;
}
SLAB_ATTR(reclaim_account);

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
}
SLAB_ATTR_RO(cache_dma);
#endif

5093 5094
static ssize_t usersize_show(struct kmem_cache *s, char *buf)
{
5095
	return sprintf(buf, "%u\n", s->usersize);
5096 5097 5098
}
SLAB_ATTR_RO(usersize);

5099 5100
static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
5101
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
5102 5103 5104
}
SLAB_ATTR_RO(destroy_by_rcu);

5105
#ifdef CONFIG_SLUB_DEBUG
5106 5107 5108 5109 5110 5111
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

5112 5113 5114 5115 5116 5117
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

C
Christoph Lameter 已提交
5118 5119
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
5120
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
C
Christoph Lameter 已提交
5121 5122 5123 5124 5125
}

static ssize_t sanity_checks_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
5126
	s->flags &= ~SLAB_CONSISTENCY_CHECKS;
5127 5128
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
5129
		s->flags |= SLAB_CONSISTENCY_CHECKS;
5130
	}
C
Christoph Lameter 已提交
5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142
	return length;
}
SLAB_ATTR(sanity_checks);

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
}

static ssize_t trace_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
5143 5144 5145 5146 5147 5148 5149 5150
	/*
	 * Tracing a merged cache is going to give confusing results
	 * as well as cause other issues like converting a mergeable
	 * cache into an umergeable one.
	 */
	if (s->refcount > 1)
		return -EINVAL;

C
Christoph Lameter 已提交
5151
	s->flags &= ~SLAB_TRACE;
5152 5153
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
5154
		s->flags |= SLAB_TRACE;
5155
	}
C
Christoph Lameter 已提交
5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
	return length;
}
SLAB_ATTR(trace);

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}

static ssize_t red_zone_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_RED_ZONE;
5172
	if (buf[0] == '1') {
C
Christoph Lameter 已提交
5173
		s->flags |= SLAB_RED_ZONE;
5174
	}
5175
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191
	return length;
}
SLAB_ATTR(red_zone);

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}

static ssize_t poison_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_POISON;
5192
	if (buf[0] == '1') {
C
Christoph Lameter 已提交
5193
		s->flags |= SLAB_POISON;
5194
	}
5195
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211
	return length;
}
SLAB_ATTR(poison);

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
}

static ssize_t store_user_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_STORE_USER;
5212 5213
	if (buf[0] == '1') {
		s->flags &= ~__CMPXCHG_DOUBLE;
C
Christoph Lameter 已提交
5214
		s->flags |= SLAB_STORE_USER;
5215
	}
5216
	calculate_sizes(s, -1);
C
Christoph Lameter 已提交
5217 5218 5219 5220
	return length;
}
SLAB_ATTR(store_user);

5221 5222 5223 5224 5225 5226 5227 5228
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
5229 5230 5231 5232 5233 5234 5235 5236
	int ret = -EINVAL;

	if (buf[0] == '1') {
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
5237 5238
}
SLAB_ATTR(validate);
5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265

static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_ALLOC);
}
SLAB_ATTR_RO(alloc_calls);

static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_FREE);
}
SLAB_ATTR_RO(free_calls);
#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
}

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
							size_t length)
{
5266 5267 5268
	if (s->refcount > 1)
		return -EINVAL;

5269 5270 5271 5272 5273 5274
	s->flags &= ~SLAB_FAILSLAB;
	if (buf[0] == '1')
		s->flags |= SLAB_FAILSLAB;
	return length;
}
SLAB_ATTR(failslab);
5275
#endif
5276

5277 5278 5279 5280 5281 5282 5283 5284
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
5285 5286 5287
	if (buf[0] == '1')
		kmem_cache_shrink(s);
	else
5288 5289 5290 5291 5292
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

C
Christoph Lameter 已提交
5293
#ifdef CONFIG_NUMA
5294
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
C
Christoph Lameter 已提交
5295
{
5296
	return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10);
C
Christoph Lameter 已提交
5297 5298
}

5299
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
C
Christoph Lameter 已提交
5300 5301
				const char *buf, size_t length)
{
5302
	unsigned int ratio;
5303 5304
	int err;

5305
	err = kstrtouint(buf, 10, &ratio);
5306 5307
	if (err)
		return err;
5308 5309
	if (ratio > 100)
		return -ERANGE;
5310

5311
	s->remote_node_defrag_ratio = ratio * 10;
C
Christoph Lameter 已提交
5312 5313 5314

	return length;
}
5315
SLAB_ATTR(remote_node_defrag_ratio);
C
Christoph Lameter 已提交
5316 5317
#endif

5318 5319 5320 5321 5322 5323
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
	int len;
5324
	int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
5325 5326 5327 5328 5329

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
5330
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5331 5332 5333 5334 5335 5336 5337

		data[cpu] = x;
		sum += x;
	}

	len = sprintf(buf, "%lu", sum);

5338
#ifdef CONFIG_SMP
5339 5340
	for_each_online_cpu(cpu) {
		if (data[cpu] && len < PAGE_SIZE - 20)
5341
			len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5342
	}
5343
#endif
5344 5345 5346 5347
	kfree(data);
	return len + sprintf(buf + len, "\n");
}

5348 5349 5350 5351 5352
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
5353
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
5354 5355
}

5356 5357 5358 5359 5360
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
5361 5362 5363 5364 5365 5366 5367 5368 5369
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
5381
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5382 5383 5384 5385 5386 5387 5388
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5389
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5390
STAT_ATTR(ORDER_FALLBACK, order_fallback);
5391 5392
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5393 5394
STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5395 5396
STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5397 5398
#endif

5399
static struct attribute *slab_attrs[] = {
C
Christoph Lameter 已提交
5400 5401 5402 5403
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
5404
	&min_partial_attr.attr,
5405
	&cpu_partial_attr.attr,
C
Christoph Lameter 已提交
5406
	&objects_attr.attr,
5407
	&objects_partial_attr.attr,
C
Christoph Lameter 已提交
5408 5409 5410 5411 5412 5413 5414 5415
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
5416
	&shrink_attr.attr,
5417
	&slabs_cpu_partial_attr.attr,
5418
#ifdef CONFIG_SLUB_DEBUG
5419 5420 5421 5422
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
C
Christoph Lameter 已提交
5423 5424 5425
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
5426
	&validate_attr.attr,
5427 5428
	&alloc_calls_attr.attr,
	&free_calls_attr.attr,
5429
#endif
C
Christoph Lameter 已提交
5430 5431 5432 5433
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
5434
	&remote_node_defrag_ratio_attr.attr,
5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
5447
	&alloc_node_mismatch_attr.attr,
5448 5449 5450 5451 5452 5453 5454
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
5455
	&deactivate_bypass_attr.attr,
5456
	&order_fallback_attr.attr,
5457 5458
	&cmpxchg_double_fail_attr.attr,
	&cmpxchg_double_cpu_fail_attr.attr,
5459 5460
	&cpu_partial_alloc_attr.attr,
	&cpu_partial_free_attr.attr,
5461 5462
	&cpu_partial_node_attr.attr,
	&cpu_partial_drain_attr.attr,
C
Christoph Lameter 已提交
5463
#endif
5464 5465 5466
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif
5467
	&usersize_attr.attr,
5468

C
Christoph Lameter 已提交
5469 5470 5471
	NULL
};

5472
static const struct attribute_group slab_attr_group = {
C
Christoph Lameter 已提交
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);
5510
#ifdef CONFIG_MEMCG
5511
	if (slab_state >= FULL && err >= 0 && is_root_cache(s)) {
5512
		struct kmem_cache *c;
C
Christoph Lameter 已提交
5513

5514 5515 5516 5517
		mutex_lock(&slab_mutex);
		if (s->max_attr_size < len)
			s->max_attr_size = len;

5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534
		/*
		 * This is a best effort propagation, so this function's return
		 * value will be determined by the parent cache only. This is
		 * basically because not all attributes will have a well
		 * defined semantics for rollbacks - most of the actions will
		 * have permanent effects.
		 *
		 * Returning the error value of any of the children that fail
		 * is not 100 % defined, in the sense that users seeing the
		 * error code won't be able to know anything about the state of
		 * the cache.
		 *
		 * Only returning the error code for the parent cache at least
		 * has well defined semantics. The cache being written to
		 * directly either failed or succeeded, in which case we loop
		 * through the descendants with best-effort propagation.
		 */
5535 5536
		for_each_memcg_cache(c, s)
			attribute->store(c, buf, len);
5537 5538 5539
		mutex_unlock(&slab_mutex);
	}
#endif
C
Christoph Lameter 已提交
5540 5541 5542
	return err;
}

5543 5544
static void memcg_propagate_slab_attrs(struct kmem_cache *s)
{
5545
#ifdef CONFIG_MEMCG
5546 5547
	int i;
	char *buffer = NULL;
5548
	struct kmem_cache *root_cache;
5549

5550
	if (is_root_cache(s))
5551 5552
		return;

5553
	root_cache = s->memcg_params.root_cache;
5554

5555 5556 5557 5558
	/*
	 * This mean this cache had no attribute written. Therefore, no point
	 * in copying default values around
	 */
5559
	if (!root_cache->max_attr_size)
5560 5561 5562 5563 5564 5565
		return;

	for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) {
		char mbuf[64];
		char *buf;
		struct slab_attribute *attr = to_slab_attr(slab_attrs[i]);
5566
		ssize_t len;
5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581

		if (!attr || !attr->store || !attr->show)
			continue;

		/*
		 * It is really bad that we have to allocate here, so we will
		 * do it only as a fallback. If we actually allocate, though,
		 * we can just use the allocated buffer until the end.
		 *
		 * Most of the slub attributes will tend to be very small in
		 * size, but sysfs allows buffers up to a page, so they can
		 * theoretically happen.
		 */
		if (buffer)
			buf = buffer;
5582
		else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf))
5583 5584 5585 5586 5587 5588 5589 5590
			buf = mbuf;
		else {
			buffer = (char *) get_zeroed_page(GFP_KERNEL);
			if (WARN_ON(!buffer))
				continue;
			buf = buffer;
		}

5591 5592 5593
		len = attr->show(root_cache, buf);
		if (len > 0)
			attr->store(s, buf, len);
5594 5595 5596 5597 5598 5599 5600
	}

	if (buffer)
		free_page((unsigned long)buffer);
#endif
}

5601 5602 5603 5604 5605
static void kmem_cache_release(struct kobject *k)
{
	slab_kmem_cache_release(to_slab(k));
}

5606
static const struct sysfs_ops slab_sysfs_ops = {
C
Christoph Lameter 已提交
5607 5608 5609 5610 5611 5612
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
5613
	.release = kmem_cache_release,
C
Christoph Lameter 已提交
5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624
};

static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;
}

5625
static const struct kset_uevent_ops slab_uevent_ops = {
C
Christoph Lameter 已提交
5626 5627 5628
	.filter = uevent_filter,
};

5629
static struct kset *slab_kset;
C
Christoph Lameter 已提交
5630

5631 5632
static inline struct kset *cache_kset(struct kmem_cache *s)
{
5633
#ifdef CONFIG_MEMCG
5634
	if (!is_root_cache(s))
5635
		return s->memcg_params.root_cache->memcg_kset;
5636 5637 5638 5639
#endif
	return slab_kset;
}

C
Christoph Lameter 已提交
5640 5641 5642
#define ID_STR_LENGTH 64

/* Create a unique string id for a slab cache:
C
Christoph Lameter 已提交
5643 5644
 *
 * Format	:[flags-]size
C
Christoph Lameter 已提交
5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

	BUG_ON(!name);

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
5665
	if (s->flags & SLAB_CONSISTENCY_CHECKS)
C
Christoph Lameter 已提交
5666
		*p++ = 'F';
5667 5668
	if (s->flags & SLAB_ACCOUNT)
		*p++ = 'A';
C
Christoph Lameter 已提交
5669 5670
	if (p != name + 1)
		*p++ = '-';
5671
	p += sprintf(p, "%07u", s->size);
5672

C
Christoph Lameter 已提交
5673 5674 5675 5676
	BUG_ON(p > name + ID_STR_LENGTH - 1);
	return name;
}

5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688
static void sysfs_slab_remove_workfn(struct work_struct *work)
{
	struct kmem_cache *s =
		container_of(work, struct kmem_cache, kobj_remove_work);

	if (!s->kobj.state_in_sysfs)
		/*
		 * For a memcg cache, this may be called during
		 * deactivation and again on shutdown.  Remove only once.
		 * A cache is never shut down before deactivation is
		 * complete, so no need to worry about synchronization.
		 */
5689
		goto out;
5690 5691 5692 5693 5694

#ifdef CONFIG_MEMCG
	kset_unregister(s->memcg_kset);
#endif
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
5695
out:
5696 5697 5698
	kobject_put(&s->kobj);
}

C
Christoph Lameter 已提交
5699 5700 5701 5702
static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
5703
	struct kset *kset = cache_kset(s);
5704
	int unmergeable = slab_unmergeable(s);
C
Christoph Lameter 已提交
5705

5706 5707
	INIT_WORK(&s->kobj_remove_work, sysfs_slab_remove_workfn);

5708 5709 5710 5711 5712
	if (!kset) {
		kobject_init(&s->kobj, &slab_ktype);
		return 0;
	}

5713 5714 5715 5716
	if (!unmergeable && disable_higher_order_debug &&
			(slub_debug & DEBUG_METADATA_FLAGS))
		unmergeable = 1;

C
Christoph Lameter 已提交
5717 5718 5719 5720 5721 5722
	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
5723
		sysfs_remove_link(&slab_kset->kobj, s->name);
C
Christoph Lameter 已提交
5724 5725 5726 5727 5728 5729 5730 5731 5732
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
	}

5733
	s->kobj.kset = kset;
5734
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
5735
	if (err)
5736
		goto out;
C
Christoph Lameter 已提交
5737 5738

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
5739 5740
	if (err)
		goto out_del_kobj;
5741

5742
#ifdef CONFIG_MEMCG
5743
	if (is_root_cache(s) && memcg_sysfs_enabled) {
5744 5745
		s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
		if (!s->memcg_kset) {
5746 5747
			err = -ENOMEM;
			goto out_del_kobj;
5748 5749 5750 5751
		}
	}
#endif

C
Christoph Lameter 已提交
5752 5753 5754 5755 5756
	kobject_uevent(&s->kobj, KOBJ_ADD);
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
	}
5757 5758 5759 5760 5761 5762 5763
out:
	if (!unmergeable)
		kfree(name);
	return err;
out_del_kobj:
	kobject_del(&s->kobj);
	goto out;
C
Christoph Lameter 已提交
5764 5765
}

5766
static void sysfs_slab_remove(struct kmem_cache *s)
C
Christoph Lameter 已提交
5767
{
5768
	if (slab_state < FULL)
5769 5770 5771 5772 5773 5774
		/*
		 * Sysfs has not been setup yet so no need to remove the
		 * cache from sysfs.
		 */
		return;

5775 5776
	kobject_get(&s->kobj);
	schedule_work(&s->kobj_remove_work);
5777 5778
}

5779 5780 5781 5782 5783 5784
void sysfs_slab_unlink(struct kmem_cache *s)
{
	if (slab_state >= FULL)
		kobject_del(&s->kobj);
}

5785 5786 5787 5788
void sysfs_slab_release(struct kmem_cache *s)
{
	if (slab_state >= FULL)
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5789 5790 5791 5792
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
5793
 * available lest we lose that information.
C
Christoph Lameter 已提交
5794 5795 5796 5797 5798 5799 5800
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

A
Adrian Bunk 已提交
5801
static struct saved_alias *alias_list;
C
Christoph Lameter 已提交
5802 5803 5804 5805 5806

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

5807
	if (slab_state == FULL) {
C
Christoph Lameter 已提交
5808 5809 5810
		/*
		 * If we have a leftover link then remove it.
		 */
5811 5812
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
C
Christoph Lameter 已提交
5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
	return 0;
}

static int __init slab_sysfs_init(void)
{
5828
	struct kmem_cache *s;
C
Christoph Lameter 已提交
5829 5830
	int err;

5831
	mutex_lock(&slab_mutex);
5832

5833
	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5834
	if (!slab_kset) {
5835
		mutex_unlock(&slab_mutex);
5836
		pr_err("Cannot register slab subsystem.\n");
C
Christoph Lameter 已提交
5837 5838 5839
		return -ENOSYS;
	}

5840
	slab_state = FULL;
5841

5842
	list_for_each_entry(s, &slab_caches, list) {
5843
		err = sysfs_slab_add(s);
5844
		if (err)
5845 5846
			pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
			       s->name);
5847
	}
C
Christoph Lameter 已提交
5848 5849 5850 5851 5852 5853

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
5854
		if (err)
5855 5856
			pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
			       al->name);
C
Christoph Lameter 已提交
5857 5858 5859
		kfree(al);
	}

5860
	mutex_unlock(&slab_mutex);
C
Christoph Lameter 已提交
5861 5862 5863 5864 5865
	resiliency_test();
	return 0;
}

__initcall(slab_sysfs_init);
5866
#endif /* CONFIG_SYSFS */
5867 5868 5869 5870

/*
 * The /proc/slabinfo ABI
 */
5871
#ifdef CONFIG_SLUB_DEBUG
5872
void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
5873 5874
{
	unsigned long nr_slabs = 0;
5875 5876
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
5877
	int node;
5878
	struct kmem_cache_node *n;
5879

5880
	for_each_kmem_cache_node(s, node, n) {
5881 5882
		nr_slabs += node_nr_slabs(n);
		nr_objs += node_nr_objs(n);
5883
		nr_free += count_partial(n, count_free);
5884 5885
	}

5886 5887 5888 5889 5890 5891
	sinfo->active_objs = nr_objs - nr_free;
	sinfo->num_objs = nr_objs;
	sinfo->active_slabs = nr_slabs;
	sinfo->num_slabs = nr_slabs;
	sinfo->objects_per_slab = oo_objects(s->oo);
	sinfo->cache_order = oo_order(s->oo);
5892 5893
}

5894
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
5895 5896 5897
{
}

5898 5899
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos)
5900
{
5901
	return -EIO;
5902
}
5903
#endif /* CONFIG_SLUB_DEBUG */
新手
引导
客服 返回
顶部