slub.c 152.6 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
C
Christoph Lameter 已提交
2 3 4 5
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
6
 * The allocator synchronizes using per slab locks or atomic operations
7
 * and only uses a centralized lock to manage a pool of partial slabs.
C
Christoph Lameter 已提交
8
 *
C
Christoph Lameter 已提交
9
 * (C) 2007 SGI, Christoph Lameter
10
 * (C) 2011 Linux Foundation, Christoph Lameter
C
Christoph Lameter 已提交
11 12 13
 */

#include <linux/mm.h>
N
Nick Piggin 已提交
14
#include <linux/swap.h> /* struct reclaim_state */
C
Christoph Lameter 已提交
15 16 17
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
18
#include <linux/swab.h>
C
Christoph Lameter 已提交
19 20
#include <linux/bitops.h>
#include <linux/slab.h>
21
#include "slab.h"
22
#include <linux/proc_fs.h>
C
Christoph Lameter 已提交
23
#include <linux/seq_file.h>
24
#include <linux/kasan.h>
C
Christoph Lameter 已提交
25 26 27 28
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
29
#include <linux/stackdepot.h>
30
#include <linux/debugobjects.h>
C
Christoph Lameter 已提交
31
#include <linux/kallsyms.h>
32
#include <linux/kfence.h>
33
#include <linux/memory.h>
R
Roman Zippel 已提交
34
#include <linux/math64.h>
A
Akinobu Mita 已提交
35
#include <linux/fault-inject.h>
36
#include <linux/stacktrace.h>
37
#include <linux/prefetch.h>
38
#include <linux/memcontrol.h>
39
#include <linux/random.h>
40
#include <kunit/test.h>
41
#include <linux/sort.h>
C
Christoph Lameter 已提交
42

43
#include <linux/debugfs.h>
44 45
#include <trace/events/kmem.h>

46 47
#include "internal.h"

C
Christoph Lameter 已提交
48 49
/*
 * Lock order:
50
 *   1. slab_mutex (Global Mutex)
51 52
 *   2. node->list_lock (Spinlock)
 *   3. kmem_cache->cpu_slab->lock (Local lock)
53
 *   4. slab_lock(slab) (Only on some arches or for debugging)
54
 *   5. object_map_lock (Only for debugging)
C
Christoph Lameter 已提交
55
 *
56
 *   slab_mutex
57
 *
58
 *   The role of the slab_mutex is to protect the list of all the slabs
59
 *   and to synchronize major metadata changes to slab cache structures.
60 61 62 63 64 65
 *   Also synchronizes memory hotplug callbacks.
 *
 *   slab_lock
 *
 *   The slab_lock is a wrapper around the page lock, thus it is a bit
 *   spinlock.
66 67
 *
 *   The slab_lock is only used for debugging and on arches that do not
68
 *   have the ability to do a cmpxchg_double. It only protects:
69 70 71 72
 *	A. slab->freelist	-> List of free objects in a slab
 *	B. slab->inuse		-> Number of objects in use
 *	C. slab->objects	-> Number of objects in slab
 *	D. slab->frozen		-> frozen state
73
 *
74 75
 *   Frozen slabs
 *
76
 *   If a slab is frozen then it is exempt from list management. It is not
77
 *   on any list except per cpu partial list. The processor that froze the
78
 *   slab is the one who can perform list operations on the slab. Other
79 80
 *   processors may put objects onto the freelist but the processor that
 *   froze the slab is the only one that can retrieve the objects from the
81
 *   slab's freelist.
C
Christoph Lameter 已提交
82
 *
83 84
 *   list_lock
 *
C
Christoph Lameter 已提交
85 86 87 88 89 90 91 92 93 94 95
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
 *
 *   cpu_slab->lock local lock
 *
 *   This locks protect slowpath manipulation of all kmem_cache_cpu fields
 *   except the stat counters. This is a percpu structure manipulated only by
 *   the local cpu, so the lock protects against being preempted or interrupted
 *   by an irq. Fast path operations rely on lockless operations instead.
 *   On PREEMPT_RT, the local lock does not actually disable irqs (and thus
 *   prevent the lockless operations), so fastpath operations also need to take
 *   the lock and are no longer lockless.
 *
 *   lockless fastpaths
 *
 *   The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
 *   are fully lockless when satisfied from the percpu slab (and when
 *   cmpxchg_double is possible to use, otherwise slab_lock is taken).
 *   They also don't disable preemption or migration or irqs. They rely on
 *   the transaction id (tid) field to detect being preempted or moved to
 *   another cpu.
 *
 *   irq, preemption, migration considerations
 *
 *   Interrupts are disabled as part of list_lock or local_lock operations, or
 *   around the slab_lock operation, in order to make the slab allocator safe
 *   to use in the context of an irq.
 *
 *   In addition, preemption (or migration on PREEMPT_RT) is disabled in the
 *   allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
 *   local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
 *   doesn't have to be revalidated in each section protected by the local lock.
C
Christoph Lameter 已提交
126 127 128 129
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
C
Christoph Lameter 已提交
130 131
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
C
Christoph Lameter 已提交
132
 * freed then the slab will show up again on the partial lists.
C
Christoph Lameter 已提交
133 134
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
C
Christoph Lameter 已提交
135 136 137 138 139
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
140
 * slab->frozen		The slab is frozen and exempt from list processing.
141 142 143 144 145 146 147 148 149 150 151
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
152
 * 			freelist that allows lockless access to
153 154
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
C
Christoph Lameter 已提交
155
 *
Y
Yu Zhao 已提交
156
 * SLAB_DEBUG_FLAGS	Slab requires special handling due to debug
C
Christoph Lameter 已提交
157
 * 			options set. This moves	slab handling out of
158
 * 			the fast path and disables lockless freelists.
C
Christoph Lameter 已提交
159 160
 */

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
/*
 * We could simply use migrate_disable()/enable() but as long as it's a
 * function call even on !PREEMPT_RT, use inline preempt_disable() there.
 */
#ifndef CONFIG_PREEMPT_RT
#define slub_get_cpu_ptr(var)	get_cpu_ptr(var)
#define slub_put_cpu_ptr(var)	put_cpu_ptr(var)
#else
#define slub_get_cpu_ptr(var)		\
({					\
	migrate_disable();		\
	this_cpu_ptr(var);		\
})
#define slub_put_cpu_ptr(var)		\
do {					\
	(void)(var);			\
	migrate_enable();		\
} while (0)
#endif

181 182 183 184 185 186
#ifdef CONFIG_SLUB_DEBUG
#ifdef CONFIG_SLUB_DEBUG_ON
DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
#else
DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
#endif
187
#endif		/* CONFIG_SLUB_DEBUG */
188

189 190 191
static inline bool kmem_cache_debug(struct kmem_cache *s)
{
	return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
192
}
193

194
void *fixup_red_left(struct kmem_cache *s, void *p)
195
{
196
	if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
197 198 199 200 201
		p += s->red_left_pad;

	return p;
}

202 203 204 205 206 207 208 209 210
static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	return !kmem_cache_debug(s);
#else
	return false;
#endif
}

C
Christoph Lameter 已提交
211 212 213 214 215 216 217 218
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

219 220 221
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG

222
/*
223
 * Minimum number of partial slabs. These will be left on the partial
224 225
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
226
#define MIN_PARTIAL 5
C
Christoph Lameter 已提交
227

228 229 230
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
231
 * sort the partial list by the number of objects in use.
232 233 234
 */
#define MAX_PARTIAL 10

235
#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
C
Christoph Lameter 已提交
236
				SLAB_POISON | SLAB_STORE_USER)
C
Christoph Lameter 已提交
237

238 239 240 241 242 243 244 245
/*
 * These debug flags cannot use CMPXCHG because there might be consistency
 * issues when checking or reading debug information
 */
#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)


246
/*
247 248 249
 * Debugging flags that require metadata to be stored in the slab.  These get
 * disabled when slub_debug=O is used and a cache's min order increases with
 * metadata.
250
 */
251
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
252

253 254
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
255
#define MAX_OBJS_PER_PAGE	32767 /* since slab.objects is u15 */
256

C
Christoph Lameter 已提交
257
/* Internal SLUB flags */
258
/* Poison object */
259
#define __OBJECT_POISON		((slab_flags_t __force)0x80000000U)
260
/* Use cmpxchg_double */
261
#define __CMPXCHG_DOUBLE	((slab_flags_t __force)0x40000000U)
C
Christoph Lameter 已提交
262

263 264 265
/*
 * Tracking user of a slab.
 */
266
#define TRACK_ADDRS_COUNT 16
267
struct track {
268
	unsigned long addr;	/* Called from address */
269 270
#ifdef CONFIG_STACKDEPOT
	depot_stack_handle_t handle;
271
#endif
272 273 274 275 276 277 278
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

279
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
280 281 282
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
#else
283 284 285
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
C
Christoph Lameter 已提交
286 287
#endif

288 289 290 291 292 293
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
static void debugfs_slab_add(struct kmem_cache *);
#else
static inline void debugfs_slab_add(struct kmem_cache *s) { }
#endif

294
static inline void stat(const struct kmem_cache *s, enum stat_item si)
295 296
{
#ifdef CONFIG_SLUB_STATS
297 298 299 300 301
	/*
	 * The rmw is racy on a preemptible kernel but this is acceptable, so
	 * avoid this_cpu_add()'s irq-disable overhead.
	 */
	raw_cpu_inc(s->cpu_slab->stat[si]);
302 303 304
#endif
}

305 306 307 308 309 310 311 312
/*
 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
 * differ during memory hotplug/hotremove operations.
 * Protected by slab_mutex.
 */
static nodemask_t slab_nodes;

C
Christoph Lameter 已提交
313 314 315 316
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

317 318 319 320 321 322 323 324 325
/*
 * Returns freelist pointer (ptr). With hardening, this is obfuscated
 * with an XOR of the address where the pointer is held and a per-cache
 * random number.
 */
static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
				 unsigned long ptr_addr)
{
#ifdef CONFIG_SLAB_FREELIST_HARDENED
326
	/*
327
	 * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
328 329 330 331 332 333 334 335 336
	 * Normally, this doesn't cause any issues, as both set_freepointer()
	 * and get_freepointer() are called with a pointer with the same tag.
	 * However, there are some issues with CONFIG_SLUB_DEBUG code. For
	 * example, when __free_slub() iterates over objects in a cache, it
	 * passes untagged pointers to check_object(). check_object() in turns
	 * calls get_freepointer() with an untagged pointer, which causes the
	 * freepointer to be restored incorrectly.
	 */
	return (void *)((unsigned long)ptr ^ s->random ^
337
			swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
338 339 340 341 342 343 344 345 346 347 348 349 350
#else
	return ptr;
#endif
}

/* Returns the freelist pointer recorded at location ptr_addr. */
static inline void *freelist_dereference(const struct kmem_cache *s,
					 void *ptr_addr)
{
	return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
			    (unsigned long)ptr_addr);
}

351 352
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
353
	object = kasan_reset_tag(object);
354
	return freelist_dereference(s, object + s->offset);
355 356
}

357 358
static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{
359
	prefetchw(object + s->offset);
360 361
}

362 363
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
364
	unsigned long freepointer_addr;
365 366
	void *p;

367
	if (!debug_pagealloc_enabled_static())
368 369
		return get_freepointer(s, object);

370
	object = kasan_reset_tag(object);
371
	freepointer_addr = (unsigned long)object + s->offset;
372
	copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
373
	return freelist_ptr(s, p, freepointer_addr);
374 375
}

376 377
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
378 379
	unsigned long freeptr_addr = (unsigned long)object + s->offset;

380 381 382 383
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	BUG_ON(object == fp); /* naive detection of double free or corruption */
#endif

384
	freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
385
	*(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
386 387 388
}

/* Loop over all objects in a slab */
389
#define for_each_object(__p, __s, __addr, __objects) \
390 391 392
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)
393

394
static inline unsigned int order_objects(unsigned int order, unsigned int size)
395
{
396
	return ((unsigned int)PAGE_SIZE << order) / size;
397 398
}

399
static inline struct kmem_cache_order_objects oo_make(unsigned int order,
400
		unsigned int size)
401 402
{
	struct kmem_cache_order_objects x = {
403
		(order << OO_SHIFT) + order_objects(order, size)
404 405 406 407 408
	};

	return x;
}

409
static inline unsigned int oo_order(struct kmem_cache_order_objects x)
410
{
411
	return x.x >> OO_SHIFT;
412 413
}

414
static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
415
{
416
	return x.x & OO_MASK;
417 418
}

419 420 421
#ifdef CONFIG_SLUB_CPU_PARTIAL
static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
{
422
	unsigned int nr_slabs;
423 424 425 426 427

	s->cpu_partial = nr_objects;

	/*
	 * We take the number of objects but actually limit the number of
428 429
	 * slabs on the per cpu partial list, in order to limit excessive
	 * growth of the list. For simplicity we assume that the slabs will
430 431
	 * be half-full.
	 */
432 433
	nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
	s->cpu_partial_slabs = nr_slabs;
434 435 436 437 438 439 440 441
}
#else
static inline void
slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
{
}
#endif /* CONFIG_SLUB_CPU_PARTIAL */

442 443 444
/*
 * Per slab locking using the pagelock
 */
445
static __always_inline void __slab_lock(struct slab *slab)
446
{
447 448
	struct page *page = slab_page(slab);

449
	VM_BUG_ON_PAGE(PageTail(page), page);
450 451 452
	bit_spin_lock(PG_locked, &page->flags);
}

453
static __always_inline void __slab_unlock(struct slab *slab)
454
{
455 456
	struct page *page = slab_page(slab);

457
	VM_BUG_ON_PAGE(PageTail(page), page);
458 459 460
	__bit_spin_unlock(PG_locked, &page->flags);
}

461
static __always_inline void slab_lock(struct slab *slab, unsigned long *flags)
462 463 464
{
	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_save(*flags);
465
	__slab_lock(slab);
466 467
}

468
static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags)
469
{
470
	__slab_unlock(slab);
471 472 473 474 475 476 477 478 479
	if (IS_ENABLED(CONFIG_PREEMPT_RT))
		local_irq_restore(*flags);
}

/*
 * Interrupts must be disabled (for the fallback code to work right), typically
 * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different
 * so we disable interrupts as part of slab_[un]lock().
 */
480
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
481 482 483 484
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
485 486
	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
		lockdep_assert_irqs_disabled();
487 488
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
489
	if (s->flags & __CMPXCHG_DOUBLE) {
490
		if (cmpxchg_double(&slab->freelist, &slab->counters,
491 492
				   freelist_old, counters_old,
				   freelist_new, counters_new))
493
			return true;
494 495 496
	} else
#endif
	{
497 498 499
		/* init to 0 to prevent spurious warnings */
		unsigned long flags = 0;

500 501 502 503 504 505
		slab_lock(slab, &flags);
		if (slab->freelist == freelist_old &&
					slab->counters == counters_old) {
			slab->freelist = freelist_new;
			slab->counters = counters_new;
			slab_unlock(slab, &flags);
506
			return true;
507
		}
508
		slab_unlock(slab, &flags);
509 510 511 512 513 514
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
515
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
516 517
#endif

518
	return false;
519 520
}

521
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,
522 523 524 525
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
526 527
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
528
	if (s->flags & __CMPXCHG_DOUBLE) {
529
		if (cmpxchg_double(&slab->freelist, &slab->counters,
530 531
				   freelist_old, counters_old,
				   freelist_new, counters_new))
532
			return true;
533 534 535
	} else
#endif
	{
536 537 538
		unsigned long flags;

		local_irq_save(flags);
539 540 541 542 543 544
		__slab_lock(slab);
		if (slab->freelist == freelist_old &&
					slab->counters == counters_old) {
			slab->freelist = freelist_new;
			slab->counters = counters_new;
			__slab_unlock(slab);
545
			local_irq_restore(flags);
546
			return true;
547
		}
548
		__slab_unlock(slab);
549
		local_irq_restore(flags);
550 551 552 553 554 555
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
556
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
557 558
#endif

559
	return false;
560 561
}

562
#ifdef CONFIG_SLUB_DEBUG
563
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
564
static DEFINE_RAW_SPINLOCK(object_map_lock);
565

566
static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
567
		       struct slab *slab)
568
{
569
	void *addr = slab_address(slab);
570 571
	void *p;

572
	bitmap_zero(obj_map, slab->objects);
573

574
	for (p = slab->freelist; p; p = get_freepointer(s, p))
575 576 577
		set_bit(__obj_to_index(s, addr, p), obj_map);
}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
#if IS_ENABLED(CONFIG_KUNIT)
static bool slab_add_kunit_errors(void)
{
	struct kunit_resource *resource;

	if (likely(!current->kunit_test))
		return false;

	resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
	if (!resource)
		return false;

	(*(int *)resource->data)++;
	kunit_put_resource(resource);
	return true;
}
#else
static inline bool slab_add_kunit_errors(void) { return false; }
#endif

598
/*
599
 * Determine a map of objects in use in a slab.
600
 *
601
 * Node listlock must be held to guarantee that the slab does
602 603
 * not vanish from under us.
 */
604
static unsigned long *get_map(struct kmem_cache *s, struct slab *slab)
605
	__acquires(&object_map_lock)
606
{
607 608
	VM_BUG_ON(!irqs_disabled());

609
	raw_spin_lock(&object_map_lock);
610

611
	__fill_map(object_map, s, slab);
612 613 614 615

	return object_map;
}

616
static void put_map(unsigned long *map) __releases(&object_map_lock)
617 618
{
	VM_BUG_ON(map != object_map);
619
	raw_spin_unlock(&object_map_lock);
620 621
}

622
static inline unsigned int size_from_object(struct kmem_cache *s)
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
{
	if (s->flags & SLAB_RED_ZONE)
		return s->size - s->red_left_pad;

	return s->size;
}

static inline void *restore_red_left(struct kmem_cache *s, void *p)
{
	if (s->flags & SLAB_RED_ZONE)
		p -= s->red_left_pad;

	return p;
}

638 639 640
/*
 * Debug settings:
 */
641
#if defined(CONFIG_SLUB_DEBUG_ON)
642
static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
643
#else
644
static slab_flags_t slub_debug;
645
#endif
646

647
static char *slub_debug_string;
648
static int disable_higher_order_debug;
649

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
/*
 * slub is about to manipulate internal object metadata.  This memory lies
 * outside the range of the allocated object, so accessing it would normally
 * be reported by kasan as a bounds error.  metadata_access_enable() is used
 * to tell kasan that these accesses are OK.
 */
static inline void metadata_access_enable(void)
{
	kasan_disable_current();
}

static inline void metadata_access_disable(void)
{
	kasan_enable_current();
}

C
Christoph Lameter 已提交
666 667 668
/*
 * Object debugging
 */
669 670 671

/* Verify that a pointer has an address that is valid within a slab page */
static inline int check_valid_pointer(struct kmem_cache *s,
672
				struct slab *slab, void *object)
673 674 675 676 677 678
{
	void *base;

	if (!object)
		return 1;

679
	base = slab_address(slab);
680
	object = kasan_reset_tag(object);
681
	object = restore_red_left(s, object);
682
	if (object < base || object >= base + slab->objects * s->size ||
683 684 685 686 687 688 689
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

690 691
static void print_section(char *level, char *text, u8 *addr,
			  unsigned int length)
C
Christoph Lameter 已提交
692
{
693
	metadata_access_enable();
694 695
	print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
			16, 1, kasan_reset_tag((void *)addr), length, 1);
696
	metadata_access_disable();
C
Christoph Lameter 已提交
697 698
}

699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
/*
 * See comment in calculate_sizes().
 */
static inline bool freeptr_outside_object(struct kmem_cache *s)
{
	return s->offset >= s->inuse;
}

/*
 * Return offset of the end of info block which is inuse + free pointer if
 * not overlapping with object.
 */
static inline unsigned int get_info_end(struct kmem_cache *s)
{
	if (freeptr_outside_object(s))
		return s->inuse + sizeof(void *);
	else
		return s->inuse;
}

C
Christoph Lameter 已提交
719 720 721 722 723
static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

724
	p = object + get_info_end(s);
C
Christoph Lameter 已提交
725

726
	return kasan_reset_tag(p + alloc);
C
Christoph Lameter 已提交
727 728
}

729
#ifdef CONFIG_STACKDEPOT
730 731 732
static noinline depot_stack_handle_t set_track_prepare(void)
{
	depot_stack_handle_t handle;
733
	unsigned long entries[TRACK_ADDRS_COUNT];
734
	unsigned int nr_entries;
735

736
	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
737 738 739 740 741 742 743 744 745
	handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);

	return handle;
}
#else
static inline depot_stack_handle_t set_track_prepare(void)
{
	return 0;
}
746
#endif
747

748 749 750 751 752 753 754 755 756
static void set_track_update(struct kmem_cache *s, void *object,
			     enum track_item alloc, unsigned long addr,
			     depot_stack_handle_t handle)
{
	struct track *p = get_track(s, object, alloc);

#ifdef CONFIG_STACKDEPOT
	p->handle = handle;
#endif
757 758 759 760
	p->addr = addr;
	p->cpu = smp_processor_id();
	p->pid = current->pid;
	p->when = jiffies;
C
Christoph Lameter 已提交
761 762
}

763 764 765 766 767 768 769 770
static __always_inline void set_track(struct kmem_cache *s, void *object,
				      enum track_item alloc, unsigned long addr)
{
	depot_stack_handle_t handle = set_track_prepare();

	set_track_update(s, object, alloc, addr, handle);
}

C
Christoph Lameter 已提交
771 772
static void init_tracking(struct kmem_cache *s, void *object)
{
773 774
	struct track *p;

775 776 777
	if (!(s->flags & SLAB_STORE_USER))
		return;

778 779
	p = get_track(s, object, TRACK_ALLOC);
	memset(p, 0, 2*sizeof(struct track));
C
Christoph Lameter 已提交
780 781
}

782
static void print_track(const char *s, struct track *t, unsigned long pr_time)
C
Christoph Lameter 已提交
783
{
784 785
	depot_stack_handle_t handle __maybe_unused;

C
Christoph Lameter 已提交
786 787 788
	if (!t->addr)
		return;

789
	pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
790
	       s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
791 792 793 794 795 796
#ifdef CONFIG_STACKDEPOT
	handle = READ_ONCE(t->handle);
	if (handle)
		stack_depot_print(handle);
	else
		pr_err("object allocation/free stack trace missing\n");
797
#endif
798 799
}

800
void print_tracking(struct kmem_cache *s, void *object)
801
{
802
	unsigned long pr_time = jiffies;
803 804 805
	if (!(s->flags & SLAB_STORE_USER))
		return;

806 807
	print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
	print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
808 809
}

810
static void print_slab_info(const struct slab *slab)
811
{
812
	struct folio *folio = (struct folio *)slab_folio(slab);
813

814 815 816
	pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
	       slab, slab->objects, slab->inuse, slab->freelist,
	       folio_flags(folio, 0));
817 818 819 820
}

static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
821
	struct va_format vaf;
822 823 824
	va_list args;

	va_start(args, fmt);
825 826
	vaf.fmt = fmt;
	vaf.va = &args;
827
	pr_err("=============================================================================\n");
828
	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
829
	pr_err("-----------------------------------------------------------------------------\n\n");
830
	va_end(args);
C
Christoph Lameter 已提交
831 832
}

833
__printf(2, 3)
834 835
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
836
	struct va_format vaf;
837 838
	va_list args;

839 840 841
	if (slab_add_kunit_errors())
		return;

842
	va_start(args, fmt);
843 844 845
	vaf.fmt = fmt;
	vaf.va = &args;
	pr_err("FIX %s: %pV\n", s->name, &vaf);
846 847 848
	va_end(args);
}

849
static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
C
Christoph Lameter 已提交
850 851
{
	unsigned int off;	/* Offset of last byte */
852
	u8 *addr = slab_address(slab);
853 854 855

	print_tracking(s, p);

856
	print_slab_info(slab);
857

858
	pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
859
	       p, p - addr, get_freepointer(s, p));
860

861
	if (s->flags & SLAB_RED_ZONE)
862
		print_section(KERN_ERR, "Redzone  ", p - s->red_left_pad,
863
			      s->red_left_pad);
864
	else if (p > addr + 16)
865
		print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
C
Christoph Lameter 已提交
866

867
	print_section(KERN_ERR,         "Object   ", p,
868
		      min_t(unsigned int, s->object_size, PAGE_SIZE));
C
Christoph Lameter 已提交
869
	if (s->flags & SLAB_RED_ZONE)
870
		print_section(KERN_ERR, "Redzone  ", p + s->object_size,
871
			s->inuse - s->object_size);
C
Christoph Lameter 已提交
872

873
	off = get_info_end(s);
C
Christoph Lameter 已提交
874

875
	if (s->flags & SLAB_STORE_USER)
C
Christoph Lameter 已提交
876 877
		off += 2 * sizeof(struct track);

878 879
	off += kasan_metadata_size(s);

880
	if (off != size_from_object(s))
C
Christoph Lameter 已提交
881
		/* Beginning of the filler is the free pointer */
882
		print_section(KERN_ERR, "Padding  ", p + off,
883
			      size_from_object(s) - off);
884 885

	dump_stack();
C
Christoph Lameter 已提交
886 887
}

888
static void object_err(struct kmem_cache *s, struct slab *slab,
C
Christoph Lameter 已提交
889 890
			u8 *object, char *reason)
{
891 892 893
	if (slab_add_kunit_errors())
		return;

894
	slab_bug(s, "%s", reason);
895
	print_trailer(s, slab, object);
896
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
C
Christoph Lameter 已提交
897 898
}

899
static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
900 901 902
			       void **freelist, void *nextfree)
{
	if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
903 904
	    !check_valid_pointer(s, slab, nextfree) && freelist) {
		object_err(s, slab, *freelist, "Freechain corrupt");
905 906 907 908 909 910 911 912
		*freelist = NULL;
		slab_fix(s, "Isolate corrupted freechain");
		return true;
	}

	return false;
}

913
static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
914
			const char *fmt, ...)
C
Christoph Lameter 已提交
915 916 917 918
{
	va_list args;
	char buf[100];

919 920 921
	if (slab_add_kunit_errors())
		return;

922 923
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
C
Christoph Lameter 已提交
924
	va_end(args);
925
	slab_bug(s, "%s", buf);
926
	print_slab_info(slab);
C
Christoph Lameter 已提交
927
	dump_stack();
928
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
C
Christoph Lameter 已提交
929 930
}

931
static void init_object(struct kmem_cache *s, void *object, u8 val)
C
Christoph Lameter 已提交
932
{
933
	u8 *p = kasan_reset_tag(object);
C
Christoph Lameter 已提交
934

935 936 937
	if (s->flags & SLAB_RED_ZONE)
		memset(p - s->red_left_pad, val, s->red_left_pad);

C
Christoph Lameter 已提交
938
	if (s->flags & __OBJECT_POISON) {
939 940
		memset(p, POISON_FREE, s->object_size - 1);
		p[s->object_size - 1] = POISON_END;
C
Christoph Lameter 已提交
941 942 943
	}

	if (s->flags & SLAB_RED_ZONE)
944
		memset(p + s->object_size, val, s->inuse - s->object_size);
C
Christoph Lameter 已提交
945 946
}

947 948 949
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
950
	slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
951 952 953
	memset(from, data, to - from);
}

954
static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
955
			u8 *object, char *what,
956
			u8 *start, unsigned int value, unsigned int bytes)
957 958 959
{
	u8 *fault;
	u8 *end;
960
	u8 *addr = slab_address(slab);
961

962
	metadata_access_enable();
963
	fault = memchr_inv(kasan_reset_tag(start), value, bytes);
964
	metadata_access_disable();
965 966 967 968 969 970 971
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

972 973 974
	if (slab_add_kunit_errors())
		goto skip_bug_print;

975
	slab_bug(s, "%s overwritten", what);
976
	pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
977 978
					fault, end - 1, fault - addr,
					fault[0], value);
979
	print_trailer(s, slab, object);
980
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
981

982
skip_bug_print:
983 984
	restore_bytes(s, what, value, fault, end);
	return 0;
C
Christoph Lameter 已提交
985 986 987 988 989 990 991 992
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
993
 *	pointer is at the middle of the object.
C
Christoph Lameter 已提交
994
 *
C
Christoph Lameter 已提交
995 996 997
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
998
 * object + s->object_size
C
Christoph Lameter 已提交
999
 * 	Padding to reach word boundary. This is also used for Redzoning.
C
Christoph Lameter 已提交
1000
 * 	Padding is extended by another word if Redzoning is enabled and
1001
 * 	object_size == inuse.
C
Christoph Lameter 已提交
1002
 *
C
Christoph Lameter 已提交
1003 1004 1005 1006
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
C
Christoph Lameter 已提交
1007 1008
 * 	Meta data starts here.
 *
C
Christoph Lameter 已提交
1009 1010
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
1011
 *	C. Padding to reach required alignment boundary or at minimum
C
Christoph Lameter 已提交
1012
 * 		one word if debugging is on to be able to detect writes
C
Christoph Lameter 已提交
1013 1014 1015
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
C
Christoph Lameter 已提交
1016 1017
 *
 * object + s->size
C
Christoph Lameter 已提交
1018
 * 	Nothing is used beyond s->size.
C
Christoph Lameter 已提交
1019
 *
1020
 * If slabcaches are merged then the object_size and inuse boundaries are mostly
C
Christoph Lameter 已提交
1021
 * ignored. And therefore no slab options that rely on these boundaries
C
Christoph Lameter 已提交
1022 1023 1024
 * may be used with merged slabcaches.
 */

1025
static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
C
Christoph Lameter 已提交
1026
{
1027
	unsigned long off = get_info_end(s);	/* The end of info */
C
Christoph Lameter 已提交
1028 1029 1030 1031 1032

	if (s->flags & SLAB_STORE_USER)
		/* We also have user information there */
		off += 2 * sizeof(struct track);

1033 1034
	off += kasan_metadata_size(s);

1035
	if (size_from_object(s) == off)
C
Christoph Lameter 已提交
1036 1037
		return 1;

1038
	return check_bytes_and_report(s, slab, p, "Object padding",
1039
			p + off, POISON_INUSE, size_from_object(s) - off);
C
Christoph Lameter 已提交
1040 1041
}

1042
/* Check the pad bytes at the end of a slab page */
1043
static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
C
Christoph Lameter 已提交
1044
{
1045 1046 1047
	u8 *start;
	u8 *fault;
	u8 *end;
1048
	u8 *pad;
1049 1050
	int length;
	int remainder;
C
Christoph Lameter 已提交
1051 1052

	if (!(s->flags & SLAB_POISON))
1053
		return;
C
Christoph Lameter 已提交
1054

1055 1056
	start = slab_address(slab);
	length = slab_size(slab);
1057 1058
	end = start + length;
	remainder = length % s->size;
C
Christoph Lameter 已提交
1059
	if (!remainder)
1060
		return;
C
Christoph Lameter 已提交
1061

1062
	pad = end - remainder;
1063
	metadata_access_enable();
1064
	fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
1065
	metadata_access_disable();
1066
	if (!fault)
1067
		return;
1068 1069 1070
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

1071
	slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1072
			fault, end - 1, fault - start);
1073
	print_section(KERN_ERR, "Padding ", pad, remainder);
1074

1075
	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
C
Christoph Lameter 已提交
1076 1077
}

1078
static int check_object(struct kmem_cache *s, struct slab *slab,
1079
					void *object, u8 val)
C
Christoph Lameter 已提交
1080 1081
{
	u8 *p = object;
1082
	u8 *endobject = object + s->object_size;
C
Christoph Lameter 已提交
1083 1084

	if (s->flags & SLAB_RED_ZONE) {
1085
		if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1086 1087 1088
			object - s->red_left_pad, val, s->red_left_pad))
			return 0;

1089
		if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1090
			endobject, val, s->inuse - s->object_size))
C
Christoph Lameter 已提交
1091 1092
			return 0;
	} else {
1093
		if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1094
			check_bytes_and_report(s, slab, p, "Alignment padding",
1095 1096
				endobject, POISON_INUSE,
				s->inuse - s->object_size);
1097
		}
C
Christoph Lameter 已提交
1098 1099 1100
	}

	if (s->flags & SLAB_POISON) {
1101
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
1102
			(!check_bytes_and_report(s, slab, p, "Poison", p,
1103
					POISON_FREE, s->object_size - 1) ||
1104
			 !check_bytes_and_report(s, slab, p, "End Poison",
1105
				p + s->object_size - 1, POISON_END, 1)))
C
Christoph Lameter 已提交
1106 1107 1108 1109
			return 0;
		/*
		 * check_pad_bytes cleans up on its own.
		 */
1110
		check_pad_bytes(s, slab, p);
C
Christoph Lameter 已提交
1111 1112
	}

1113
	if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
C
Christoph Lameter 已提交
1114 1115 1116 1117 1118 1119 1120
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
1121 1122
	if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
		object_err(s, slab, p, "Freepointer corrupt");
C
Christoph Lameter 已提交
1123
		/*
1124
		 * No choice but to zap it and thus lose the remainder
C
Christoph Lameter 已提交
1125
		 * of the free objects in this slab. May cause
C
Christoph Lameter 已提交
1126
		 * another error because the object count is now wrong.
C
Christoph Lameter 已提交
1127
		 */
1128
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
1129 1130 1131 1132 1133
		return 0;
	}
	return 1;
}

1134
static int check_slab(struct kmem_cache *s, struct slab *slab)
C
Christoph Lameter 已提交
1135
{
1136 1137
	int maxobj;

1138 1139
	if (!folio_test_slab(slab_folio(slab))) {
		slab_err(s, slab, "Not a valid slab page");
C
Christoph Lameter 已提交
1140 1141
		return 0;
	}
1142

1143 1144 1145 1146
	maxobj = order_objects(slab_order(slab), s->size);
	if (slab->objects > maxobj) {
		slab_err(s, slab, "objects %u > max %u",
			slab->objects, maxobj);
1147 1148
		return 0;
	}
1149 1150 1151
	if (slab->inuse > slab->objects) {
		slab_err(s, slab, "inuse %u > max %u",
			slab->inuse, slab->objects);
C
Christoph Lameter 已提交
1152 1153 1154
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
1155
	slab_pad_check(s, slab);
C
Christoph Lameter 已提交
1156 1157 1158 1159
	return 1;
}

/*
1160
 * Determine if a certain object in a slab is on the freelist. Must hold the
C
Christoph Lameter 已提交
1161
 * slab lock to guarantee that the chains are in a consistent state.
C
Christoph Lameter 已提交
1162
 */
1163
static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
C
Christoph Lameter 已提交
1164 1165
{
	int nr = 0;
1166
	void *fp;
C
Christoph Lameter 已提交
1167
	void *object = NULL;
1168
	int max_objects;
C
Christoph Lameter 已提交
1169

1170 1171
	fp = slab->freelist;
	while (fp && nr <= slab->objects) {
C
Christoph Lameter 已提交
1172 1173
		if (fp == search)
			return 1;
1174
		if (!check_valid_pointer(s, slab, fp)) {
C
Christoph Lameter 已提交
1175
			if (object) {
1176
				object_err(s, slab, object,
C
Christoph Lameter 已提交
1177
					"Freechain corrupt");
1178
				set_freepointer(s, object, NULL);
C
Christoph Lameter 已提交
1179
			} else {
1180 1181 1182
				slab_err(s, slab, "Freepointer corrupt");
				slab->freelist = NULL;
				slab->inuse = slab->objects;
1183
				slab_fix(s, "Freelist cleared");
C
Christoph Lameter 已提交
1184 1185 1186 1187 1188 1189 1190 1191 1192
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

1193
	max_objects = order_objects(slab_order(slab), s->size);
1194 1195
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
1196

1197 1198 1199 1200
	if (slab->objects != max_objects) {
		slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
			 slab->objects, max_objects);
		slab->objects = max_objects;
1201
		slab_fix(s, "Number of objects adjusted");
1202
	}
1203 1204 1205 1206
	if (slab->inuse != slab->objects - nr) {
		slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
			 slab->inuse, slab->objects - nr);
		slab->inuse = slab->objects - nr;
1207
		slab_fix(s, "Object count adjusted");
C
Christoph Lameter 已提交
1208 1209 1210 1211
	}
	return search == NULL;
}

1212
static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1213
								int alloc)
1214 1215
{
	if (s->flags & SLAB_TRACE) {
1216
		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1217 1218
			s->name,
			alloc ? "alloc" : "free",
1219 1220
			object, slab->inuse,
			slab->freelist);
1221 1222

		if (!alloc)
1223
			print_section(KERN_INFO, "Object ", (void *)object,
1224
					s->object_size);
1225 1226 1227 1228 1229

		dump_stack();
	}
}

1230
/*
C
Christoph Lameter 已提交
1231
 * Tracking of fully allocated slabs for debugging purposes.
1232
 */
1233
static void add_full(struct kmem_cache *s,
1234
	struct kmem_cache_node *n, struct slab *slab)
1235
{
1236 1237 1238
	if (!(s->flags & SLAB_STORE_USER))
		return;

1239
	lockdep_assert_held(&n->list_lock);
1240
	list_add(&slab->slab_list, &n->full);
1241 1242
}

1243
static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1244 1245 1246 1247
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

1248
	lockdep_assert_held(&n->list_lock);
1249
	list_del(&slab->slab_list);
1250 1251
}

1252 1253 1254 1255 1256 1257 1258 1259
/* Tracking of the number of slabs for debugging purposes */
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
{
	struct kmem_cache_node *n = get_node(s, node);

	return atomic_long_read(&n->nr_slabs);
}

1260 1261 1262 1263 1264
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

1265
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1266 1267 1268 1269 1270 1271 1272 1273 1274
{
	struct kmem_cache_node *n = get_node(s, node);

	/*
	 * May be called early in order to allocate a slab for the
	 * kmem_cache_node structure. Solve the chicken-egg
	 * dilemma by deferring the increment of the count during
	 * bootstrap (see early_kmem_cache_node_alloc).
	 */
1275
	if (likely(n)) {
1276
		atomic_long_inc(&n->nr_slabs);
1277 1278
		atomic_long_add(objects, &n->total_objects);
	}
1279
}
1280
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1281 1282 1283 1284
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
1285
	atomic_long_sub(objects, &n->total_objects);
1286 1287 1288
}

/* Object debug checks for alloc/free paths */
1289
static void setup_object_debug(struct kmem_cache *s, void *object)
1290
{
1291
	if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1292 1293
		return;

1294
	init_object(s, object, SLUB_RED_INACTIVE);
1295 1296 1297
	init_tracking(s, object);
}

1298
static
1299
void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1300
{
1301
	if (!kmem_cache_debug_flags(s, SLAB_POISON))
1302 1303 1304
		return;

	metadata_access_enable();
1305
	memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1306 1307 1308
	metadata_access_disable();
}

1309
static inline int alloc_consistency_checks(struct kmem_cache *s,
1310
					struct slab *slab, void *object)
C
Christoph Lameter 已提交
1311
{
1312
	if (!check_slab(s, slab))
1313
		return 0;
C
Christoph Lameter 已提交
1314

1315 1316
	if (!check_valid_pointer(s, slab, object)) {
		object_err(s, slab, object, "Freelist Pointer check fails");
1317
		return 0;
C
Christoph Lameter 已提交
1318 1319
	}

1320
	if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1321 1322 1323 1324 1325 1326
		return 0;

	return 1;
}

static noinline int alloc_debug_processing(struct kmem_cache *s,
1327
					struct slab *slab,
1328 1329 1330
					void *object, unsigned long addr)
{
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1331
		if (!alloc_consistency_checks(s, slab, object))
1332 1333
			goto bad;
	}
C
Christoph Lameter 已提交
1334

1335 1336 1337
	/* Success perform special debug activities for allocs */
	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_ALLOC, addr);
1338
	trace(s, slab, object, 1);
1339
	init_object(s, object, SLUB_RED_ACTIVE);
C
Christoph Lameter 已提交
1340
	return 1;
1341

C
Christoph Lameter 已提交
1342
bad:
1343
	if (folio_test_slab(slab_folio(slab))) {
C
Christoph Lameter 已提交
1344 1345 1346
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
C
Christoph Lameter 已提交
1347
		 * as used avoids touching the remaining objects.
C
Christoph Lameter 已提交
1348
		 */
1349
		slab_fix(s, "Marking all objects used");
1350 1351
		slab->inuse = slab->objects;
		slab->freelist = NULL;
C
Christoph Lameter 已提交
1352 1353 1354 1355
	}
	return 0;
}

1356
static inline int free_consistency_checks(struct kmem_cache *s,
1357
		struct slab *slab, void *object, unsigned long addr)
C
Christoph Lameter 已提交
1358
{
1359 1360
	if (!check_valid_pointer(s, slab, object)) {
		slab_err(s, slab, "Invalid object pointer 0x%p", object);
1361
		return 0;
C
Christoph Lameter 已提交
1362 1363
	}

1364 1365
	if (on_freelist(s, slab, object)) {
		object_err(s, slab, object, "Object already free");
1366
		return 0;
C
Christoph Lameter 已提交
1367 1368
	}

1369
	if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1370
		return 0;
C
Christoph Lameter 已提交
1371

1372 1373 1374
	if (unlikely(s != slab->slab_cache)) {
		if (!folio_test_slab(slab_folio(slab))) {
			slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
1375
				 object);
1376
		} else if (!slab->slab_cache) {
1377 1378
			pr_err("SLUB <none>: no slab for object 0x%p.\n",
			       object);
1379
			dump_stack();
1380
		} else
1381
			object_err(s, slab, object,
1382
					"page slab pointer corrupt.");
1383 1384 1385 1386 1387 1388 1389
		return 0;
	}
	return 1;
}

/* Supports checking bulk free of a constructed freelist */
static noinline int free_debug_processing(
1390
	struct kmem_cache *s, struct slab *slab,
1391 1392 1393
	void *head, void *tail, int bulk_cnt,
	unsigned long addr)
{
1394
	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
1395 1396
	void *object = head;
	int cnt = 0;
1397
	unsigned long flags, flags2;
1398
	int ret = 0;
1399 1400 1401 1402
	depot_stack_handle_t handle = 0;

	if (s->flags & SLAB_STORE_USER)
		handle = set_track_prepare();
1403 1404

	spin_lock_irqsave(&n->list_lock, flags);
1405
	slab_lock(slab, &flags2);
1406 1407

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1408
		if (!check_slab(s, slab))
1409 1410 1411 1412 1413 1414 1415
			goto out;
	}

next_object:
	cnt++;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1416
		if (!free_consistency_checks(s, slab, object, addr))
1417
			goto out;
C
Christoph Lameter 已提交
1418
	}
1419 1420

	if (s->flags & SLAB_STORE_USER)
1421
		set_track_update(s, object, TRACK_FREE, addr, handle);
1422
	trace(s, slab, object, 0);
1423
	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
1424
	init_object(s, object, SLUB_RED_INACTIVE);
1425 1426 1427 1428 1429 1430

	/* Reached end of constructed freelist yet? */
	if (object != tail) {
		object = get_freepointer(s, object);
		goto next_object;
	}
1431 1432
	ret = 1;

1433
out:
1434
	if (cnt != bulk_cnt)
1435
		slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n",
1436 1437
			 bulk_cnt, cnt);

1438
	slab_unlock(slab, &flags2);
1439
	spin_unlock_irqrestore(&n->list_lock, flags);
1440 1441 1442
	if (!ret)
		slab_fix(s, "Object at 0x%p not freed", object);
	return ret;
C
Christoph Lameter 已提交
1443 1444
}

1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
/*
 * Parse a block of slub_debug options. Blocks are delimited by ';'
 *
 * @str:    start of block
 * @flags:  returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
 * @slabs:  return start of list of slabs, or NULL when there's no list
 * @init:   assume this is initial parsing and not per-kmem-create parsing
 *
 * returns the start of next block if there's any, or NULL
 */
static char *
parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1457
{
1458
	bool higher_order_disable = false;
1459

1460 1461 1462 1463 1464
	/* Skip any completely empty blocks */
	while (*str && *str == ';')
		str++;

	if (*str == ',') {
1465 1466 1467 1468
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
1469
		*flags = DEBUG_DEFAULT_FLAGS;
1470
		goto check_slabs;
1471 1472
	}
	*flags = 0;
1473

1474 1475
	/* Determine which debug features should be switched on */
	for (; *str && *str != ',' && *str != ';'; str++) {
1476
		switch (tolower(*str)) {
1477 1478 1479
		case '-':
			*flags = 0;
			break;
1480
		case 'f':
1481
			*flags |= SLAB_CONSISTENCY_CHECKS;
1482 1483
			break;
		case 'z':
1484
			*flags |= SLAB_RED_ZONE;
1485 1486
			break;
		case 'p':
1487
			*flags |= SLAB_POISON;
1488 1489
			break;
		case 'u':
1490
			*flags |= SLAB_STORE_USER;
1491 1492
			break;
		case 't':
1493
			*flags |= SLAB_TRACE;
1494
			break;
1495
		case 'a':
1496
			*flags |= SLAB_FAILSLAB;
1497
			break;
1498 1499 1500 1501 1502
		case 'o':
			/*
			 * Avoid enabling debugging on caches if its minimum
			 * order would increase as a result.
			 */
1503
			higher_order_disable = true;
1504
			break;
1505
		default:
1506 1507
			if (init)
				pr_err("slub_debug option '%c' unknown. skipped\n", *str);
1508
		}
1509
	}
1510
check_slabs:
1511
	if (*str == ',')
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
		*slabs = ++str;
	else
		*slabs = NULL;

	/* Skip over the slab list */
	while (*str && *str != ';')
		str++;

	/* Skip any completely empty blocks */
	while (*str && *str == ';')
		str++;

	if (init && higher_order_disable)
		disable_higher_order_debug = 1;

	if (*str)
		return str;
	else
		return NULL;
}

static int __init setup_slub_debug(char *str)
{
	slab_flags_t flags;
1536
	slab_flags_t global_flags;
1537 1538 1539 1540 1541
	char *saved_str;
	char *slab_list;
	bool global_slub_debug_changed = false;
	bool slab_list_specified = false;

1542
	global_flags = DEBUG_DEFAULT_FLAGS;
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	saved_str = str;
	while (str) {
		str = parse_slub_debug_flags(str, &flags, &slab_list, true);

		if (!slab_list) {
1554
			global_flags = flags;
1555 1556 1557
			global_slub_debug_changed = true;
		} else {
			slab_list_specified = true;
1558 1559
			if (flags & SLAB_STORE_USER)
				stack_depot_want_early_init();
1560 1561 1562 1563 1564
		}
	}

	/*
	 * For backwards compatibility, a single list of flags with list of
1565 1566 1567
	 * slabs means debugging is only changed for those slabs, so the global
	 * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
	 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
1568 1569 1570 1571
	 * long as there is no option specifying flags without a slab list.
	 */
	if (slab_list_specified) {
		if (!global_slub_debug_changed)
1572
			global_flags = slub_debug;
1573 1574
		slub_debug_string = saved_str;
	}
1575
out:
1576
	slub_debug = global_flags;
1577 1578
	if (slub_debug & SLAB_STORE_USER)
		stack_depot_want_early_init();
1579 1580
	if (slub_debug != 0 || slub_debug_string)
		static_branch_enable(&slub_debug_enabled);
1581 1582
	else
		static_branch_disable(&slub_debug_enabled);
1583 1584 1585 1586
	if ((static_branch_unlikely(&init_on_alloc) ||
	     static_branch_unlikely(&init_on_free)) &&
	    (slub_debug & SLAB_POISON))
		pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
1587 1588 1589 1590 1591
	return 1;
}

__setup("slub_debug", setup_slub_debug);

1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
/*
 * kmem_cache_flags - apply debugging options to the cache
 * @object_size:	the size of an object without meta data
 * @flags:		flags to set
 * @name:		name of the cache
 *
 * Debug option(s) are applied to @flags. In addition to the debug
 * option(s), if a slab name (or multiple) is specified i.e.
 * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
 * then only the select slabs will receive the debug option(s).
 */
1603
slab_flags_t kmem_cache_flags(unsigned int object_size,
1604
	slab_flags_t flags, const char *name)
1605
{
1606 1607
	char *iter;
	size_t len;
1608 1609
	char *next_block;
	slab_flags_t block_flags;
1610 1611
	slab_flags_t slub_debug_local = slub_debug;

1612 1613 1614
	if (flags & SLAB_NO_USER_FLAGS)
		return flags;

1615 1616 1617 1618 1619 1620 1621
	/*
	 * If the slab cache is for debugging (e.g. kmemleak) then
	 * don't store user (stack trace) information by default,
	 * but let the user enable it via the command line below.
	 */
	if (flags & SLAB_NOLEAKTRACE)
		slub_debug_local &= ~SLAB_STORE_USER;
1622 1623

	len = strlen(name);
1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
	next_block = slub_debug_string;
	/* Go through all blocks of debug options, see if any matches our slab's name */
	while (next_block) {
		next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
		if (!iter)
			continue;
		/* Found a block that has a slab list, search it */
		while (*iter) {
			char *end, *glob;
			size_t cmplen;

			end = strchrnul(iter, ',');
			if (next_block && next_block < end)
				end = next_block - 1;

			glob = strnchr(iter, end - iter, '*');
			if (glob)
				cmplen = glob - iter;
			else
				cmplen = max_t(size_t, len, (end - iter));
1644

1645 1646 1647 1648
			if (!strncmp(name, iter, cmplen)) {
				flags |= block_flags;
				return flags;
			}
1649

1650 1651 1652
			if (!*end || *end == ';')
				break;
			iter = end + 1;
1653 1654
		}
	}
1655

1656
	return flags | slub_debug_local;
1657
}
1658
#else /* !CONFIG_SLUB_DEBUG */
1659
static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
1660
static inline
1661
void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1662

1663
static inline int alloc_debug_processing(struct kmem_cache *s,
1664
	struct slab *slab, void *object, unsigned long addr) { return 0; }
1665

1666
static inline int free_debug_processing(
1667
	struct kmem_cache *s, struct slab *slab,
1668
	void *head, void *tail, int bulk_cnt,
1669
	unsigned long addr) { return 0; }
1670

1671
static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
1672
static inline int check_object(struct kmem_cache *s, struct slab *slab,
1673
			void *object, u8 val) { return 1; }
1674
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1675
					struct slab *slab) {}
1676
static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1677
					struct slab *slab) {}
1678
slab_flags_t kmem_cache_flags(unsigned int object_size,
1679
	slab_flags_t flags, const char *name)
1680 1681 1682
{
	return flags;
}
1683
#define slub_debug 0
1684

1685 1686
#define disable_higher_order_debug 0

1687 1688
static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }
1689 1690
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1691 1692 1693 1694
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1695

1696
static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1697
			       void **freelist, void *nextfree)
1698 1699 1700
{
	return false;
}
1701 1702 1703 1704 1705 1706
#endif /* CONFIG_SLUB_DEBUG */

/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
 */
1707 1708
static __always_inline bool slab_free_hook(struct kmem_cache *s,
						void *x, bool init)
1709 1710
{
	kmemleak_free_recursive(x, s->flags);
1711

1712
	debug_check_no_locks_freed(x, s->object_size);
1713 1714 1715

	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->object_size);
1716

1717 1718 1719 1720 1721
	/* Use KCSAN to help debug racy use-after-free. */
	if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
		__kcsan_check_access(x, s->object_size,
				     KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);

1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
	/*
	 * As memory initialization might be integrated into KASAN,
	 * kasan_slab_free and initialization memset's must be
	 * kept together to avoid discrepancies in behavior.
	 *
	 * The initialization memset's clear the object and the metadata,
	 * but don't touch the SLAB redzone.
	 */
	if (init) {
		int rsize;

		if (!kasan_has_integrated_init())
			memset(kasan_reset_tag(x), 0, s->object_size);
		rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
		memset((char *)kasan_reset_tag(x) + s->inuse, 0,
		       s->size - s->inuse - rsize);
	}
	/* KASAN might put x into memory quarantine, delaying its reuse. */
	return kasan_slab_free(s, x, init);
1741
}
1742

1743
static inline bool slab_free_freelist_hook(struct kmem_cache *s,
1744 1745
					   void **head, void **tail,
					   int *cnt)
1746
{
1747 1748 1749 1750 1751

	void *object;
	void *next = *head;
	void *old_tail = *tail ? *tail : *head;

1752
	if (is_kfence_address(next)) {
1753
		slab_free_hook(s, next, false);
1754 1755 1756
		return true;
	}

1757 1758 1759
	/* Head and tail of the reconstructed freelist */
	*head = NULL;
	*tail = NULL;
1760

1761 1762 1763 1764
	do {
		object = next;
		next = get_freepointer(s, object);

1765
		/* If object's reuse doesn't have to be delayed */
1766
		if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
1767 1768 1769 1770 1771
			/* Move object to the new freelist */
			set_freepointer(s, object, *head);
			*head = object;
			if (!*tail)
				*tail = object;
1772 1773 1774 1775 1776 1777
		} else {
			/*
			 * Adjust the reconstructed freelist depth
			 * accordingly if object's reuse is delayed.
			 */
			--(*cnt);
1778 1779 1780 1781 1782 1783 1784
		}
	} while (object != old_tail);

	if (*head == *tail)
		*tail = NULL;

	return *head != NULL;
1785 1786
}

1787
static void *setup_object(struct kmem_cache *s, void *object)
1788
{
1789
	setup_object_debug(s, object);
1790
	object = kasan_init_slab_obj(s, object);
1791 1792 1793 1794 1795
	if (unlikely(s->ctor)) {
		kasan_unpoison_object_data(s, object);
		s->ctor(object);
		kasan_poison_object_data(s, object);
	}
1796
	return object;
1797 1798
}

C
Christoph Lameter 已提交
1799 1800 1801
/*
 * Slab allocation and freeing
 */
1802 1803
static inline struct slab *alloc_slab_page(gfp_t flags, int node,
		struct kmem_cache_order_objects oo)
1804
{
1805 1806
	struct folio *folio;
	struct slab *slab;
1807
	unsigned int order = oo_order(oo);
1808

1809
	if (node == NUMA_NO_NODE)
1810
		folio = (struct folio *)alloc_pages(flags, order);
1811
	else
1812
		folio = (struct folio *)__alloc_pages_node(node, flags, order);
1813

1814 1815 1816 1817 1818 1819 1820 1821 1822
	if (!folio)
		return NULL;

	slab = folio_slab(folio);
	__folio_set_slab(folio);
	if (page_is_pfmemalloc(folio_page(folio, 0)))
		slab_set_pfmemalloc(slab);

	return slab;
1823 1824
}

1825 1826 1827 1828
#ifdef CONFIG_SLAB_FREELIST_RANDOM
/* Pre-initialize the random sequence cache */
static int init_cache_random_seq(struct kmem_cache *s)
{
1829
	unsigned int count = oo_objects(s->oo);
1830 1831
	int err;

1832 1833 1834 1835
	/* Bailout if already initialised */
	if (s->random_seq)
		return 0;

1836 1837 1838 1839 1840 1841 1842 1843 1844
	err = cache_random_seq_create(s, count, GFP_KERNEL);
	if (err) {
		pr_err("SLUB: Unable to initialize free list for %s\n",
			s->name);
		return err;
	}

	/* Transform to an offset on the set of pages */
	if (s->random_seq) {
1845 1846
		unsigned int i;

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
		for (i = 0; i < count; i++)
			s->random_seq[i] *= s->size;
	}
	return 0;
}

/* Initialize each random sequence freelist per cache */
static void __init init_freelist_randomization(void)
{
	struct kmem_cache *s;

	mutex_lock(&slab_mutex);

	list_for_each_entry(s, &slab_caches, list)
		init_cache_random_seq(s);

	mutex_unlock(&slab_mutex);
}

/* Get the next entry on the pre-computed freelist randomized */
1867
static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab,
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
				unsigned long *pos, void *start,
				unsigned long page_limit,
				unsigned long freelist_count)
{
	unsigned int idx;

	/*
	 * If the target page allocation failed, the number of objects on the
	 * page might be smaller than the usual size defined by the cache.
	 */
	do {
		idx = s->random_seq[*pos];
		*pos += 1;
		if (*pos >= freelist_count)
			*pos = 0;
	} while (unlikely(idx >= page_limit));

	return (char *)start + idx;
}

/* Shuffle the single linked freelist based on a random pre-computed sequence */
1889
static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
1890 1891 1892 1893 1894 1895
{
	void *start;
	void *cur;
	void *next;
	unsigned long idx, pos, page_limit, freelist_count;

1896
	if (slab->objects < 2 || !s->random_seq)
1897 1898 1899 1900 1901
		return false;

	freelist_count = oo_objects(s->oo);
	pos = get_random_int() % freelist_count;

1902 1903
	page_limit = slab->objects * s->size;
	start = fixup_red_left(s, slab_address(slab));
1904 1905

	/* First entry is used as the base of the freelist */
1906
	cur = next_freelist_entry(s, slab, &pos, start, page_limit,
1907
				freelist_count);
1908
	cur = setup_object(s, cur);
1909
	slab->freelist = cur;
1910

1911 1912
	for (idx = 1; idx < slab->objects; idx++) {
		next = next_freelist_entry(s, slab, &pos, start, page_limit,
1913
			freelist_count);
1914
		next = setup_object(s, next);
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
		set_freepointer(s, cur, next);
		cur = next;
	}
	set_freepointer(s, cur, NULL);

	return true;
}
#else
static inline int init_cache_random_seq(struct kmem_cache *s)
{
	return 0;
}
static inline void init_freelist_randomization(void) { }
1928
static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
1929 1930 1931 1932 1933
{
	return false;
}
#endif /* CONFIG_SLAB_FREELIST_RANDOM */

1934
static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
C
Christoph Lameter 已提交
1935
{
1936
	struct slab *slab;
1937
	struct kmem_cache_order_objects oo = s->oo;
1938
	gfp_t alloc_gfp;
1939
	void *start, *p, *next;
1940
	int idx;
1941
	bool shuffle;
C
Christoph Lameter 已提交
1942

1943 1944
	flags &= gfp_allowed_mask;

1945
	flags |= s->allocflags;
1946

1947 1948 1949 1950 1951
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1952
	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1953
		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
1954

1955
	slab = alloc_slab_page(alloc_gfp, node, oo);
1956
	if (unlikely(!slab)) {
1957
		oo = s->min;
1958
		alloc_gfp = flags;
1959 1960 1961 1962
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
1963
		slab = alloc_slab_page(alloc_gfp, node, oo);
1964
		if (unlikely(!slab))
1965 1966
			goto out;
		stat(s, ORDER_FALLBACK);
1967
	}
1968

1969
	slab->objects = oo_objects(oo);
C
Christoph Lameter 已提交
1970

1971
	account_slab(slab, oo_order(oo), s, flags);
1972

1973
	slab->slab_cache = s;
C
Christoph Lameter 已提交
1974

1975
	kasan_poison_slab(slab);
C
Christoph Lameter 已提交
1976

1977
	start = slab_address(slab);
C
Christoph Lameter 已提交
1978

1979
	setup_slab_debug(s, slab, start);
1980

1981
	shuffle = shuffle_freelist(s, slab);
1982 1983

	if (!shuffle) {
1984
		start = fixup_red_left(s, start);
1985
		start = setup_object(s, start);
1986 1987
		slab->freelist = start;
		for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
1988
			next = p + s->size;
1989
			next = setup_object(s, next);
1990 1991 1992 1993
			set_freepointer(s, p, next);
			p = next;
		}
		set_freepointer(s, p, NULL);
C
Christoph Lameter 已提交
1994 1995
	}

1996 1997
	slab->inuse = slab->objects;
	slab->frozen = 1;
1998

C
Christoph Lameter 已提交
1999
out:
2000
	if (!slab)
2001 2002
		return NULL;

2003
	inc_slabs_node(s, slab_nid(slab), slab->objects);
2004

2005
	return slab;
C
Christoph Lameter 已提交
2006 2007
}

2008
static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
2009
{
2010 2011
	if (unlikely(flags & GFP_SLAB_BUG_MASK))
		flags = kmalloc_fix_flags(flags);
2012

2013 2014
	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));

2015 2016 2017 2018
	return allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
}

2019
static void __free_slab(struct kmem_cache *s, struct slab *slab)
C
Christoph Lameter 已提交
2020
{
2021 2022
	struct folio *folio = slab_folio(slab);
	int order = folio_order(folio);
2023
	int pages = 1 << order;
C
Christoph Lameter 已提交
2024

2025
	if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
C
Christoph Lameter 已提交
2026 2027
		void *p;

2028
		slab_pad_check(s, slab);
2029
		for_each_object(p, s, slab_address(slab), slab->objects)
2030
			check_object(s, slab, p, SLUB_RED_INACTIVE);
C
Christoph Lameter 已提交
2031 2032
	}

2033 2034 2035
	__slab_clear_pfmemalloc(slab);
	__folio_clear_slab(folio);
	folio->mapping = NULL;
N
Nick Piggin 已提交
2036 2037
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
2038 2039
	unaccount_slab(slab, order, s);
	__free_pages(folio_page(folio, 0), order);
C
Christoph Lameter 已提交
2040 2041 2042 2043
}

static void rcu_free_slab(struct rcu_head *h)
{
2044
	struct slab *slab = container_of(h, struct slab, rcu_head);
2045

2046
	__free_slab(slab->slab_cache, slab);
C
Christoph Lameter 已提交
2047 2048
}

2049
static void free_slab(struct kmem_cache *s, struct slab *slab)
C
Christoph Lameter 已提交
2050
{
2051
	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
2052
		call_rcu(&slab->rcu_head, rcu_free_slab);
C
Christoph Lameter 已提交
2053
	} else
2054
		__free_slab(s, slab);
C
Christoph Lameter 已提交
2055 2056
}

2057
static void discard_slab(struct kmem_cache *s, struct slab *slab)
C
Christoph Lameter 已提交
2058
{
2059 2060
	dec_slabs_node(s, slab_nid(slab), slab->objects);
	free_slab(s, slab);
C
Christoph Lameter 已提交
2061 2062 2063
}

/*
2064
 * Management of partially allocated slabs.
C
Christoph Lameter 已提交
2065
 */
2066
static inline void
2067
__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
C
Christoph Lameter 已提交
2068
{
C
Christoph Lameter 已提交
2069
	n->nr_partial++;
2070
	if (tail == DEACTIVATE_TO_TAIL)
2071
		list_add_tail(&slab->slab_list, &n->partial);
2072
	else
2073
		list_add(&slab->slab_list, &n->partial);
C
Christoph Lameter 已提交
2074 2075
}

2076
static inline void add_partial(struct kmem_cache_node *n,
2077
				struct slab *slab, int tail)
2078
{
2079
	lockdep_assert_held(&n->list_lock);
2080
	__add_partial(n, slab, tail);
2081
}
2082

2083
static inline void remove_partial(struct kmem_cache_node *n,
2084
					struct slab *slab)
2085 2086
{
	lockdep_assert_held(&n->list_lock);
2087
	list_del(&slab->slab_list);
2088
	n->nr_partial--;
2089 2090
}

C
Christoph Lameter 已提交
2091
/*
2092 2093
 * Remove slab from the partial list, freeze it and
 * return the pointer to the freelist.
C
Christoph Lameter 已提交
2094
 *
2095
 * Returns a list of objects or NULL if it fails.
C
Christoph Lameter 已提交
2096
 */
2097
static inline void *acquire_slab(struct kmem_cache *s,
2098
		struct kmem_cache_node *n, struct slab *slab,
2099
		int mode)
C
Christoph Lameter 已提交
2100
{
2101 2102
	void *freelist;
	unsigned long counters;
2103
	struct slab new;
2104

2105 2106
	lockdep_assert_held(&n->list_lock);

2107 2108 2109 2110 2111
	/*
	 * Zap the freelist and set the frozen bit.
	 * The old freelist is the list of objects for the
	 * per cpu allocation list.
	 */
2112 2113
	freelist = slab->freelist;
	counters = slab->counters;
2114
	new.counters = counters;
2115
	if (mode) {
2116
		new.inuse = slab->objects;
2117 2118 2119 2120
		new.freelist = NULL;
	} else {
		new.freelist = freelist;
	}
2121

2122
	VM_BUG_ON(new.frozen);
2123
	new.frozen = 1;
2124

2125
	if (!__cmpxchg_double_slab(s, slab,
2126
			freelist, counters,
2127
			new.freelist, new.counters,
2128 2129
			"acquire_slab"))
		return NULL;
2130

2131
	remove_partial(n, slab);
2132
	WARN_ON(!freelist);
2133
	return freelist;
C
Christoph Lameter 已提交
2134 2135
}

2136
#ifdef CONFIG_SLUB_CPU_PARTIAL
2137
static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
2138
#else
2139
static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
2140 2141
				   int drain) { }
#endif
2142
static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
2143

C
Christoph Lameter 已提交
2144
/*
C
Christoph Lameter 已提交
2145
 * Try to allocate a partial slab from a specific node.
C
Christoph Lameter 已提交
2146
 */
2147
static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
2148
			      struct slab **ret_slab, gfp_t gfpflags)
C
Christoph Lameter 已提交
2149
{
2150
	struct slab *slab, *slab2;
2151
	void *object = NULL;
2152
	unsigned long flags;
2153
	unsigned int partial_slabs = 0;
C
Christoph Lameter 已提交
2154 2155 2156 2157

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
2158
	 * partial slab and there is none available then get_partial()
C
Christoph Lameter 已提交
2159
	 * will return NULL.
C
Christoph Lameter 已提交
2160 2161 2162 2163
	 */
	if (!n || !n->nr_partial)
		return NULL;

2164
	spin_lock_irqsave(&n->list_lock, flags);
2165
	list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
2166
		void *t;
2167

2168
		if (!pfmemalloc_match(slab, gfpflags))
2169 2170
			continue;

2171
		t = acquire_slab(s, n, slab, object == NULL);
2172
		if (!t)
2173
			break;
2174

2175
		if (!object) {
2176
			*ret_slab = slab;
2177 2178 2179
			stat(s, ALLOC_FROM_PARTIAL);
			object = t;
		} else {
2180
			put_cpu_partial(s, slab, 0);
2181
			stat(s, CPU_PARTIAL_NODE);
2182
			partial_slabs++;
2183
		}
2184
#ifdef CONFIG_SLUB_CPU_PARTIAL
2185
		if (!kmem_cache_has_cpu_partial(s)
2186
			|| partial_slabs > s->cpu_partial_slabs / 2)
2187
			break;
2188 2189 2190
#else
		break;
#endif
2191

2192
	}
2193
	spin_unlock_irqrestore(&n->list_lock, flags);
2194
	return object;
C
Christoph Lameter 已提交
2195 2196 2197
}

/*
2198
 * Get a slab from somewhere. Search in increasing NUMA distances.
C
Christoph Lameter 已提交
2199
 */
2200
static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
2201
			     struct slab **ret_slab)
C
Christoph Lameter 已提交
2202 2203 2204
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
2205
	struct zoneref *z;
2206
	struct zone *zone;
2207
	enum zone_type highest_zoneidx = gfp_zone(flags);
2208
	void *object;
2209
	unsigned int cpuset_mems_cookie;
C
Christoph Lameter 已提交
2210 2211

	/*
C
Christoph Lameter 已提交
2212 2213 2214 2215
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
C
Christoph Lameter 已提交
2216
	 *
C
Christoph Lameter 已提交
2217 2218 2219 2220
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
C
Christoph Lameter 已提交
2221
	 *
2222 2223 2224 2225 2226
	 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
	 * (which makes defrag_ratio = 1000) then every (well almost)
	 * allocation will first attempt to defrag slab caches on other nodes.
	 * This means scanning over all nodes to look for partial slabs which
	 * may be expensive if we do it every time we are trying to find a slab
C
Christoph Lameter 已提交
2227
	 * with available objects.
C
Christoph Lameter 已提交
2228
	 */
2229 2230
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
C
Christoph Lameter 已提交
2231 2232
		return NULL;

2233
	do {
2234
		cpuset_mems_cookie = read_mems_allowed_begin();
2235
		zonelist = node_zonelist(mempolicy_slab_node(), flags);
2236
		for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
2237 2238 2239 2240
			struct kmem_cache_node *n;

			n = get_node(s, zone_to_nid(zone));

2241
			if (n && cpuset_zone_allowed(zone, flags) &&
2242
					n->nr_partial > s->min_partial) {
2243
				object = get_partial_node(s, n, ret_slab, flags);
2244 2245
				if (object) {
					/*
2246 2247 2248 2249 2250
					 * Don't check read_mems_allowed_retry()
					 * here - if mems_allowed was updated in
					 * parallel, that was a harmless race
					 * between allocation and the cpuset
					 * update
2251 2252 2253
					 */
					return object;
				}
2254
			}
C
Christoph Lameter 已提交
2255
		}
2256
	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2257
#endif	/* CONFIG_NUMA */
C
Christoph Lameter 已提交
2258 2259 2260 2261
	return NULL;
}

/*
2262
 * Get a partial slab, lock it and return it.
C
Christoph Lameter 已提交
2263
 */
2264
static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
2265
			 struct slab **ret_slab)
C
Christoph Lameter 已提交
2266
{
2267
	void *object;
2268 2269 2270 2271
	int searchnode = node;

	if (node == NUMA_NO_NODE)
		searchnode = numa_mem_id();
C
Christoph Lameter 已提交
2272

2273
	object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags);
2274 2275
	if (object || node != NUMA_NO_NODE)
		return object;
C
Christoph Lameter 已提交
2276

2277
	return get_any_partial(s, flags, ret_slab);
C
Christoph Lameter 已提交
2278 2279
}

2280
#ifdef CONFIG_PREEMPTION
2281
/*
2282
 * Calculate the next globally unique transaction for disambiguation
2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
#endif

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

2300
#ifdef SLUB_DEBUG_CMPXCHG
2301 2302 2303 2304 2305 2306 2307 2308 2309
static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}
2310
#endif
2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

2323
	pr_info("%s %s: cmpxchg redo ", n, s->name);
2324

2325
#ifdef CONFIG_PREEMPTION
2326
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
2327
		pr_warn("due to cpu change %d -> %d\n",
2328 2329 2330 2331
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
2332
		pr_warn("due to cpu running other code. Event %ld->%ld\n",
2333 2334
			tid_to_event(tid), tid_to_event(actual_tid));
	else
2335
		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
2336 2337
			actual_tid, tid, next_tid(tid));
#endif
2338
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
2339 2340
}

2341
static void init_kmem_cache_cpus(struct kmem_cache *s)
2342 2343
{
	int cpu;
2344
	struct kmem_cache_cpu *c;
2345

2346 2347 2348 2349 2350
	for_each_possible_cpu(cpu) {
		c = per_cpu_ptr(s->cpu_slab, cpu);
		local_lock_init(&c->lock);
		c->tid = init_tid(cpu);
	}
2351
}
2352

C
Christoph Lameter 已提交
2353
/*
2354
 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
2355 2356 2357
 * unfreezes the slabs and puts it on the proper list.
 * Assumes the slab has been already safely taken away from kmem_cache_cpu
 * by the caller.
C
Christoph Lameter 已提交
2358
 */
2359
static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
2360
			    void *freelist)
C
Christoph Lameter 已提交
2361
{
2362
	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE, M_FULL_NOLIST };
2363
	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
2364 2365
	int free_delta = 0;
	enum slab_modes mode = M_NONE;
2366
	void *nextfree, *freelist_iter, *freelist_tail;
2367
	int tail = DEACTIVATE_TO_HEAD;
2368
	unsigned long flags = 0;
2369 2370
	struct slab new;
	struct slab old;
2371

2372
	if (slab->freelist) {
2373
		stat(s, DEACTIVATE_REMOTE_FREES);
2374
		tail = DEACTIVATE_TO_TAIL;
2375 2376
	}

2377
	/*
2378 2379
	 * Stage one: Count the objects on cpu's freelist as free_delta and
	 * remember the last object in freelist_tail for later splicing.
2380
	 */
2381 2382 2383 2384
	freelist_tail = NULL;
	freelist_iter = freelist;
	while (freelist_iter) {
		nextfree = get_freepointer(s, freelist_iter);
2385

2386 2387
		/*
		 * If 'nextfree' is invalid, it is possible that the object at
2388 2389
		 * 'freelist_iter' is already corrupted.  So isolate all objects
		 * starting at 'freelist_iter' by skipping them.
2390
		 */
2391
		if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
2392 2393
			break;

2394 2395
		freelist_tail = freelist_iter;
		free_delta++;
2396

2397
		freelist_iter = nextfree;
2398 2399
	}

2400
	/*
2401 2402
	 * Stage two: Unfreeze the slab while splicing the per-cpu
	 * freelist to the head of slab's freelist.
2403
	 *
2404
	 * Ensure that the slab is unfrozen while the list presence
2405
	 * reflects the actual number of objects during unfreeze.
2406
	 *
2407 2408 2409 2410
	 * We first perform cmpxchg holding lock and insert to list
	 * when it succeed. If there is mismatch then the slab is not
	 * unfrozen and number of objects in the slab may have changed.
	 * Then release lock and retry cmpxchg again.
2411
	 */
2412
redo:
2413

2414 2415
	old.freelist = READ_ONCE(slab->freelist);
	old.counters = READ_ONCE(slab->counters);
2416
	VM_BUG_ON(!old.frozen);
2417

2418 2419
	/* Determine target state of the slab */
	new.counters = old.counters;
2420 2421 2422
	if (freelist_tail) {
		new.inuse -= free_delta;
		set_freepointer(s, freelist_tail, old.freelist);
2423 2424 2425 2426 2427 2428
		new.freelist = freelist;
	} else
		new.freelist = old.freelist;

	new.frozen = 0;

2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445
	if (!new.inuse && n->nr_partial >= s->min_partial) {
		mode = M_FREE;
	} else if (new.freelist) {
		mode = M_PARTIAL;
		/*
		 * Taking the spinlock removes the possibility that
		 * acquire_slab() will see a slab that is frozen
		 */
		spin_lock_irqsave(&n->list_lock, flags);
	} else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
		mode = M_FULL;
		/*
		 * This also ensures that the scanning of full
		 * slabs from diagnostic functions will not see
		 * any frozen slabs.
		 */
		spin_lock_irqsave(&n->list_lock, flags);
2446
	} else {
2447
		mode = M_FULL_NOLIST;
2448 2449 2450
	}


2451
	if (!cmpxchg_double_slab(s, slab,
2452 2453
				old.freelist, old.counters,
				new.freelist, new.counters,
2454 2455 2456
				"unfreezing slab")) {
		if (mode == M_PARTIAL || mode == M_FULL)
			spin_unlock_irqrestore(&n->list_lock, flags);
2457
		goto redo;
2458
	}
2459 2460


2461 2462 2463
	if (mode == M_PARTIAL) {
		add_partial(n, slab, tail);
		spin_unlock_irqrestore(&n->list_lock, flags);
2464
		stat(s, tail);
2465
	} else if (mode == M_FREE) {
2466
		stat(s, DEACTIVATE_EMPTY);
2467
		discard_slab(s, slab);
2468
		stat(s, FREE_SLAB);
2469 2470 2471 2472 2473 2474
	} else if (mode == M_FULL) {
		add_full(s, n, slab);
		spin_unlock_irqrestore(&n->list_lock, flags);
		stat(s, DEACTIVATE_FULL);
	} else if (mode == M_FULL_NOLIST) {
		stat(s, DEACTIVATE_FULL);
2475
	}
C
Christoph Lameter 已提交
2476 2477
}

2478
#ifdef CONFIG_SLUB_CPU_PARTIAL
2479
static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
2480
{
2481
	struct kmem_cache_node *n = NULL, *n2 = NULL;
2482
	struct slab *slab, *slab_to_discard = NULL;
2483
	unsigned long flags = 0;
2484

2485 2486 2487
	while (partial_slab) {
		struct slab new;
		struct slab old;
2488

2489 2490
		slab = partial_slab;
		partial_slab = slab->next;
2491

2492
		n2 = get_node(s, slab_nid(slab));
2493 2494
		if (n != n2) {
			if (n)
2495
				spin_unlock_irqrestore(&n->list_lock, flags);
2496 2497

			n = n2;
2498
			spin_lock_irqsave(&n->list_lock, flags);
2499
		}
2500 2501 2502

		do {

2503 2504
			old.freelist = slab->freelist;
			old.counters = slab->counters;
2505
			VM_BUG_ON(!old.frozen);
2506 2507 2508 2509 2510 2511

			new.counters = old.counters;
			new.freelist = old.freelist;

			new.frozen = 0;

2512
		} while (!__cmpxchg_double_slab(s, slab,
2513 2514 2515 2516
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"));

2517
		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
2518 2519
			slab->next = slab_to_discard;
			slab_to_discard = slab;
2520
		} else {
2521
			add_partial(n, slab, DEACTIVATE_TO_TAIL);
2522
			stat(s, FREE_ADD_PARTIAL);
2523 2524 2525 2526
		}
	}

	if (n)
2527
		spin_unlock_irqrestore(&n->list_lock, flags);
2528

2529 2530 2531
	while (slab_to_discard) {
		slab = slab_to_discard;
		slab_to_discard = slab_to_discard->next;
2532 2533

		stat(s, DEACTIVATE_EMPTY);
2534
		discard_slab(s, slab);
2535 2536
		stat(s, FREE_SLAB);
	}
2537
}
2538

2539 2540 2541 2542 2543
/*
 * Unfreeze all the cpu partial slabs.
 */
static void unfreeze_partials(struct kmem_cache *s)
{
2544
	struct slab *partial_slab;
2545 2546
	unsigned long flags;

2547
	local_lock_irqsave(&s->cpu_slab->lock, flags);
2548
	partial_slab = this_cpu_read(s->cpu_slab->partial);
2549
	this_cpu_write(s->cpu_slab->partial, NULL);
2550
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2551

2552 2553
	if (partial_slab)
		__unfreeze_partials(s, partial_slab);
2554 2555 2556 2557 2558
}

static void unfreeze_partials_cpu(struct kmem_cache *s,
				  struct kmem_cache_cpu *c)
{
2559
	struct slab *partial_slab;
2560

2561
	partial_slab = slub_percpu_partial(c);
2562 2563
	c->partial = NULL;

2564 2565
	if (partial_slab)
		__unfreeze_partials(s, partial_slab);
2566 2567 2568
}

/*
2569 2570
 * Put a slab that was just frozen (in __slab_free|get_partial_node) into a
 * partial slab slot if available.
2571 2572 2573 2574
 *
 * If we did not find a slot then simply move all the partials to the
 * per node partial list.
 */
2575
static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
2576
{
2577 2578
	struct slab *oldslab;
	struct slab *slab_to_unfreeze = NULL;
2579
	unsigned long flags;
2580
	int slabs = 0;
2581

2582
	local_lock_irqsave(&s->cpu_slab->lock, flags);
2583

2584
	oldslab = this_cpu_read(s->cpu_slab->partial);
2585

2586 2587
	if (oldslab) {
		if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
2588 2589 2590 2591 2592
			/*
			 * Partial array is full. Move the existing set to the
			 * per node partial list. Postpone the actual unfreezing
			 * outside of the critical section.
			 */
2593 2594
			slab_to_unfreeze = oldslab;
			oldslab = NULL;
2595
		} else {
2596
			slabs = oldslab->slabs;
2597
		}
2598
	}
2599

2600
	slabs++;
2601

2602 2603
	slab->slabs = slabs;
	slab->next = oldslab;
2604

2605
	this_cpu_write(s->cpu_slab->partial, slab);
2606

2607
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2608

2609 2610
	if (slab_to_unfreeze) {
		__unfreeze_partials(s, slab_to_unfreeze);
2611 2612
		stat(s, CPU_PARTIAL_DRAIN);
	}
2613 2614
}

2615 2616 2617 2618 2619 2620 2621 2622
#else	/* CONFIG_SLUB_CPU_PARTIAL */

static inline void unfreeze_partials(struct kmem_cache *s) { }
static inline void unfreeze_partials_cpu(struct kmem_cache *s,
				  struct kmem_cache_cpu *c) { }

#endif	/* CONFIG_SLUB_CPU_PARTIAL */

2623
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2624
{
2625
	unsigned long flags;
2626
	struct slab *slab;
2627 2628
	void *freelist;

2629
	local_lock_irqsave(&s->cpu_slab->lock, flags);
2630

2631
	slab = c->slab;
2632
	freelist = c->freelist;
2633

2634
	c->slab = NULL;
2635
	c->freelist = NULL;
2636
	c->tid = next_tid(c->tid);
2637

2638
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2639

2640 2641
	if (slab) {
		deactivate_slab(s, slab, freelist);
2642 2643
		stat(s, CPUSLAB_FLUSH);
	}
C
Christoph Lameter 已提交
2644 2645
}

2646
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
C
Christoph Lameter 已提交
2647
{
2648
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2649
	void *freelist = c->freelist;
2650
	struct slab *slab = c->slab;
C
Christoph Lameter 已提交
2651

2652
	c->slab = NULL;
2653 2654 2655
	c->freelist = NULL;
	c->tid = next_tid(c->tid);

2656 2657
	if (slab) {
		deactivate_slab(s, slab, freelist);
2658 2659
		stat(s, CPUSLAB_FLUSH);
	}
2660

2661
	unfreeze_partials_cpu(s, c);
C
Christoph Lameter 已提交
2662 2663
}

2664 2665 2666 2667 2668 2669
struct slub_flush_work {
	struct work_struct work;
	struct kmem_cache *s;
	bool skip;
};

2670 2671 2672
/*
 * Flush cpu slab.
 *
2673
 * Called from CPU work handler with migration disabled.
2674
 */
2675
static void flush_cpu_slab(struct work_struct *w)
C
Christoph Lameter 已提交
2676
{
2677 2678 2679 2680 2681 2682 2683 2684
	struct kmem_cache *s;
	struct kmem_cache_cpu *c;
	struct slub_flush_work *sfw;

	sfw = container_of(w, struct slub_flush_work, work);

	s = sfw->s;
	c = this_cpu_ptr(s->cpu_slab);
2685

2686
	if (c->slab)
2687
		flush_slab(s, c);
C
Christoph Lameter 已提交
2688

2689
	unfreeze_partials(s);
C
Christoph Lameter 已提交
2690 2691
}

2692
static bool has_cpu_slab(int cpu, struct kmem_cache *s)
2693 2694 2695
{
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

2696
	return c->slab || slub_percpu_partial(c);
2697 2698
}

2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731
static DEFINE_MUTEX(flush_lock);
static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);

static void flush_all_cpus_locked(struct kmem_cache *s)
{
	struct slub_flush_work *sfw;
	unsigned int cpu;

	lockdep_assert_cpus_held();
	mutex_lock(&flush_lock);

	for_each_online_cpu(cpu) {
		sfw = &per_cpu(slub_flush, cpu);
		if (!has_cpu_slab(cpu, s)) {
			sfw->skip = true;
			continue;
		}
		INIT_WORK(&sfw->work, flush_cpu_slab);
		sfw->skip = false;
		sfw->s = s;
		schedule_work_on(cpu, &sfw->work);
	}

	for_each_online_cpu(cpu) {
		sfw = &per_cpu(slub_flush, cpu);
		if (sfw->skip)
			continue;
		flush_work(&sfw->work);
	}

	mutex_unlock(&flush_lock);
}

C
Christoph Lameter 已提交
2732 2733
static void flush_all(struct kmem_cache *s)
{
2734 2735 2736
	cpus_read_lock();
	flush_all_cpus_locked(s);
	cpus_read_unlock();
C
Christoph Lameter 已提交
2737 2738
}

2739 2740 2741 2742 2743 2744 2745 2746 2747
/*
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
 */
static int slub_cpu_dead(unsigned int cpu)
{
	struct kmem_cache *s;

	mutex_lock(&slab_mutex);
2748
	list_for_each_entry(s, &slab_caches, list)
2749 2750 2751 2752 2753
		__flush_cpu_slab(s, cpu);
	mutex_unlock(&slab_mutex);
	return 0;
}

2754 2755 2756 2757
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
2758
static inline int node_match(struct slab *slab, int node)
2759 2760
{
#ifdef CONFIG_NUMA
2761
	if (node != NUMA_NO_NODE && slab_nid(slab) != node)
2762 2763 2764 2765 2766
		return 0;
#endif
	return 1;
}

2767
#ifdef CONFIG_SLUB_DEBUG
2768
static int count_free(struct slab *slab)
2769
{
2770
	return slab->objects - slab->inuse;
2771 2772
}

2773 2774 2775 2776 2777 2778 2779
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->total_objects);
}
#endif /* CONFIG_SLUB_DEBUG */

#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
2780
static unsigned long count_partial(struct kmem_cache_node *n,
2781
					int (*get_count)(struct slab *))
2782 2783 2784
{
	unsigned long flags;
	unsigned long x = 0;
2785
	struct slab *slab;
2786 2787

	spin_lock_irqsave(&n->list_lock, flags);
2788 2789
	list_for_each_entry(slab, &n->partial, slab_list)
		x += get_count(slab);
2790 2791 2792
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}
2793
#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
2794

2795 2796 2797
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
2798 2799 2800
#ifdef CONFIG_SLUB_DEBUG
	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);
2801
	int node;
2802
	struct kmem_cache_node *n;
2803

2804 2805 2806
	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
		return;

2807 2808
	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
		nid, gfpflags, &gfpflags);
2809
	pr_warn("  cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
2810 2811
		s->name, s->object_size, s->size, oo_order(s->oo),
		oo_order(s->min));
2812

2813
	if (oo_order(s->min) > get_order(s->object_size))
2814 2815
		pr_warn("  %s debugging increased min order, use slub_debug=O to disable.\n",
			s->name);
2816

2817
	for_each_kmem_cache_node(s, node, n) {
2818 2819 2820 2821
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

2822 2823 2824
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
2825

2826
		pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
2827 2828
			node, nr_slabs, nr_objs, nr_free);
	}
2829
#endif
2830 2831
}

2832
static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
2833
{
2834
	if (unlikely(slab_test_pfmemalloc(slab)))
2835 2836 2837 2838 2839
		return gfp_pfmemalloc_allowed(gfpflags);

	return true;
}

2840
/*
2841 2842
 * Check the slab->freelist and either transfer the freelist to the
 * per cpu freelist or deactivate the slab.
2843
 *
2844
 * The slab is still frozen if the return value is not NULL.
2845
 *
2846
 * If this function returns NULL then the slab has been unfrozen.
2847
 */
2848
static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
2849
{
2850
	struct slab new;
2851 2852 2853
	unsigned long counters;
	void *freelist;

2854 2855
	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));

2856
	do {
2857 2858
		freelist = slab->freelist;
		counters = slab->counters;
2859

2860
		new.counters = counters;
2861
		VM_BUG_ON(!new.frozen);
2862

2863
		new.inuse = slab->objects;
2864 2865
		new.frozen = freelist != NULL;

2866
	} while (!__cmpxchg_double_slab(s, slab,
2867 2868 2869 2870 2871 2872 2873
		freelist, counters,
		NULL, new.counters,
		"get_freelist"));

	return freelist;
}

C
Christoph Lameter 已提交
2874
/*
2875 2876 2877 2878 2879 2880
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
C
Christoph Lameter 已提交
2881
 *
2882 2883 2884
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
C
Christoph Lameter 已提交
2885
 *
2886
 * And if we were unable to get a new slab from the partial slab lists then
C
Christoph Lameter 已提交
2887 2888
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
2889
 *
2890
 * Version of __slab_alloc to use when we know that preemption is
2891
 * already disabled (which is the case for bulk allocation).
C
Christoph Lameter 已提交
2892
 */
2893
static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2894
			  unsigned long addr, struct kmem_cache_cpu *c)
C
Christoph Lameter 已提交
2895
{
2896
	void *freelist;
2897
	struct slab *slab;
2898
	unsigned long flags;
C
Christoph Lameter 已提交
2899

2900 2901
	stat(s, ALLOC_SLOWPATH);

2902
reread_slab:
2903

2904 2905
	slab = READ_ONCE(c->slab);
	if (!slab) {
2906 2907 2908 2909 2910
		/*
		 * if the node is not online or has no normal memory, just
		 * ignore the node constraint
		 */
		if (unlikely(node != NUMA_NO_NODE &&
2911
			     !node_isset(node, slab_nodes)))
2912
			node = NUMA_NO_NODE;
C
Christoph Lameter 已提交
2913
		goto new_slab;
2914
	}
2915
redo:
2916

2917
	if (unlikely(!node_match(slab, node))) {
2918 2919 2920 2921
		/*
		 * same as above but node_match() being false already
		 * implies node != NUMA_NO_NODE
		 */
2922
		if (!node_isset(node, slab_nodes)) {
2923 2924
			node = NUMA_NO_NODE;
		} else {
2925
			stat(s, ALLOC_NODE_MISMATCH);
2926
			goto deactivate_slab;
2927
		}
2928
	}
C
Christoph Lameter 已提交
2929

2930 2931 2932 2933 2934
	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
2935
	if (unlikely(!pfmemalloc_match(slab, gfpflags)))
2936
		goto deactivate_slab;
2937

2938
	/* must check again c->slab in case we got preempted and it changed */
2939
	local_lock_irqsave(&s->cpu_slab->lock, flags);
2940
	if (unlikely(slab != c->slab)) {
2941
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2942
		goto reread_slab;
2943
	}
2944 2945
	freelist = c->freelist;
	if (freelist)
2946
		goto load_freelist;
2947

2948
	freelist = get_freelist(s, slab);
C
Christoph Lameter 已提交
2949

2950
	if (!freelist) {
2951
		c->slab = NULL;
2952
		c->tid = next_tid(c->tid);
2953
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2954
		stat(s, DEACTIVATE_BYPASS);
2955
		goto new_slab;
2956
	}
C
Christoph Lameter 已提交
2957

2958
	stat(s, ALLOC_REFILL);
C
Christoph Lameter 已提交
2959

2960
load_freelist:
2961

2962
	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
2963

2964 2965
	/*
	 * freelist is pointing to the list of objects to be used.
2966 2967
	 * slab is pointing to the slab from which the objects are obtained.
	 * That slab must be frozen for per cpu allocations to work.
2968
	 */
2969
	VM_BUG_ON(!c->slab->frozen);
2970
	c->freelist = get_freepointer(s, freelist);
2971
	c->tid = next_tid(c->tid);
2972
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2973
	return freelist;
C
Christoph Lameter 已提交
2974

2975 2976
deactivate_slab:

2977
	local_lock_irqsave(&s->cpu_slab->lock, flags);
2978
	if (slab != c->slab) {
2979
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2980
		goto reread_slab;
2981
	}
2982
	freelist = c->freelist;
2983
	c->slab = NULL;
2984
	c->freelist = NULL;
2985
	c->tid = next_tid(c->tid);
2986
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2987
	deactivate_slab(s, slab, freelist);
2988

C
Christoph Lameter 已提交
2989
new_slab:
2990

2991
	if (slub_percpu_partial(c)) {
2992
		local_lock_irqsave(&s->cpu_slab->lock, flags);
2993
		if (unlikely(c->slab)) {
2994
			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2995
			goto reread_slab;
2996
		}
2997
		if (unlikely(!slub_percpu_partial(c))) {
2998
			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
2999 3000
			/* we were preempted and partial list got empty */
			goto new_objects;
3001
		}
3002

3003 3004
		slab = c->slab = slub_percpu_partial(c);
		slub_set_percpu_partial(c, slab);
3005
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3006 3007
		stat(s, CPU_PARTIAL_ALLOC);
		goto redo;
C
Christoph Lameter 已提交
3008 3009
	}

3010 3011
new_objects:

3012
	freelist = get_partial(s, gfpflags, node, &slab);
3013
	if (freelist)
3014
		goto check_new_slab;
3015

3016
	slub_put_cpu_ptr(s->cpu_slab);
3017
	slab = new_slab(s, gfpflags, node);
3018
	c = slub_get_cpu_ptr(s->cpu_slab);
3019

3020
	if (unlikely(!slab)) {
3021
		slab_out_of_memory(s, gfpflags, node);
3022
		return NULL;
C
Christoph Lameter 已提交
3023
	}
3024

3025
	/*
3026
	 * No other reference to the slab yet so we can
3027 3028
	 * muck around with it freely without cmpxchg
	 */
3029 3030
	freelist = slab->freelist;
	slab->freelist = NULL;
3031 3032 3033

	stat(s, ALLOC_SLAB);

3034
check_new_slab:
3035

3036
	if (kmem_cache_debug(s)) {
3037
		if (!alloc_debug_processing(s, slab, freelist, addr)) {
3038 3039
			/* Slab failed checks. Next slab needed */
			goto new_slab;
3040
		} else {
3041 3042 3043 3044 3045
			/*
			 * For debug case, we don't load freelist so that all
			 * allocations go through alloc_debug_processing()
			 */
			goto return_single;
3046
		}
3047 3048
	}

3049
	if (unlikely(!pfmemalloc_match(slab, gfpflags)))
3050 3051 3052 3053 3054 3055
		/*
		 * For !pfmemalloc_match() case we don't load freelist so that
		 * we don't make further mismatched allocations easier.
		 */
		goto return_single;

3056
retry_load_slab:
3057

3058
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3059
	if (unlikely(c->slab)) {
3060
		void *flush_freelist = c->freelist;
3061
		struct slab *flush_slab = c->slab;
3062

3063
		c->slab = NULL;
3064 3065 3066
		c->freelist = NULL;
		c->tid = next_tid(c->tid);

3067
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3068

3069
		deactivate_slab(s, flush_slab, flush_freelist);
3070 3071 3072

		stat(s, CPUSLAB_FLUSH);

3073
		goto retry_load_slab;
3074
	}
3075
	c->slab = slab;
3076

3077 3078 3079
	goto load_freelist;

return_single:
3080

3081
	deactivate_slab(s, slab, get_freepointer(s, freelist));
3082
	return freelist;
3083 3084
}

3085
/*
3086 3087 3088
 * A wrapper for ___slab_alloc() for contexts where preemption is not yet
 * disabled. Compensates for possible cpu changes by refetching the per cpu area
 * pointer.
3089 3090 3091 3092 3093 3094
 */
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *p;

3095
#ifdef CONFIG_PREEMPT_COUNT
3096 3097
	/*
	 * We may have been preempted and rescheduled on a different
3098
	 * cpu before disabling preemption. Need to reload cpu area
3099 3100
	 * pointer.
	 */
3101
	c = slub_get_cpu_ptr(s->cpu_slab);
3102 3103 3104
#endif

	p = ___slab_alloc(s, gfpflags, node, addr, c);
3105
#ifdef CONFIG_PREEMPT_COUNT
3106
	slub_put_cpu_ptr(s->cpu_slab);
3107
#endif
3108 3109 3110
	return p;
}

3111 3112 3113 3114 3115 3116 3117 3118
/*
 * If the object has been wiped upon free, make sure it's fully initialized by
 * zeroing out freelist pointer.
 */
static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
						   void *obj)
{
	if (unlikely(slab_want_init_on_free(s)) && obj)
3119 3120
		memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
			0, sizeof(void *));
3121 3122
}

3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
3133
static __always_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
3134
		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3135
{
3136
	void *object;
3137
	struct kmem_cache_cpu *c;
3138
	struct slab *slab;
3139
	unsigned long tid;
3140
	struct obj_cgroup *objcg = NULL;
3141
	bool init = false;
3142

3143
	s = slab_pre_alloc_hook(s, lru, &objcg, 1, gfpflags);
3144
	if (!s)
A
Akinobu Mita 已提交
3145
		return NULL;
3146 3147 3148 3149 3150

	object = kfence_alloc(s, orig_size, gfpflags);
	if (unlikely(object))
		goto out;

3151 3152 3153 3154 3155 3156
redo:
	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
3157
	 *
3158 3159 3160 3161 3162
	 * We must guarantee that tid and kmem_cache_cpu are retrieved on the
	 * same cpu. We read first the kmem_cache_cpu pointer and use it to read
	 * the tid. If we are preempted and switched to another cpu between the
	 * two reads, it's OK as the two are still associated with the same cpu
	 * and cmpxchg later will validate the cpu.
3163
	 */
3164 3165
	c = raw_cpu_ptr(s->cpu_slab);
	tid = READ_ONCE(c->tid);
3166 3167 3168 3169

	/*
	 * Irqless object alloc/free algorithm used here depends on sequence
	 * of fetching cpu_slab's data. tid should be fetched before anything
3170
	 * on c to guarantee that object and slab associated with previous tid
3171
	 * won't be used with current tid. If we fetch tid first, object and
3172
	 * slab could be one associated with next tid and our alloc/free
3173 3174 3175
	 * request will be failed. In this case, we will retry. So, no problem.
	 */
	barrier();
3176 3177 3178 3179 3180 3181 3182 3183

	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */

3184
	object = c->freelist;
3185
	slab = c->slab;
3186 3187 3188 3189 3190 3191 3192 3193
	/*
	 * We cannot use the lockless fastpath on PREEMPT_RT because if a
	 * slowpath has taken the local_lock_irqsave(), it is not protected
	 * against a fast path operation in an irq handler. So we need to take
	 * the slow path which uses local_lock. It is still relatively fast if
	 * there is a suitable cpu freelist.
	 */
	if (IS_ENABLED(CONFIG_PREEMPT_RT) ||
3194
	    unlikely(!object || !slab || !node_match(slab, node))) {
3195
		object = __slab_alloc(s, gfpflags, node, addr, c);
3196
	} else {
3197 3198
		void *next_object = get_freepointer_safe(s, object);

3199
		/*
L
Lucas De Marchi 已提交
3200
		 * The cmpxchg will only match if there was no additional
3201 3202
		 * operation and if we are on the right processor.
		 *
3203 3204
		 * The cmpxchg does the following atomically (without lock
		 * semantics!)
3205 3206 3207 3208
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
3209 3210 3211
		 * Since this is without lock semantics the protection is only
		 * against code executing on this cpu *not* from access by
		 * other cpus.
3212
		 */
3213
		if (unlikely(!this_cpu_cmpxchg_double(
3214 3215
				s->cpu_slab->freelist, s->cpu_slab->tid,
				object, tid,
3216
				next_object, next_tid(tid)))) {
3217 3218 3219 3220

			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
3221
		prefetch_freepointer(s, next_object);
3222
		stat(s, ALLOC_FASTPATH);
3223
	}
3224

3225
	maybe_wipe_obj_freeptr(s, object);
3226
	init = slab_want_init_on_alloc(gfpflags, s);
3227

3228
out:
3229
	slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
3230

3231
	return object;
C
Christoph Lameter 已提交
3232 3233
}

3234
static __always_inline void *slab_alloc(struct kmem_cache *s, struct list_lru *lru,
3235
		gfp_t gfpflags, unsigned long addr, size_t orig_size)
3236
{
3237
	return slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, addr, orig_size);
3238 3239
}

3240 3241 3242
static __always_inline
void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
			     gfp_t gfpflags)
C
Christoph Lameter 已提交
3243
{
3244
	void *ret = slab_alloc(s, lru, gfpflags, _RET_IP_, s->object_size);
3245

3246
	trace_kmem_cache_alloc(_RET_IP_, ret, s, s->object_size,
3247
				s->size, gfpflags);
3248 3249

	return ret;
C
Christoph Lameter 已提交
3250
}
3251 3252 3253 3254 3255

void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
	return __kmem_cache_alloc_lru(s, NULL, gfpflags);
}
C
Christoph Lameter 已提交
3256 3257
EXPORT_SYMBOL(kmem_cache_alloc);

3258 3259 3260 3261 3262 3263 3264
void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
			   gfp_t gfpflags)
{
	return __kmem_cache_alloc_lru(s, lru, gfpflags);
}
EXPORT_SYMBOL(kmem_cache_alloc_lru);

3265
#ifdef CONFIG_TRACING
3266 3267
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
3268
	void *ret = slab_alloc(s, NULL, gfpflags, _RET_IP_, size);
3269
	trace_kmalloc(_RET_IP_, ret, s, size, s->size, gfpflags);
3270
	ret = kasan_kmalloc(s, ret, size, gfpflags);
3271 3272 3273
	return ret;
}
EXPORT_SYMBOL(kmem_cache_alloc_trace);
3274 3275
#endif

C
Christoph Lameter 已提交
3276 3277
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
3278
	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
3279

3280
	trace_kmem_cache_alloc_node(_RET_IP_, ret, s,
3281
				    s->object_size, s->size, gfpflags, node);
3282 3283

	return ret;
C
Christoph Lameter 已提交
3284 3285 3286
}
EXPORT_SYMBOL(kmem_cache_alloc_node);

3287
#ifdef CONFIG_TRACING
3288
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
3289
				    gfp_t gfpflags,
3290
				    int node, size_t size)
3291
{
3292
	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
3293

3294
	trace_kmalloc_node(_RET_IP_, ret, s,
3295
			   size, s->size, gfpflags, node);
3296

3297
	ret = kasan_kmalloc(s, ret, size, gfpflags);
3298
	return ret;
3299
}
3300
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3301 3302
#endif

C
Christoph Lameter 已提交
3303
/*
3304
 * Slow path handling. This may still be called frequently since objects
3305
 * have a longer lifetime than the cpu slabs in most processing loads.
C
Christoph Lameter 已提交
3306
 *
3307
 * So we still attempt to reduce cache line usage. Just take the slab
3308
 * lock and free the item. If there is no additional partial slab
3309
 * handling required then we can return immediately.
C
Christoph Lameter 已提交
3310
 */
3311
static void __slab_free(struct kmem_cache *s, struct slab *slab,
3312 3313 3314
			void *head, void *tail, int cnt,
			unsigned long addr)

C
Christoph Lameter 已提交
3315 3316
{
	void *prior;
3317
	int was_frozen;
3318
	struct slab new;
3319 3320
	unsigned long counters;
	struct kmem_cache_node *n = NULL;
3321
	unsigned long flags;
C
Christoph Lameter 已提交
3322

3323
	stat(s, FREE_SLOWPATH);
C
Christoph Lameter 已提交
3324

3325 3326 3327
	if (kfence_free(head))
		return;

3328
	if (kmem_cache_debug(s) &&
3329
	    !free_debug_processing(s, slab, head, tail, cnt, addr))
3330
		return;
C
Christoph Lameter 已提交
3331

3332
	do {
3333 3334 3335 3336
		if (unlikely(n)) {
			spin_unlock_irqrestore(&n->list_lock, flags);
			n = NULL;
		}
3337 3338
		prior = slab->freelist;
		counters = slab->counters;
3339
		set_freepointer(s, tail, prior);
3340 3341
		new.counters = counters;
		was_frozen = new.frozen;
3342
		new.inuse -= cnt;
3343
		if ((!new.inuse || !prior) && !was_frozen) {
3344

3345
			if (kmem_cache_has_cpu_partial(s) && !prior) {
3346 3347

				/*
3348 3349 3350 3351
				 * Slab was on no list before and will be
				 * partially empty
				 * We can defer the list move and instead
				 * freeze it.
3352 3353 3354
				 */
				new.frozen = 1;

3355
			} else { /* Needs to be taken off a list */
3356

3357
				n = get_node(s, slab_nid(slab));
3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368
				/*
				 * Speculatively acquire the list_lock.
				 * If the cmpxchg does not succeed then we may
				 * drop the list_lock without any processing.
				 *
				 * Otherwise the list_lock will synchronize with
				 * other processors updating the list of slabs.
				 */
				spin_lock_irqsave(&n->list_lock, flags);

			}
3369
		}
C
Christoph Lameter 已提交
3370

3371
	} while (!cmpxchg_double_slab(s, slab,
3372
		prior, counters,
3373
		head, new.counters,
3374
		"__slab_free"));
C
Christoph Lameter 已提交
3375

3376
	if (likely(!n)) {
3377

3378 3379 3380 3381 3382 3383 3384 3385
		if (likely(was_frozen)) {
			/*
			 * The list lock was not taken therefore no list
			 * activity can be necessary.
			 */
			stat(s, FREE_FROZEN);
		} else if (new.frozen) {
			/*
3386
			 * If we just froze the slab then put it onto the
3387 3388
			 * per cpu partial list.
			 */
3389
			put_cpu_partial(s, slab, 1);
3390 3391
			stat(s, CPU_PARTIAL_FREE);
		}
3392

3393 3394
		return;
	}
C
Christoph Lameter 已提交
3395

3396
	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
3397 3398
		goto slab_empty;

C
Christoph Lameter 已提交
3399
	/*
3400 3401
	 * Objects left in the slab. If it was not on the partial list before
	 * then add it.
C
Christoph Lameter 已提交
3402
	 */
3403
	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
3404 3405
		remove_full(s, n, slab);
		add_partial(n, slab, DEACTIVATE_TO_TAIL);
3406
		stat(s, FREE_ADD_PARTIAL);
3407
	}
3408
	spin_unlock_irqrestore(&n->list_lock, flags);
C
Christoph Lameter 已提交
3409 3410 3411
	return;

slab_empty:
3412
	if (prior) {
C
Christoph Lameter 已提交
3413
		/*
3414
		 * Slab on the partial list.
C
Christoph Lameter 已提交
3415
		 */
3416
		remove_partial(n, slab);
3417
		stat(s, FREE_REMOVE_PARTIAL);
3418
	} else {
3419
		/* Slab must be on the full list */
3420
		remove_full(s, n, slab);
3421
	}
3422

3423
	spin_unlock_irqrestore(&n->list_lock, flags);
3424
	stat(s, FREE_SLAB);
3425
	discard_slab(s, slab);
C
Christoph Lameter 已提交
3426 3427
}

3428 3429 3430 3431 3432 3433 3434 3435 3436 3437
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
3438 3439
 *
 * Bulk free of a freelist with several objects (all pointing to the
3440
 * same slab) possible by specifying head and tail ptr, plus objects
3441
 * count (cnt). Bulk free indicated by tail pointer being set.
3442
 */
3443
static __always_inline void do_slab_free(struct kmem_cache *s,
3444
				struct slab *slab, void *head, void *tail,
3445
				int cnt, unsigned long addr)
3446
{
3447
	void *tail_obj = tail ? : head;
3448
	struct kmem_cache_cpu *c;
3449
	unsigned long tid;
3450

3451 3452 3453 3454 3455
redo:
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
3456
	 * during the cmpxchg then the free will succeed.
3457
	 */
3458 3459
	c = raw_cpu_ptr(s->cpu_slab);
	tid = READ_ONCE(c->tid);
3460

3461 3462
	/* Same with comment on barrier() in slab_alloc_node() */
	barrier();
3463

3464
	if (likely(slab == c->slab)) {
3465
#ifndef CONFIG_PREEMPT_RT
3466 3467 3468
		void **freelist = READ_ONCE(c->freelist);

		set_freepointer(s, tail_obj, freelist);
3469

3470
		if (unlikely(!this_cpu_cmpxchg_double(
3471
				s->cpu_slab->freelist, s->cpu_slab->tid,
3472
				freelist, tid,
3473
				head, next_tid(tid)))) {
3474 3475 3476 3477

			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
#else /* CONFIG_PREEMPT_RT */
		/*
		 * We cannot use the lockless fastpath on PREEMPT_RT because if
		 * a slowpath has taken the local_lock_irqsave(), it is not
		 * protected against a fast path operation in an irq handler. So
		 * we need to take the local_lock. We shouldn't simply defer to
		 * __slab_free() as that wouldn't use the cpu freelist at all.
		 */
		void **freelist;

		local_lock(&s->cpu_slab->lock);
		c = this_cpu_ptr(s->cpu_slab);
3490
		if (unlikely(slab != c->slab)) {
3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502
			local_unlock(&s->cpu_slab->lock);
			goto redo;
		}
		tid = c->tid;
		freelist = c->freelist;

		set_freepointer(s, tail_obj, freelist);
		c->freelist = head;
		c->tid = next_tid(tid);

		local_unlock(&s->cpu_slab->lock);
#endif
3503
		stat(s, FREE_FASTPATH);
3504
	} else
3505
		__slab_free(s, slab, head, tail_obj, cnt, addr);
3506 3507 3508

}

3509
static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
3510
				      void *head, void *tail, void **p, int cnt,
3511 3512
				      unsigned long addr)
{
3513
	memcg_slab_free_hook(s, slab, p, cnt);
3514
	/*
3515 3516
	 * With KASAN enabled slab_free_freelist_hook modifies the freelist
	 * to remove objects, whose reuse must be delayed.
3517
	 */
3518
	if (slab_free_freelist_hook(s, &head, &tail, &cnt))
3519
		do_slab_free(s, slab, head, tail, cnt, addr);
3520 3521
}

3522
#ifdef CONFIG_KASAN_GENERIC
3523 3524
void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
{
3525
	do_slab_free(cache, virt_to_slab(x), x, NULL, 1, addr);
3526 3527 3528
}
#endif

C
Christoph Lameter 已提交
3529 3530
void kmem_cache_free(struct kmem_cache *s, void *x)
{
3531 3532
	s = cache_from_obj(s, x);
	if (!s)
3533
		return;
3534
	trace_kmem_cache_free(_RET_IP_, x, s->name);
3535
	slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_);
C
Christoph Lameter 已提交
3536 3537 3538
}
EXPORT_SYMBOL(kmem_cache_free);

3539
struct detached_freelist {
3540
	struct slab *slab;
3541 3542 3543
	void *tail;
	void *freelist;
	int cnt;
3544
	struct kmem_cache *s;
3545
};
3546

3547 3548 3549
/*
 * This function progressively scans the array with free objects (with
 * a limited look ahead) and extract objects belonging to the same
3550 3551
 * slab.  It builds a detached freelist directly within the given
 * slab/objects.  This can happen without any need for
3552 3553 3554 3555 3556 3557 3558
 * synchronization, because the objects are owned by running process.
 * The freelist is build up as a single linked list in the objects.
 * The idea is, that this detached freelist can then be bulk
 * transferred to the real freelist(s), but only requiring a single
 * synchronization primitive.  Look ahead in the array is limited due
 * to performance reasons.
 */
3559 3560 3561
static inline
int build_detached_freelist(struct kmem_cache *s, size_t size,
			    void **p, struct detached_freelist *df)
3562 3563 3564
{
	int lookahead = 3;
	void *object;
3565
	struct folio *folio;
3566
	size_t same;
3567

3568
	object = p[--size];
3569
	folio = virt_to_folio(object);
3570 3571
	if (!s) {
		/* Handle kalloc'ed objects */
3572
		if (unlikely(!folio_test_slab(folio))) {
3573
			free_large_kmalloc(folio, object);
3574
			df->slab = NULL;
3575 3576 3577
			return size;
		}
		/* Derive kmem_cache from object */
3578 3579
		df->slab = folio_slab(folio);
		df->s = df->slab->slab_cache;
3580
	} else {
3581
		df->slab = folio_slab(folio);
3582 3583
		df->s = cache_from_obj(s, object); /* Support for memcg */
	}
3584

3585 3586 3587 3588 3589
	/* Start new detached freelist */
	df->tail = object;
	df->freelist = object;
	df->cnt = 1;

3590 3591 3592 3593 3594 3595
	if (is_kfence_address(object))
		return size;

	set_freepointer(df->s, object, NULL);

	same = size;
3596 3597
	while (size) {
		object = p[--size];
3598 3599
		/* df->slab is always set at this point */
		if (df->slab == virt_to_slab(object)) {
3600
			/* Opportunity build freelist */
3601
			set_freepointer(df->s, object, df->freelist);
3602 3603
			df->freelist = object;
			df->cnt++;
3604 3605 3606
			same--;
			if (size != same)
				swap(p[size], p[same]);
3607
			continue;
3608
		}
3609 3610 3611 3612

		/* Limit look ahead search */
		if (!--lookahead)
			break;
3613
	}
3614

3615
	return same;
3616 3617 3618
}

/* Note that interrupts must be enabled when calling this function. */
3619
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
3620
{
3621
	if (!size)
3622 3623 3624 3625 3626 3627
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
3628
		if (!df.slab)
3629 3630
			continue;

3631 3632
		slab_free(df.s, df.slab, df.freelist, df.tail, &p[size], df.cnt,
			  _RET_IP_);
3633
	} while (likely(size));
3634 3635 3636
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

3637
/* Note that interrupts must be enabled when calling this function. */
3638 3639
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
			  void **p)
3640
{
3641 3642
	struct kmem_cache_cpu *c;
	int i;
3643
	struct obj_cgroup *objcg = NULL;
3644

3645
	/* memcg and kmem_cache debug support */
3646
	s = slab_pre_alloc_hook(s, NULL, &objcg, size, flags);
3647 3648
	if (unlikely(!s))
		return false;
3649 3650 3651 3652 3653
	/*
	 * Drain objects in the per cpu slab, while disabling local
	 * IRQs, which protects against PREEMPT and interrupts
	 * handlers invoking normal fastpath.
	 */
3654
	c = slub_get_cpu_ptr(s->cpu_slab);
3655
	local_lock_irq(&s->cpu_slab->lock);
3656 3657

	for (i = 0; i < size; i++) {
3658
		void *object = kfence_alloc(s, s->object_size, flags);
3659

3660 3661 3662 3663 3664 3665
		if (unlikely(object)) {
			p[i] = object;
			continue;
		}

		object = c->freelist;
3666
		if (unlikely(!object)) {
3667 3668 3669 3670 3671 3672 3673 3674 3675
			/*
			 * We may have removed an object from c->freelist using
			 * the fastpath in the previous iteration; in that case,
			 * c->tid has not been bumped yet.
			 * Since ___slab_alloc() may reenable interrupts while
			 * allocating memory, we should bump c->tid now.
			 */
			c->tid = next_tid(c->tid);

3676
			local_unlock_irq(&s->cpu_slab->lock);
3677

3678 3679 3680 3681
			/*
			 * Invoking slow path likely have side-effect
			 * of re-populating per CPU c->freelist
			 */
3682
			p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
3683
					    _RET_IP_, c);
3684 3685 3686
			if (unlikely(!p[i]))
				goto error;

3687
			c = this_cpu_ptr(s->cpu_slab);
3688 3689
			maybe_wipe_obj_freeptr(s, p[i]);

3690
			local_lock_irq(&s->cpu_slab->lock);
3691

3692 3693
			continue; /* goto for-loop */
		}
3694 3695
		c->freelist = get_freepointer(s, object);
		p[i] = object;
3696
		maybe_wipe_obj_freeptr(s, p[i]);
3697 3698
	}
	c->tid = next_tid(c->tid);
3699
	local_unlock_irq(&s->cpu_slab->lock);
3700
	slub_put_cpu_ptr(s->cpu_slab);
3701

3702 3703 3704 3705 3706 3707
	/*
	 * memcg and kmem_cache debug support and memory initialization.
	 * Done outside of the IRQ disabled fastpath loop.
	 */
	slab_post_alloc_hook(s, objcg, flags, size, p,
				slab_want_init_on_alloc(flags, s));
3708
	return i;
3709
error:
3710
	slub_put_cpu_ptr(s->cpu_slab);
3711
	slab_post_alloc_hook(s, objcg, flags, i, p, false);
3712
	kmem_cache_free_bulk(s, i, p);
3713
	return 0;
3714 3715 3716 3717
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk);


C
Christoph Lameter 已提交
3718
/*
C
Christoph Lameter 已提交
3719 3720 3721 3722
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
C
Christoph Lameter 已提交
3723 3724 3725 3726
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
C
Christoph Lameter 已提交
3727
 * must be moved on and off the partial lists and is therefore a factor in
C
Christoph Lameter 已提交
3728 3729 3730 3731
 * locking overhead.
 */

/*
I
Ingo Molnar 已提交
3732
 * Minimum / Maximum order of slab pages. This influences locking overhead
C
Christoph Lameter 已提交
3733 3734 3735 3736
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
3737 3738 3739
static unsigned int slub_min_order;
static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
static unsigned int slub_min_objects;
C
Christoph Lameter 已提交
3740 3741 3742 3743

/*
 * Calculate the order of allocation given an slab object size.
 *
C
Christoph Lameter 已提交
3744 3745 3746 3747
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
3748
 * unused space left. We go to a higher order if more than 1/16th of the slab
C
Christoph Lameter 已提交
3749 3750 3751 3752 3753 3754
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
C
Christoph Lameter 已提交
3755
 *
C
Christoph Lameter 已提交
3756 3757 3758 3759
 * slub_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slub_max_order then
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
C
Christoph Lameter 已提交
3760
 *
C
Christoph Lameter 已提交
3761 3762
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
3763
 * requested a higher minimum order then we start with that one instead of
C
Christoph Lameter 已提交
3764
 * the smallest order which will fit the object.
C
Christoph Lameter 已提交
3765
 */
3766
static inline unsigned int calc_slab_order(unsigned int size,
3767
		unsigned int min_objects, unsigned int max_order,
3768
		unsigned int fract_leftover)
C
Christoph Lameter 已提交
3769
{
3770 3771
	unsigned int min_order = slub_min_order;
	unsigned int order;
C
Christoph Lameter 已提交
3772

3773
	if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
3774
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;
3775

3776
	for (order = max(min_order, (unsigned int)get_order(min_objects * size));
3777
			order <= max_order; order++) {
C
Christoph Lameter 已提交
3778

3779 3780
		unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
		unsigned int rem;
C
Christoph Lameter 已提交
3781

3782
		rem = slab_size % size;
C
Christoph Lameter 已提交
3783

3784
		if (rem <= slab_size / fract_leftover)
C
Christoph Lameter 已提交
3785 3786
			break;
	}
C
Christoph Lameter 已提交
3787

C
Christoph Lameter 已提交
3788 3789 3790
	return order;
}

3791
static inline int calculate_order(unsigned int size)
3792
{
3793 3794 3795
	unsigned int order;
	unsigned int min_objects;
	unsigned int max_objects;
3796
	unsigned int nr_cpus;
3797 3798 3799 3800 3801 3802

	/*
	 * Attempt to find best configuration for a slab. This
	 * works by first attempting to generate a layout with
	 * the best configuration and backing off gradually.
	 *
3803
	 * First we increase the acceptable waste in a slab. Then
3804 3805 3806
	 * we reduce the minimum objects required in a slab.
	 */
	min_objects = slub_min_objects;
3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821
	if (!min_objects) {
		/*
		 * Some architectures will only update present cpus when
		 * onlining them, so don't trust the number if it's just 1. But
		 * we also don't want to use nr_cpu_ids always, as on some other
		 * architectures, there can be many possible cpus, but never
		 * onlined. Here we compromise between trying to avoid too high
		 * order on systems that appear larger than they are, and too
		 * low order on systems that appear smaller than they are.
		 */
		nr_cpus = num_present_cpus();
		if (nr_cpus <= 1)
			nr_cpus = nr_cpu_ids;
		min_objects = 4 * (fls(nr_cpus) + 1);
	}
3822
	max_objects = order_objects(slub_max_order, size);
3823 3824
	min_objects = min(min_objects, max_objects);

3825
	while (min_objects > 1) {
3826 3827
		unsigned int fraction;

3828
		fraction = 16;
3829
		while (fraction >= 4) {
3830
			order = calc_slab_order(size, min_objects,
3831
					slub_max_order, fraction);
3832 3833 3834 3835
			if (order <= slub_max_order)
				return order;
			fraction /= 2;
		}
3836
		min_objects--;
3837 3838 3839 3840 3841 3842
	}

	/*
	 * We were unable to place multiple objects in a slab. Now
	 * lets see if we can place a single object there.
	 */
3843
	order = calc_slab_order(size, 1, slub_max_order, 1);
3844 3845 3846 3847 3848 3849
	if (order <= slub_max_order)
		return order;

	/*
	 * Doh this slab cannot be placed using slub_max_order.
	 */
3850
	order = calc_slab_order(size, 1, MAX_ORDER, 1);
D
David Rientjes 已提交
3851
	if (order < MAX_ORDER)
3852 3853 3854 3855
		return order;
	return -ENOSYS;
}

3856
static void
3857
init_kmem_cache_node(struct kmem_cache_node *n)
C
Christoph Lameter 已提交
3858 3859 3860 3861
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
3862
#ifdef CONFIG_SLUB_DEBUG
3863
	atomic_long_set(&n->nr_slabs, 0);
3864
	atomic_long_set(&n->total_objects, 0);
3865
	INIT_LIST_HEAD(&n->full);
3866
#endif
C
Christoph Lameter 已提交
3867 3868
}

3869
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
3870
{
3871
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
3872
			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
3873

3874
	/*
3875 3876
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
3877
	 */
3878 3879
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));
3880 3881 3882 3883 3884

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
3885

3886
	return 1;
3887 3888
}

3889 3890
static struct kmem_cache *kmem_cache_node;

C
Christoph Lameter 已提交
3891 3892 3893 3894 3895
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
3896 3897
 * Note that this function only works on the kmem_cache_node
 * when allocating for the kmem_cache_node. This is used for bootstrapping
3898
 * memory on a fresh node that has no slab structures yet.
C
Christoph Lameter 已提交
3899
 */
3900
static void early_kmem_cache_node_alloc(int node)
C
Christoph Lameter 已提交
3901
{
3902
	struct slab *slab;
C
Christoph Lameter 已提交
3903 3904
	struct kmem_cache_node *n;

3905
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
C
Christoph Lameter 已提交
3906

3907
	slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
C
Christoph Lameter 已提交
3908

3909 3910
	BUG_ON(!slab);
	if (slab_nid(slab) != node) {
3911 3912
		pr_err("SLUB: Unable to allocate memory from node %d\n", node);
		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
3913 3914
	}

3915
	n = slab->freelist;
C
Christoph Lameter 已提交
3916
	BUG_ON(!n);
3917
#ifdef CONFIG_SLUB_DEBUG
3918
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
3919
	init_tracking(kmem_cache_node, n);
3920
#endif
3921
	n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
3922 3923 3924
	slab->freelist = get_freepointer(kmem_cache_node, n);
	slab->inuse = 1;
	slab->frozen = 0;
3925
	kmem_cache_node->node[node] = n;
3926
	init_kmem_cache_node(n);
3927
	inc_slabs_node(kmem_cache_node, node, slab->objects);
C
Christoph Lameter 已提交
3928

3929
	/*
3930 3931
	 * No locks need to be taken here as it has just been
	 * initialized and there is no concurrent access.
3932
	 */
3933
	__add_partial(n, slab, DEACTIVATE_TO_HEAD);
C
Christoph Lameter 已提交
3934 3935 3936 3937 3938
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;
3939
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
3940

3941
	for_each_kmem_cache_node(s, node, n) {
C
Christoph Lameter 已提交
3942
		s->node[node] = NULL;
3943
		kmem_cache_free(kmem_cache_node, n);
C
Christoph Lameter 已提交
3944 3945 3946
	}
}

3947 3948
void __kmem_cache_release(struct kmem_cache *s)
{
3949
	cache_random_seq_destroy(s);
3950 3951 3952 3953
	free_percpu(s->cpu_slab);
	free_kmem_cache_nodes(s);
}

3954
static int init_kmem_cache_nodes(struct kmem_cache *s)
C
Christoph Lameter 已提交
3955 3956 3957
{
	int node;

3958
	for_each_node_mask(node, slab_nodes) {
C
Christoph Lameter 已提交
3959 3960
		struct kmem_cache_node *n;

3961
		if (slab_state == DOWN) {
3962
			early_kmem_cache_node_alloc(node);
3963 3964
			continue;
		}
3965
		n = kmem_cache_alloc_node(kmem_cache_node,
3966
						GFP_KERNEL, node);
C
Christoph Lameter 已提交
3967

3968 3969 3970
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
C
Christoph Lameter 已提交
3971
		}
3972

3973
		init_kmem_cache_node(n);
3974
		s->node[node] = n;
C
Christoph Lameter 已提交
3975 3976 3977 3978
	}
	return 1;
}

3979 3980 3981
static void set_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
3982 3983
	unsigned int nr_objects;

3984 3985 3986 3987 3988 3989 3990 3991 3992
	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
3993 3994 3995
	 * For backwards compatibility reasons, this is determined as number
	 * of objects, even though we now limit maximum number of pages, see
	 * slub_set_cpu_partial()
3996 3997
	 */
	if (!kmem_cache_has_cpu_partial(s))
3998
		nr_objects = 0;
3999
	else if (s->size >= PAGE_SIZE)
4000
		nr_objects = 6;
4001
	else if (s->size >= 1024)
4002
		nr_objects = 24;
4003
	else if (s->size >= 256)
4004
		nr_objects = 52;
4005
	else
4006
		nr_objects = 120;
4007 4008

	slub_set_cpu_partial(s, nr_objects);
4009 4010 4011
#endif
}

C
Christoph Lameter 已提交
4012 4013 4014 4015
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
4016
static int calculate_sizes(struct kmem_cache *s)
C
Christoph Lameter 已提交
4017
{
4018
	slab_flags_t flags = s->flags;
4019
	unsigned int size = s->object_size;
4020
	unsigned int order;
C
Christoph Lameter 已提交
4021

4022 4023 4024 4025 4026 4027 4028 4029
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4030 4031 4032 4033 4034
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
4035
	if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
4036
			!s->ctor)
C
Christoph Lameter 已提交
4037 4038 4039 4040 4041 4042
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
C
Christoph Lameter 已提交
4043
	 * If we are Redzoning then check if there is some space between the
C
Christoph Lameter 已提交
4044
	 * end of the object and the free pointer. If not then add an
C
Christoph Lameter 已提交
4045
	 * additional word to have some bytes to store Redzone information.
C
Christoph Lameter 已提交
4046
	 */
4047
	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
C
Christoph Lameter 已提交
4048
		size += sizeof(void *);
4049
#endif
C
Christoph Lameter 已提交
4050 4051

	/*
C
Christoph Lameter 已提交
4052
	 * With that we have determined the number of bytes in actual use
4053
	 * by the object and redzoning.
C
Christoph Lameter 已提交
4054 4055 4056
	 */
	s->inuse = size;

4057 4058 4059
	if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
	    ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
	    s->ctor) {
C
Christoph Lameter 已提交
4060 4061 4062 4063 4064 4065
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
4066 4067
		 * destructor, are poisoning the objects, or are
		 * redzoning an object smaller than sizeof(void *).
4068 4069 4070 4071 4072
		 *
		 * The assumption that s->offset >= s->inuse means free
		 * pointer is outside of the object is used in the
		 * freeptr_outside_object() function. If that is no
		 * longer true, the function needs to be modified.
C
Christoph Lameter 已提交
4073 4074 4075
		 */
		s->offset = size;
		size += sizeof(void *);
4076
	} else {
4077 4078 4079 4080 4081
		/*
		 * Store freelist pointer near middle of object to keep
		 * it away from the edges of the object to avoid small
		 * sized over/underflows from neighboring allocations.
		 */
4082
		s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
C
Christoph Lameter 已提交
4083 4084
	}

4085
#ifdef CONFIG_SLUB_DEBUG
C
Christoph Lameter 已提交
4086 4087 4088 4089 4090 4091
	if (flags & SLAB_STORE_USER)
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);
4092
#endif
C
Christoph Lameter 已提交
4093

4094 4095
	kasan_cache_create(s, &size, &s->flags);
#ifdef CONFIG_SLUB_DEBUG
4096
	if (flags & SLAB_RED_ZONE) {
C
Christoph Lameter 已提交
4097 4098 4099 4100
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
4101
		 * corrupted if a user writes before the start
C
Christoph Lameter 已提交
4102 4103 4104
		 * of the object.
		 */
		size += sizeof(void *);
4105 4106 4107 4108 4109

		s->red_left_pad = sizeof(void *);
		s->red_left_pad = ALIGN(s->red_left_pad, s->align);
		size += s->red_left_pad;
	}
4110
#endif
C
Christoph Lameter 已提交
4111

C
Christoph Lameter 已提交
4112 4113 4114 4115 4116
	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
4117
	size = ALIGN(size, s->align);
C
Christoph Lameter 已提交
4118
	s->size = size;
4119
	s->reciprocal_size = reciprocal_value(size);
4120
	order = calculate_order(size);
C
Christoph Lameter 已提交
4121

4122
	if ((int)order < 0)
C
Christoph Lameter 已提交
4123 4124
		return 0;

4125
	s->allocflags = 0;
4126
	if (order)
4127 4128 4129
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
4130
		s->allocflags |= GFP_DMA;
4131

4132 4133 4134
	if (s->flags & SLAB_CACHE_DMA32)
		s->allocflags |= GFP_DMA32;

4135 4136 4137
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

C
Christoph Lameter 已提交
4138 4139 4140
	/*
	 * Determine the number of objects per slab
	 */
4141 4142
	s->oo = oo_make(order, size);
	s->min = oo_make(get_order(size), size);
C
Christoph Lameter 已提交
4143

4144
	return !!oo_objects(s->oo);
C
Christoph Lameter 已提交
4145 4146
}

4147
static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
C
Christoph Lameter 已提交
4148
{
4149
	s->flags = kmem_cache_flags(s->size, flags, s->name);
4150 4151 4152
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	s->random = get_random_long();
#endif
C
Christoph Lameter 已提交
4153

4154
	if (!calculate_sizes(s))
C
Christoph Lameter 已提交
4155
		goto error;
4156 4157 4158 4159 4160
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
4161
		if (get_order(s->size) > get_order(s->object_size)) {
4162 4163
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
4164
			if (!calculate_sizes(s))
4165 4166 4167
				goto error;
		}
	}
C
Christoph Lameter 已提交
4168

4169 4170
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
4171
	if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
4172 4173 4174 4175
		/* Enable fast mode */
		s->flags |= __CMPXCHG_DOUBLE;
#endif

4176
	/*
4177
	 * The larger the object size is, the more slabs we want on the partial
4178 4179
	 * list to avoid pounding the page allocator excessively.
	 */
4180 4181
	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
4182

4183
	set_cpu_partial(s);
4184

C
Christoph Lameter 已提交
4185
#ifdef CONFIG_NUMA
4186
	s->remote_node_defrag_ratio = 1000;
C
Christoph Lameter 已提交
4187
#endif
4188 4189 4190 4191 4192 4193 4194

	/* Initialize the pre-computed randomized freelist if slab is up */
	if (slab_state >= UP) {
		if (init_cache_random_seq(s))
			goto error;
	}

4195
	if (!init_kmem_cache_nodes(s))
4196
		goto error;
C
Christoph Lameter 已提交
4197

4198
	if (alloc_kmem_cache_cpus(s))
4199
		return 0;
4200

C
Christoph Lameter 已提交
4201
error:
4202
	__kmem_cache_release(s);
4203
	return -EINVAL;
C
Christoph Lameter 已提交
4204 4205
}

4206
static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
4207
			      const char *text)
4208 4209
{
#ifdef CONFIG_SLUB_DEBUG
4210
	void *addr = slab_address(slab);
4211
	unsigned long flags;
4212
	unsigned long *map;
4213
	void *p;
4214

4215 4216
	slab_err(s, slab, text, s->name);
	slab_lock(slab, &flags);
4217

4218 4219
	map = get_map(s, slab);
	for_each_object(p, s, addr, slab->objects) {
4220

4221
		if (!test_bit(__obj_to_index(s, addr, p), map)) {
4222
			pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
4223 4224 4225
			print_tracking(s, p);
		}
	}
4226
	put_map(map);
4227
	slab_unlock(slab, &flags);
4228 4229 4230
#endif
}

C
Christoph Lameter 已提交
4231
/*
4232
 * Attempt to free all partial slabs on a node.
4233 4234
 * This is called from __kmem_cache_shutdown(). We must take list_lock
 * because sysfs file might still access partial list after the shutdowning.
C
Christoph Lameter 已提交
4235
 */
4236
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
C
Christoph Lameter 已提交
4237
{
4238
	LIST_HEAD(discard);
4239
	struct slab *slab, *h;
C
Christoph Lameter 已提交
4240

4241 4242
	BUG_ON(irqs_disabled());
	spin_lock_irq(&n->list_lock);
4243 4244 4245 4246
	list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
		if (!slab->inuse) {
			remove_partial(n, slab);
			list_add(&slab->slab_list, &discard);
4247
		} else {
4248
			list_slab_objects(s, slab,
4249
			  "Objects remaining in %s on __kmem_cache_shutdown()");
4250
		}
4251
	}
4252
	spin_unlock_irq(&n->list_lock);
4253

4254 4255
	list_for_each_entry_safe(slab, h, &discard, slab_list)
		discard_slab(s, slab);
C
Christoph Lameter 已提交
4256 4257
}

4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268
bool __kmem_cache_empty(struct kmem_cache *s)
{
	int node;
	struct kmem_cache_node *n;

	for_each_kmem_cache_node(s, node, n)
		if (n->nr_partial || slabs_node(s, node))
			return false;
	return true;
}

C
Christoph Lameter 已提交
4269
/*
C
Christoph Lameter 已提交
4270
 * Release all resources used by a slab cache.
C
Christoph Lameter 已提交
4271
 */
4272
int __kmem_cache_shutdown(struct kmem_cache *s)
C
Christoph Lameter 已提交
4273 4274
{
	int node;
4275
	struct kmem_cache_node *n;
C
Christoph Lameter 已提交
4276

4277
	flush_all_cpus_locked(s);
C
Christoph Lameter 已提交
4278
	/* Attempt to free all objects */
4279
	for_each_kmem_cache_node(s, node, n) {
4280 4281
		free_partial(s, n);
		if (n->nr_partial || slabs_node(s, node))
C
Christoph Lameter 已提交
4282 4283 4284 4285 4286
			return 1;
	}
	return 0;
}

4287
#ifdef CONFIG_PRINTK
4288
void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
4289 4290 4291 4292 4293 4294
{
	void *base;
	int __maybe_unused i;
	unsigned int objnr;
	void *objp;
	void *objp0;
4295
	struct kmem_cache *s = slab->slab_cache;
4296 4297 4298
	struct track __maybe_unused *trackp;

	kpp->kp_ptr = object;
4299
	kpp->kp_slab = slab;
4300
	kpp->kp_slab_cache = s;
4301
	base = slab_address(slab);
4302 4303 4304 4305 4306 4307
	objp0 = kasan_reset_tag(object);
#ifdef CONFIG_SLUB_DEBUG
	objp = restore_red_left(s, objp0);
#else
	objp = objp0;
#endif
4308
	objnr = obj_to_index(s, slab, objp);
4309 4310 4311
	kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
	objp = base + s->size * objnr;
	kpp->kp_objp = objp;
4312 4313
	if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
			 || (objp - base) % s->size) ||
4314 4315 4316
	    !(s->flags & SLAB_STORE_USER))
		return;
#ifdef CONFIG_SLUB_DEBUG
4317
	objp = fixup_red_left(s, objp);
4318 4319
	trackp = get_track(s, objp, TRACK_ALLOC);
	kpp->kp_ret = (void *)trackp->addr;
4320 4321 4322 4323 4324
#ifdef CONFIG_STACKDEPOT
	{
		depot_stack_handle_t handle;
		unsigned long *entries;
		unsigned int nr_entries;
4325

4326 4327 4328 4329 4330 4331
		handle = READ_ONCE(trackp->handle);
		if (handle) {
			nr_entries = stack_depot_fetch(handle, &entries);
			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
				kpp->kp_stack[i] = (void *)entries[i];
		}
4332

4333 4334 4335 4336 4337 4338 4339
		trackp = get_track(s, objp, TRACK_FREE);
		handle = READ_ONCE(trackp->handle);
		if (handle) {
			nr_entries = stack_depot_fetch(handle, &entries);
			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
				kpp->kp_free_stack[i] = (void *)entries[i];
		}
4340
	}
4341 4342 4343
#endif
#endif
}
4344
#endif
4345

C
Christoph Lameter 已提交
4346 4347 4348 4349 4350 4351
/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

static int __init setup_slub_min_order(char *str)
{
4352
	get_option(&str, (int *)&slub_min_order);
C
Christoph Lameter 已提交
4353 4354 4355 4356 4357 4358 4359 4360

	return 1;
}

__setup("slub_min_order=", setup_slub_min_order);

static int __init setup_slub_max_order(char *str)
{
4361 4362
	get_option(&str, (int *)&slub_max_order);
	slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
C
Christoph Lameter 已提交
4363 4364 4365 4366 4367 4368 4369 4370

	return 1;
}

__setup("slub_max_order=", setup_slub_max_order);

static int __init setup_slub_min_objects(char *str)
{
4371
	get_option(&str, (int *)&slub_min_objects);
C
Christoph Lameter 已提交
4372 4373 4374 4375 4376 4377

	return 1;
}

__setup("slub_min_objects=", setup_slub_min_objects);

4378 4379
static __always_inline
void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
C
Christoph Lameter 已提交
4380
{
4381
	struct kmem_cache *s;
4382
	void *ret;
C
Christoph Lameter 已提交
4383

4384
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
4385
		ret = kmalloc_large_node_notrace(size, flags, node);
4386

4387
		trace_kmalloc_node(caller, ret, NULL,
4388 4389
				   size, PAGE_SIZE << get_order(size),
				   flags, node);
4390 4391 4392

		return ret;
	}
4393

4394
	s = kmalloc_slab(size, flags);
4395 4396

	if (unlikely(ZERO_OR_NULL_PTR(s)))
4397 4398
		return s;

4399
	ret = slab_alloc_node(s, NULL, flags, node, caller, size);
4400

4401
	trace_kmalloc_node(caller, ret, s, size, s->size, flags, node);
4402

4403
	ret = kasan_kmalloc(s, ret, size, flags);
4404

4405
	return ret;
C
Christoph Lameter 已提交
4406
}
4407 4408 4409 4410 4411

void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
	return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
C
Christoph Lameter 已提交
4412 4413
EXPORT_SYMBOL(__kmalloc_node);

4414 4415 4416 4417 4418 4419 4420
void *__kmalloc(size_t size, gfp_t flags)
{
	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
}
EXPORT_SYMBOL(__kmalloc);


4421 4422
#ifdef CONFIG_HARDENED_USERCOPY
/*
4423 4424 4425
 * Rejects incorrectly sized objects and objects that are to be copied
 * to/from userspace but do not fall entirely within the containing slab
 * cache's usercopy region.
4426 4427 4428 4429
 *
 * Returns NULL if check passes, otherwise const char * to name of cache
 * to indicate an error.
 */
4430 4431
void __check_heap_object(const void *ptr, unsigned long n,
			 const struct slab *slab, bool to_user)
4432 4433
{
	struct kmem_cache *s;
4434
	unsigned int offset;
4435
	bool is_kfence = is_kfence_address(ptr);
4436

4437 4438
	ptr = kasan_reset_tag(ptr);

4439
	/* Find object and usable object size. */
4440
	s = slab->slab_cache;
4441 4442

	/* Reject impossible pointers. */
4443
	if (ptr < slab_address(slab))
4444 4445
		usercopy_abort("SLUB object not in SLUB page?!", NULL,
			       to_user, 0, n);
4446 4447

	/* Find offset within object. */
4448 4449 4450
	if (is_kfence)
		offset = ptr - kfence_object_start(ptr);
	else
4451
		offset = (ptr - slab_address(slab)) % s->size;
4452 4453

	/* Adjust for redzone and reject if within the redzone. */
4454
	if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
4455
		if (offset < s->red_left_pad)
4456 4457
			usercopy_abort("SLUB object in left red zone",
				       s->name, to_user, offset, n);
4458 4459 4460
		offset -= s->red_left_pad;
	}

4461 4462 4463 4464
	/* Allow address range falling entirely within usercopy region. */
	if (offset >= s->useroffset &&
	    offset - s->useroffset <= s->usersize &&
	    n <= s->useroffset - offset + s->usersize)
4465
		return;
4466

4467
	usercopy_abort("SLUB object", s->name, to_user, offset, n);
4468 4469 4470
}
#endif /* CONFIG_HARDENED_USERCOPY */

4471
size_t __ksize(const void *object)
C
Christoph Lameter 已提交
4472
{
4473
	struct folio *folio;
C
Christoph Lameter 已提交
4474

4475
	if (unlikely(object == ZERO_SIZE_PTR))
4476 4477
		return 0;

4478
	folio = virt_to_folio(object);
4479

4480 4481
	if (unlikely(!folio_test_slab(folio)))
		return folio_size(folio);
C
Christoph Lameter 已提交
4482

4483
	return slab_ksize(folio_slab(folio)->slab_cache);
C
Christoph Lameter 已提交
4484
}
4485
EXPORT_SYMBOL(__ksize);
C
Christoph Lameter 已提交
4486 4487 4488

void kfree(const void *x)
{
4489 4490
	struct folio *folio;
	struct slab *slab;
4491
	void *object = (void *)x;
C
Christoph Lameter 已提交
4492

4493 4494
	trace_kfree(_RET_IP_, x);

4495
	if (unlikely(ZERO_OR_NULL_PTR(x)))
C
Christoph Lameter 已提交
4496 4497
		return;

4498 4499 4500
	folio = virt_to_folio(x);
	if (unlikely(!folio_test_slab(folio))) {
		free_large_kmalloc(folio, object);
4501 4502
		return;
	}
4503
	slab = folio_slab(folio);
4504
	slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);
C
Christoph Lameter 已提交
4505 4506 4507
}
EXPORT_SYMBOL(kfree);

4508 4509
#define SHRINK_PROMOTE_MAX 32

4510
/*
4511 4512 4513
 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
 * up most to the head of the partial lists. New allocations will then
 * fill those up and thus they can be removed from the partial lists.
C
Christoph Lameter 已提交
4514 4515 4516 4517
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
4518
 */
4519
static int __kmem_cache_do_shrink(struct kmem_cache *s)
4520 4521 4522 4523
{
	int node;
	int i;
	struct kmem_cache_node *n;
4524 4525
	struct slab *slab;
	struct slab *t;
4526 4527
	struct list_head discard;
	struct list_head promote[SHRINK_PROMOTE_MAX];
4528
	unsigned long flags;
4529
	int ret = 0;
4530

4531
	for_each_kmem_cache_node(s, node, n) {
4532 4533 4534
		INIT_LIST_HEAD(&discard);
		for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
			INIT_LIST_HEAD(promote + i);
4535 4536 4537 4538

		spin_lock_irqsave(&n->list_lock, flags);

		/*
4539
		 * Build lists of slabs to discard or promote.
4540
		 *
C
Christoph Lameter 已提交
4541
		 * Note that concurrent frees may occur while we hold the
4542
		 * list_lock. slab->inuse here is the upper limit.
4543
		 */
4544 4545
		list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
			int free = slab->objects - slab->inuse;
4546

4547
			/* Do not reread slab->inuse */
4548 4549 4550 4551 4552
			barrier();

			/* We do not keep full slabs on the list */
			BUG_ON(free <= 0);

4553 4554
			if (free == slab->objects) {
				list_move(&slab->slab_list, &discard);
4555
				n->nr_partial--;
4556
			} else if (free <= SHRINK_PROMOTE_MAX)
4557
				list_move(&slab->slab_list, promote + free - 1);
4558 4559 4560
		}

		/*
4561 4562
		 * Promote the slabs filled up most to the head of the
		 * partial list.
4563
		 */
4564 4565
		for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
			list_splice(promote + i, &n->partial);
4566 4567

		spin_unlock_irqrestore(&n->list_lock, flags);
4568 4569

		/* Release empty slabs */
4570 4571
		list_for_each_entry_safe(slab, t, &discard, slab_list)
			discard_slab(s, slab);
4572 4573 4574

		if (slabs_node(s, node))
			ret = 1;
4575 4576
	}

4577
	return ret;
4578 4579
}

4580 4581 4582 4583 4584 4585
int __kmem_cache_shrink(struct kmem_cache *s)
{
	flush_all(s);
	return __kmem_cache_do_shrink(s);
}

4586 4587 4588 4589
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

4590
	mutex_lock(&slab_mutex);
4591 4592 4593 4594
	list_for_each_entry(s, &slab_caches, list) {
		flush_all_cpus_locked(s);
		__kmem_cache_do_shrink(s);
	}
4595
	mutex_unlock(&slab_mutex);
4596 4597 4598 4599 4600 4601 4602 4603 4604

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct memory_notify *marg = arg;
	int offline_node;

4605
	offline_node = marg->status_change_nid_normal;
4606 4607 4608 4609 4610 4611 4612 4613

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

4614
	mutex_lock(&slab_mutex);
4615
	node_clear(offline_node, slab_nodes);
4616 4617 4618 4619 4620
	/*
	 * We no longer free kmem_cache_node structures here, as it would be
	 * racy with all get_node() users, and infeasible to protect them with
	 * slab_mutex.
	 */
4621
	mutex_unlock(&slab_mutex);
4622 4623 4624 4625 4626 4627 4628
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
4629
	int nid = marg->status_change_nid_normal;
4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
4640
	 * We are bringing a node online. No memory is available yet. We must
4641 4642 4643
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
4644
	mutex_lock(&slab_mutex);
4645
	list_for_each_entry(s, &slab_caches, list) {
4646 4647 4648 4649 4650 4651
		/*
		 * The structure may already exist if the node was previously
		 * onlined and offlined.
		 */
		if (get_node(s, nid))
			continue;
4652 4653 4654 4655 4656
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
4657
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
4658 4659 4660 4661
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
4662
		init_kmem_cache_node(n);
4663 4664
		s->node[nid] = n;
	}
4665 4666 4667 4668 4669
	/*
	 * Any cache created after this point will also have kmem_cache_node
	 * initialized for the new node.
	 */
	node_set(nid, slab_nodes);
4670
out:
4671
	mutex_unlock(&slab_mutex);
4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
4695 4696 4697 4698
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
4699 4700 4701
	return ret;
}

4702 4703 4704 4705
static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};
4706

C
Christoph Lameter 已提交
4707 4708 4709 4710
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

4711 4712
/*
 * Used for early kmem_cache structures that were allocated using
4713 4714
 * the page allocator. Allocate them properly then fix up the pointers
 * that may be pointing to the wrong kmem_cache structure.
4715 4716
 */

4717
static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
4718 4719
{
	int node;
4720
	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
4721
	struct kmem_cache_node *n;
4722

4723
	memcpy(s, static_cache, kmem_cache->object_size);
4724

4725 4726 4727 4728 4729 4730
	/*
	 * This runs very early, and only the boot processor is supposed to be
	 * up.  Even if it weren't true, IRQs are not up so we couldn't fire
	 * IPIs around.
	 */
	__flush_cpu_slab(s, smp_processor_id());
4731
	for_each_kmem_cache_node(s, node, n) {
4732
		struct slab *p;
4733

4734
		list_for_each_entry(p, &n->partial, slab_list)
4735
			p->slab_cache = s;
4736

4737
#ifdef CONFIG_SLUB_DEBUG
4738
		list_for_each_entry(p, &n->full, slab_list)
4739
			p->slab_cache = s;
4740 4741
#endif
	}
4742 4743
	list_add(&s->list, &slab_caches);
	return s;
4744 4745
}

C
Christoph Lameter 已提交
4746 4747
void __init kmem_cache_init(void)
{
4748 4749
	static __initdata struct kmem_cache boot_kmem_cache,
		boot_kmem_cache_node;
4750
	int node;
4751

4752 4753 4754
	if (debug_guardpage_minorder())
		slub_max_order = 0;

4755 4756 4757 4758
	/* Print slub debugging pointers without hashing */
	if (__slub_debug_enabled())
		no_hash_pointers_enable(NULL);

4759 4760
	kmem_cache_node = &boot_kmem_cache_node;
	kmem_cache = &boot_kmem_cache;
4761

4762 4763 4764 4765 4766 4767 4768
	/*
	 * Initialize the nodemask for which we will allocate per node
	 * structures. Here we don't need taking slab_mutex yet.
	 */
	for_each_node_state(node, N_NORMAL_MEMORY)
		node_set(node, slab_nodes);

4769
	create_boot_cache(kmem_cache_node, "kmem_cache_node",
4770
		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
4771

4772
	register_hotmemory_notifier(&slab_memory_callback_nb);
C
Christoph Lameter 已提交
4773 4774 4775 4776

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

4777 4778 4779
	create_boot_cache(kmem_cache, "kmem_cache",
			offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *),
4780
		       SLAB_HWCACHE_ALIGN, 0, 0);
4781

4782 4783
	kmem_cache = bootstrap(&boot_kmem_cache);
	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
4784 4785

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
4786
	setup_kmalloc_cache_index_table();
4787
	create_kmalloc_caches(0);
C
Christoph Lameter 已提交
4788

4789 4790 4791
	/* Setup random freelists for each cache */
	init_freelist_randomization();

4792 4793
	cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
				  slub_cpu_dead);
C
Christoph Lameter 已提交
4794

4795
	pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
4796
		cache_line_size(),
C
Christoph Lameter 已提交
4797 4798 4799 4800
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

4801 4802 4803 4804
void __init kmem_cache_init_late(void)
{
}

4805
struct kmem_cache *
4806
__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
4807
		   slab_flags_t flags, void (*ctor)(void *))
C
Christoph Lameter 已提交
4808
{
4809
	struct kmem_cache *s;
C
Christoph Lameter 已提交
4810

4811
	s = find_mergeable(size, align, flags, name, ctor);
C
Christoph Lameter 已提交
4812
	if (s) {
4813 4814 4815
		if (sysfs_slab_alias(s, name))
			return NULL;

C
Christoph Lameter 已提交
4816
		s->refcount++;
4817

C
Christoph Lameter 已提交
4818 4819 4820 4821
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
4822
		s->object_size = max(s->object_size, size);
4823
		s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
4824
	}
C
Christoph Lameter 已提交
4825

4826 4827
	return s;
}
4828

4829
int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
4830
{
4831 4832 4833 4834 4835
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;
4836

4837 4838 4839 4840
	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

4841
	err = sysfs_slab_add(s);
4842
	if (err) {
4843
		__kmem_cache_release(s);
4844 4845
		return err;
	}
4846

4847 4848 4849
	if (s->flags & SLAB_STORE_USER)
		debugfs_slab_add(s);

4850
	return 0;
C
Christoph Lameter 已提交
4851 4852 4853
}

void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4854
				  int node, unsigned long caller)
C
Christoph Lameter 已提交
4855
{
4856
	return __do_kmalloc_node(size, gfpflags, node, caller);
C
Christoph Lameter 已提交
4857
}
4858
EXPORT_SYMBOL(__kmalloc_node_track_caller);
C
Christoph Lameter 已提交
4859

4860
#ifdef CONFIG_SYSFS
4861
static int count_inuse(struct slab *slab)
4862
{
4863
	return slab->inuse;
4864 4865
}

4866
static int count_total(struct slab *slab)
4867
{
4868
	return slab->objects;
4869
}
4870
#endif
4871

4872
#ifdef CONFIG_SLUB_DEBUG
4873
static void validate_slab(struct kmem_cache *s, struct slab *slab,
4874
			  unsigned long *obj_map)
4875 4876
{
	void *p;
4877
	void *addr = slab_address(slab);
4878
	unsigned long flags;
4879

4880
	slab_lock(slab, &flags);
4881

4882
	if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
4883
		goto unlock;
4884 4885

	/* Now we know that a valid freelist exists */
4886 4887
	__fill_map(obj_map, s, slab);
	for_each_object(p, s, addr, slab->objects) {
4888
		u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
4889
			 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
4890

4891
		if (!check_object(s, slab, p, val))
4892 4893
			break;
	}
4894
unlock:
4895
	slab_unlock(slab, &flags);
4896 4897
}

4898
static int validate_slab_node(struct kmem_cache *s,
4899
		struct kmem_cache_node *n, unsigned long *obj_map)
4900 4901
{
	unsigned long count = 0;
4902
	struct slab *slab;
4903 4904 4905 4906
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

4907 4908
	list_for_each_entry(slab, &n->partial, slab_list) {
		validate_slab(s, slab, obj_map);
4909 4910
		count++;
	}
4911
	if (count != n->nr_partial) {
4912 4913
		pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
		       s->name, count, n->nr_partial);
4914 4915
		slab_add_kunit_errors();
	}
4916 4917 4918 4919

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

4920 4921
	list_for_each_entry(slab, &n->full, slab_list) {
		validate_slab(s, slab, obj_map);
4922 4923
		count++;
	}
4924
	if (count != atomic_long_read(&n->nr_slabs)) {
4925 4926
		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
		       s->name, count, atomic_long_read(&n->nr_slabs));
4927 4928
		slab_add_kunit_errors();
	}
4929 4930 4931 4932 4933 4934

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

4935
long validate_slab_cache(struct kmem_cache *s)
4936 4937 4938
{
	int node;
	unsigned long count = 0;
4939
	struct kmem_cache_node *n;
4940 4941 4942 4943 4944
	unsigned long *obj_map;

	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
	if (!obj_map)
		return -ENOMEM;
4945 4946

	flush_all(s);
4947
	for_each_kmem_cache_node(s, node, n)
4948 4949 4950
		count += validate_slab_node(s, n, obj_map);

	bitmap_free(obj_map);
4951

4952 4953
	return count;
}
4954 4955
EXPORT_SYMBOL(validate_slab_cache);

4956
#ifdef CONFIG_DEBUG_FS
4957
/*
C
Christoph Lameter 已提交
4958
 * Generate lists of code addresses where slabcache objects are allocated
4959 4960 4961 4962
 * and freed.
 */

struct location {
4963
	depot_stack_handle_t handle;
4964
	unsigned long count;
4965
	unsigned long addr;
4966 4967 4968 4969 4970
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
R
Rusty Russell 已提交
4971
	DECLARE_BITMAP(cpus, NR_CPUS);
4972
	nodemask_t nodes;
4973 4974 4975 4976 4977 4978
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
4979
	loff_t idx;
4980 4981
};

4982 4983
static struct dentry *slab_debugfs_root;

4984 4985 4986 4987 4988 4989 4990
static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

4991
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4992 4993 4994 4995 4996 4997
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

4998
	l = (void *)__get_free_pages(flags, order);
4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
5012
				const struct track *track)
5013 5014 5015
{
	long start, end, pos;
	struct location *l;
5016
	unsigned long caddr, chandle;
5017
	unsigned long age = jiffies - track->when;
5018
	depot_stack_handle_t handle = 0;
5019

5020 5021 5022
#ifdef CONFIG_STACKDEPOT
	handle = READ_ONCE(track->handle);
#endif
5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036
	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

		caddr = t->loc[pos].addr;
5037 5038
		chandle = t->loc[pos].handle;
		if ((track->addr == caddr) && (handle == chandle)) {
5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053

			l = &t->loc[pos];
			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

R
Rusty Russell 已提交
5054 5055
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
5056 5057
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
5058 5059 5060
			return 1;
		}

5061
		if (track->addr < caddr)
5062
			end = pos;
5063 5064
		else if (track->addr == caddr && handle < chandle)
			end = pos;
5065 5066 5067 5068 5069
		else
			start = pos;
	}

	/*
C
Christoph Lameter 已提交
5070
	 * Not found. Insert new tracking element.
5071
	 */
5072
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
5073 5074 5075 5076 5077 5078 5079 5080
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
5081 5082 5083 5084 5085 5086
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
5087
	l->handle = handle;
R
Rusty Russell 已提交
5088 5089
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
5090 5091
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
5092 5093 5094 5095
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
5096
		struct slab *slab, enum track_item alloc,
5097
		unsigned long *obj_map)
5098
{
5099
	void *addr = slab_address(slab);
5100 5101
	void *p;

5102
	__fill_map(obj_map, s, slab);
5103

5104
	for_each_object(p, s, addr, slab->objects)
5105
		if (!test_bit(__obj_to_index(s, addr, p), obj_map))
5106
			add_location(t, s, get_track(s, p, alloc));
5107
}
5108
#endif  /* CONFIG_DEBUG_FS   */
5109
#endif	/* CONFIG_SLUB_DEBUG */
5110

5111
#ifdef CONFIG_SYSFS
C
Christoph Lameter 已提交
5112
enum slab_stat_type {
5113 5114 5115 5116 5117
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
C
Christoph Lameter 已提交
5118 5119
};

5120
#define SO_ALL		(1 << SL_ALL)
C
Christoph Lameter 已提交
5121 5122 5123
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
5124
#define SO_TOTAL	(1 << SL_TOTAL)
C
Christoph Lameter 已提交
5125

5126
static ssize_t show_slab_objects(struct kmem_cache *s,
5127
				 char *buf, unsigned long flags)
C
Christoph Lameter 已提交
5128 5129 5130 5131 5132
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;
5133
	int len = 0;
C
Christoph Lameter 已提交
5134

5135
	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
5136 5137
	if (!nodes)
		return -ENOMEM;
C
Christoph Lameter 已提交
5138

5139 5140
	if (flags & SO_CPU) {
		int cpu;
C
Christoph Lameter 已提交
5141

5142
		for_each_possible_cpu(cpu) {
5143 5144
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
							       cpu);
5145
			int node;
5146
			struct slab *slab;
5147

5148 5149
			slab = READ_ONCE(c->slab);
			if (!slab)
5150
				continue;
5151

5152
			node = slab_nid(slab);
5153
			if (flags & SO_TOTAL)
5154
				x = slab->objects;
5155
			else if (flags & SO_OBJECTS)
5156
				x = slab->inuse;
5157 5158
			else
				x = 1;
5159

5160 5161 5162
			total += x;
			nodes[node] += x;

5163
#ifdef CONFIG_SLUB_CPU_PARTIAL
5164 5165 5166
			slab = slub_percpu_partial_read_once(c);
			if (slab) {
				node = slab_nid(slab);
5167 5168 5169 5170 5171
				if (flags & SO_TOTAL)
					WARN_ON_ONCE(1);
				else if (flags & SO_OBJECTS)
					WARN_ON_ONCE(1);
				else
5172
					x = slab->slabs;
5173 5174
				total += x;
				nodes[node] += x;
5175
			}
5176
#endif
C
Christoph Lameter 已提交
5177 5178 5179
		}
	}

5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190
	/*
	 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
	 * already held which will conflict with an existing lock order:
	 *
	 * mem_hotplug_lock->slab_mutex->kernfs_mutex
	 *
	 * We don't really need mem_hotplug_lock (to hold off
	 * slab_mem_going_offline_callback) here because slab's memory hot
	 * unplug code doesn't destroy the kmem_cache->node[] data.
	 */

5191
#ifdef CONFIG_SLUB_DEBUG
5192
	if (flags & SO_ALL) {
5193 5194 5195
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {
5196

5197 5198 5199 5200 5201
			if (flags & SO_TOTAL)
				x = atomic_long_read(&n->total_objects);
			else if (flags & SO_OBJECTS)
				x = atomic_long_read(&n->total_objects) -
					count_partial(n, count_free);
C
Christoph Lameter 已提交
5202
			else
5203
				x = atomic_long_read(&n->nr_slabs);
C
Christoph Lameter 已提交
5204 5205 5206 5207
			total += x;
			nodes[node] += x;
		}

5208 5209 5210
	} else
#endif
	if (flags & SO_PARTIAL) {
5211
		struct kmem_cache_node *n;
C
Christoph Lameter 已提交
5212

5213
		for_each_kmem_cache_node(s, node, n) {
5214 5215 5216 5217
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
C
Christoph Lameter 已提交
5218
			else
5219
				x = n->nr_partial;
C
Christoph Lameter 已提交
5220 5221 5222 5223
			total += x;
			nodes[node] += x;
		}
	}
5224 5225

	len += sysfs_emit_at(buf, len, "%lu", total);
C
Christoph Lameter 已提交
5226
#ifdef CONFIG_NUMA
5227
	for (node = 0; node < nr_node_ids; node++) {
C
Christoph Lameter 已提交
5228
		if (nodes[node])
5229 5230 5231
			len += sysfs_emit_at(buf, len, " N%d=%lu",
					     node, nodes[node]);
	}
C
Christoph Lameter 已提交
5232
#endif
5233
	len += sysfs_emit_at(buf, len, "\n");
C
Christoph Lameter 已提交
5234
	kfree(nodes);
5235 5236

	return len;
C
Christoph Lameter 已提交
5237 5238 5239
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
5240
#define to_slab(n) container_of(n, struct kmem_cache, kobj)
C
Christoph Lameter 已提交
5241 5242 5243 5244 5245 5246 5247 5248

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
5249
	static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
C
Christoph Lameter 已提交
5250 5251

#define SLAB_ATTR(_name) \
5252
	static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
C
Christoph Lameter 已提交
5253 5254 5255

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
5256
	return sysfs_emit(buf, "%u\n", s->size);
C
Christoph Lameter 已提交
5257 5258 5259 5260 5261
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
5262
	return sysfs_emit(buf, "%u\n", s->align);
C
Christoph Lameter 已提交
5263 5264 5265 5266 5267
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
5268
	return sysfs_emit(buf, "%u\n", s->object_size);
C
Christoph Lameter 已提交
5269 5270 5271 5272 5273
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
5274
	return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
C
Christoph Lameter 已提交
5275 5276 5277 5278 5279
}
SLAB_ATTR_RO(objs_per_slab);

static ssize_t order_show(struct kmem_cache *s, char *buf)
{
5280
	return sysfs_emit(buf, "%u\n", oo_order(s->oo));
C
Christoph Lameter 已提交
5281
}
5282
SLAB_ATTR_RO(order);
C
Christoph Lameter 已提交
5283

5284 5285
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
5286
	return sysfs_emit(buf, "%lu\n", s->min_partial);
5287 5288 5289 5290 5291 5292 5293 5294
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

5295
	err = kstrtoul(buf, 10, &min);
5296 5297 5298
	if (err)
		return err;

5299
	s->min_partial = min;
5300 5301 5302 5303
	return length;
}
SLAB_ATTR(min_partial);

5304 5305
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
5306 5307 5308 5309 5310 5311
	unsigned int nr_partial = 0;
#ifdef CONFIG_SLUB_CPU_PARTIAL
	nr_partial = s->cpu_partial;
#endif

	return sysfs_emit(buf, "%u\n", nr_partial);
5312 5313 5314 5315 5316
}

static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
5317
	unsigned int objects;
5318 5319
	int err;

5320
	err = kstrtouint(buf, 10, &objects);
5321 5322
	if (err)
		return err;
5323
	if (objects && !kmem_cache_has_cpu_partial(s))
5324
		return -EINVAL;
5325

5326
	slub_set_cpu_partial(s, objects);
5327 5328 5329 5330 5331
	flush_all(s);
	return length;
}
SLAB_ATTR(cpu_partial);

C
Christoph Lameter 已提交
5332 5333
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
5334 5335
	if (!s->ctor)
		return 0;
5336
	return sysfs_emit(buf, "%pS\n", s->ctor);
C
Christoph Lameter 已提交
5337 5338 5339 5340 5341
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
5342
	return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
C
Christoph Lameter 已提交
5343 5344 5345 5346 5347
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
5348
	return show_slab_objects(s, buf, SO_PARTIAL);
C
Christoph Lameter 已提交
5349 5350 5351 5352 5353
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
5354
	return show_slab_objects(s, buf, SO_CPU);
C
Christoph Lameter 已提交
5355 5356 5357 5358 5359
}
SLAB_ATTR_RO(cpu_slabs);

static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
5360
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
C
Christoph Lameter 已提交
5361 5362 5363
}
SLAB_ATTR_RO(objects);

5364 5365 5366 5367 5368 5369
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

5370 5371 5372
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
5373
	int slabs = 0;
5374
	int cpu __maybe_unused;
5375
	int len = 0;
5376

5377
#ifdef CONFIG_SLUB_CPU_PARTIAL
5378
	for_each_online_cpu(cpu) {
5379
		struct slab *slab;
5380

5381
		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
5382

5383 5384
		if (slab)
			slabs += slab->slabs;
5385
	}
5386
#endif
5387

5388
	/* Approximate half-full slabs, see slub_set_cpu_partial() */
5389 5390
	objects = (slabs * oo_objects(s->oo)) / 2;
	len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
5391

5392
#if defined(CONFIG_SLUB_CPU_PARTIAL) && defined(CONFIG_SMP)
5393
	for_each_online_cpu(cpu) {
5394
		struct slab *slab;
5395

5396 5397 5398 5399
		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
		if (slab) {
			slabs = READ_ONCE(slab->slabs);
			objects = (slabs * oo_objects(s->oo)) / 2;
5400
			len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
5401
					     cpu, objects, slabs);
5402
		}
5403 5404
	}
#endif
5405 5406 5407
	len += sysfs_emit_at(buf, len, "\n");

	return len;
5408 5409 5410
}
SLAB_ATTR_RO(slabs_cpu_partial);

5411 5412
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
5413
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
5414
}
5415
SLAB_ATTR_RO(reclaim_account);
5416 5417 5418

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
5419
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
5420 5421 5422 5423 5424 5425
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
5426
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
5427 5428 5429 5430
}
SLAB_ATTR_RO(cache_dma);
#endif

5431 5432
static ssize_t usersize_show(struct kmem_cache *s, char *buf)
{
5433
	return sysfs_emit(buf, "%u\n", s->usersize);
5434 5435 5436
}
SLAB_ATTR_RO(usersize);

5437 5438
static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
5439
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
5440 5441 5442
}
SLAB_ATTR_RO(destroy_by_rcu);

5443
#ifdef CONFIG_SLUB_DEBUG
5444 5445 5446 5447 5448 5449
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

5450 5451 5452 5453 5454 5455
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

C
Christoph Lameter 已提交
5456 5457
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
5458
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
C
Christoph Lameter 已提交
5459
}
5460
SLAB_ATTR_RO(sanity_checks);
C
Christoph Lameter 已提交
5461 5462 5463

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
5464
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
C
Christoph Lameter 已提交
5465
}
5466
SLAB_ATTR_RO(trace);
C
Christoph Lameter 已提交
5467 5468 5469

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
5470
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
C
Christoph Lameter 已提交
5471 5472
}

5473
SLAB_ATTR_RO(red_zone);
C
Christoph Lameter 已提交
5474 5475 5476

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
5477
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
C
Christoph Lameter 已提交
5478 5479
}

5480
SLAB_ATTR_RO(poison);
C
Christoph Lameter 已提交
5481 5482 5483

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
5484
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
C
Christoph Lameter 已提交
5485 5486
}

5487
SLAB_ATTR_RO(store_user);
C
Christoph Lameter 已提交
5488

5489 5490 5491 5492 5493 5494 5495 5496
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
5497 5498 5499 5500 5501 5502 5503 5504
	int ret = -EINVAL;

	if (buf[0] == '1') {
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
5505 5506
}
SLAB_ATTR(validate);
5507 5508 5509 5510 5511 5512

#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
5513
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
5514
}
5515
SLAB_ATTR_RO(failslab);
5516
#endif
5517

5518 5519 5520 5521 5522 5523 5524 5525
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
5526
	if (buf[0] == '1')
5527
		kmem_cache_shrink(s);
5528
	else
5529 5530 5531 5532 5533
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

C
Christoph Lameter 已提交
5534
#ifdef CONFIG_NUMA
5535
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
C
Christoph Lameter 已提交
5536
{
5537
	return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
C
Christoph Lameter 已提交
5538 5539
}

5540
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
C
Christoph Lameter 已提交
5541 5542
				const char *buf, size_t length)
{
5543
	unsigned int ratio;
5544 5545
	int err;

5546
	err = kstrtouint(buf, 10, &ratio);
5547 5548
	if (err)
		return err;
5549 5550
	if (ratio > 100)
		return -ERANGE;
5551

5552
	s->remote_node_defrag_ratio = ratio * 10;
C
Christoph Lameter 已提交
5553 5554 5555

	return length;
}
5556
SLAB_ATTR(remote_node_defrag_ratio);
C
Christoph Lameter 已提交
5557 5558
#endif

5559 5560 5561 5562 5563
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
5564
	int len = 0;
5565
	int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
5566 5567 5568 5569 5570

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
5571
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5572 5573 5574 5575 5576

		data[cpu] = x;
		sum += x;
	}

5577
	len += sysfs_emit_at(buf, len, "%lu", sum);
5578

5579
#ifdef CONFIG_SMP
5580
	for_each_online_cpu(cpu) {
5581 5582 5583
		if (data[cpu])
			len += sysfs_emit_at(buf, len, " C%d=%u",
					     cpu, data[cpu]);
5584
	}
5585
#endif
5586
	kfree(data);
5587 5588 5589
	len += sysfs_emit_at(buf, len, "\n");

	return len;
5590 5591
}

5592 5593 5594 5595 5596
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
5597
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
5598 5599
}

5600 5601 5602 5603 5604
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
5605 5606 5607 5608 5609 5610 5611 5612 5613
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
5625
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5626 5627 5628 5629 5630 5631 5632
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5633
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5634
STAT_ATTR(ORDER_FALLBACK, order_fallback);
5635 5636
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5637 5638
STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5639 5640
STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5641
#endif	/* CONFIG_SLUB_STATS */
5642

5643
static struct attribute *slab_attrs[] = {
C
Christoph Lameter 已提交
5644 5645 5646 5647
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
5648
	&min_partial_attr.attr,
5649
	&cpu_partial_attr.attr,
C
Christoph Lameter 已提交
5650
	&objects_attr.attr,
5651
	&objects_partial_attr.attr,
C
Christoph Lameter 已提交
5652 5653 5654 5655 5656 5657 5658 5659
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
5660
	&shrink_attr.attr,
5661
	&slabs_cpu_partial_attr.attr,
5662
#ifdef CONFIG_SLUB_DEBUG
5663 5664 5665 5666
	&total_objects_attr.attr,
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
C
Christoph Lameter 已提交
5667 5668 5669
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
5670
	&validate_attr.attr,
5671
#endif
C
Christoph Lameter 已提交
5672 5673 5674 5675
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
5676
	&remote_node_defrag_ratio_attr.attr,
5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
5689
	&alloc_node_mismatch_attr.attr,
5690 5691 5692 5693 5694 5695 5696
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
5697
	&deactivate_bypass_attr.attr,
5698
	&order_fallback_attr.attr,
5699 5700
	&cmpxchg_double_fail_attr.attr,
	&cmpxchg_double_cpu_fail_attr.attr,
5701 5702
	&cpu_partial_alloc_attr.attr,
	&cpu_partial_free_attr.attr,
5703 5704
	&cpu_partial_node_attr.attr,
	&cpu_partial_drain_attr.attr,
C
Christoph Lameter 已提交
5705
#endif
5706 5707 5708
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif
5709
	&usersize_attr.attr,
5710

C
Christoph Lameter 已提交
5711 5712 5713
	NULL
};

5714
static const struct attribute_group slab_attr_group = {
C
Christoph Lameter 已提交
5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

	err = attribute->show(s, buf);

	return err;
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;
	int err;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

	err = attribute->store(s, buf, len);
	return err;
}

5755 5756 5757 5758 5759
static void kmem_cache_release(struct kobject *k)
{
	slab_kmem_cache_release(to_slab(k));
}

5760
static const struct sysfs_ops slab_sysfs_ops = {
C
Christoph Lameter 已提交
5761 5762 5763 5764 5765 5766
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static struct kobj_type slab_ktype = {
	.sysfs_ops = &slab_sysfs_ops,
5767
	.release = kmem_cache_release,
C
Christoph Lameter 已提交
5768 5769
};

5770
static struct kset *slab_kset;
C
Christoph Lameter 已提交
5771

5772 5773 5774 5775 5776
static inline struct kset *cache_kset(struct kmem_cache *s)
{
	return slab_kset;
}

C
Christoph Lameter 已提交
5777 5778 5779
#define ID_STR_LENGTH 64

/* Create a unique string id for a slab cache:
C
Christoph Lameter 已提交
5780 5781
 *
 * Format	:[flags-]size
C
Christoph Lameter 已提交
5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

	BUG_ON(!name);

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
5800 5801
	if (s->flags & SLAB_CACHE_DMA32)
		*p++ = 'D';
C
Christoph Lameter 已提交
5802 5803
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
5804
	if (s->flags & SLAB_CONSISTENCY_CHECKS)
C
Christoph Lameter 已提交
5805
		*p++ = 'F';
5806 5807
	if (s->flags & SLAB_ACCOUNT)
		*p++ = 'A';
C
Christoph Lameter 已提交
5808 5809
	if (p != name + 1)
		*p++ = '-';
5810
	p += sprintf(p, "%07u", s->size);
5811

C
Christoph Lameter 已提交
5812 5813 5814 5815 5816 5817 5818 5819
	BUG_ON(p > name + ID_STR_LENGTH - 1);
	return name;
}

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
5820
	struct kset *kset = cache_kset(s);
5821
	int unmergeable = slab_unmergeable(s);
C
Christoph Lameter 已提交
5822

5823 5824 5825 5826 5827
	if (!kset) {
		kobject_init(&s->kobj, &slab_ktype);
		return 0;
	}

5828 5829 5830 5831
	if (!unmergeable && disable_higher_order_debug &&
			(slub_debug & DEBUG_METADATA_FLAGS))
		unmergeable = 1;

C
Christoph Lameter 已提交
5832 5833 5834 5835 5836 5837
	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
5838
		sysfs_remove_link(&slab_kset->kobj, s->name);
C
Christoph Lameter 已提交
5839 5840 5841 5842 5843 5844 5845 5846 5847
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
	}

5848
	s->kobj.kset = kset;
5849
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
5850
	if (err)
5851
		goto out;
C
Christoph Lameter 已提交
5852 5853

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
5854 5855
	if (err)
		goto out_del_kobj;
5856

C
Christoph Lameter 已提交
5857 5858 5859 5860
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
	}
5861 5862 5863 5864 5865 5866 5867
out:
	if (!unmergeable)
		kfree(name);
	return err;
out_del_kobj:
	kobject_del(&s->kobj);
	goto out;
C
Christoph Lameter 已提交
5868 5869
}

5870 5871 5872 5873 5874 5875
void sysfs_slab_unlink(struct kmem_cache *s)
{
	if (slab_state >= FULL)
		kobject_del(&s->kobj);
}

5876 5877 5878 5879
void sysfs_slab_release(struct kmem_cache *s)
{
	if (slab_state >= FULL)
		kobject_put(&s->kobj);
C
Christoph Lameter 已提交
5880 5881 5882 5883
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
5884
 * available lest we lose that information.
C
Christoph Lameter 已提交
5885 5886 5887 5888 5889 5890 5891
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

A
Adrian Bunk 已提交
5892
static struct saved_alias *alias_list;
C
Christoph Lameter 已提交
5893 5894 5895 5896 5897

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

5898
	if (slab_state == FULL) {
C
Christoph Lameter 已提交
5899 5900 5901
		/*
		 * If we have a leftover link then remove it.
		 */
5902 5903
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
C
Christoph Lameter 已提交
5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
	return 0;
}

static int __init slab_sysfs_init(void)
{
5919
	struct kmem_cache *s;
C
Christoph Lameter 已提交
5920 5921
	int err;

5922
	mutex_lock(&slab_mutex);
5923

5924
	slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
5925
	if (!slab_kset) {
5926
		mutex_unlock(&slab_mutex);
5927
		pr_err("Cannot register slab subsystem.\n");
C
Christoph Lameter 已提交
5928 5929 5930
		return -ENOSYS;
	}

5931
	slab_state = FULL;
5932

5933
	list_for_each_entry(s, &slab_caches, list) {
5934
		err = sysfs_slab_add(s);
5935
		if (err)
5936 5937
			pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
			       s->name);
5938
	}
C
Christoph Lameter 已提交
5939 5940 5941 5942 5943 5944

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
5945
		if (err)
5946 5947
			pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
			       al->name);
C
Christoph Lameter 已提交
5948 5949 5950
		kfree(al);
	}

5951
	mutex_unlock(&slab_mutex);
C
Christoph Lameter 已提交
5952 5953 5954 5955
	return 0;
}

__initcall(slab_sysfs_init);
5956
#endif /* CONFIG_SYSFS */
5957

5958 5959 5960 5961
#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
static int slab_debugfs_show(struct seq_file *seq, void *v)
{
	struct loc_track *t = seq->private;
5962 5963
	struct location *l;
	unsigned long idx;
5964

5965
	idx = (unsigned long) t->idx;
5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996
	if (idx < t->count) {
		l = &t->loc[idx];

		seq_printf(seq, "%7ld ", l->count);

		if (l->addr)
			seq_printf(seq, "%pS", (void *)l->addr);
		else
			seq_puts(seq, "<not-available>");

		if (l->sum_time != l->min_time) {
			seq_printf(seq, " age=%ld/%llu/%ld",
				l->min_time, div_u64(l->sum_time, l->count),
				l->max_time);
		} else
			seq_printf(seq, " age=%ld", l->min_time);

		if (l->min_pid != l->max_pid)
			seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
		else
			seq_printf(seq, " pid=%ld",
				l->min_pid);

		if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
			seq_printf(seq, " cpus=%*pbl",
				 cpumask_pr_args(to_cpumask(l->cpus)));

		if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
			seq_printf(seq, " nodes=%*pbl",
				 nodemask_pr_args(&l->nodes));

5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011
#ifdef CONFIG_STACKDEPOT
		{
			depot_stack_handle_t handle;
			unsigned long *entries;
			unsigned int nr_entries, j;

			handle = READ_ONCE(l->handle);
			if (handle) {
				nr_entries = stack_depot_fetch(handle, &entries);
				seq_puts(seq, "\n");
				for (j = 0; j < nr_entries; j++)
					seq_printf(seq, "        %pS\n", (void *)entries[j]);
			}
		}
#endif
6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028
		seq_puts(seq, "\n");
	}

	if (!idx && !t->count)
		seq_puts(seq, "No data\n");

	return 0;
}

static void slab_debugfs_stop(struct seq_file *seq, void *v)
{
}

static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
{
	struct loc_track *t = seq->private;

6029
	t->idx = ++(*ppos);
6030
	if (*ppos <= t->count)
6031
		return ppos;
6032 6033 6034 6035

	return NULL;
}

6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046
static int cmp_loc_by_count(const void *a, const void *b, const void *data)
{
	struct location *loc1 = (struct location *)a;
	struct location *loc2 = (struct location *)b;

	if (loc1->count > loc2->count)
		return -1;
	else
		return 1;
}

6047 6048
static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
{
6049 6050 6051
	struct loc_track *t = seq->private;

	t->idx = *ppos;
6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070
	return ppos;
}

static const struct seq_operations slab_debugfs_sops = {
	.start  = slab_debugfs_start,
	.next   = slab_debugfs_next,
	.stop   = slab_debugfs_stop,
	.show   = slab_debugfs_show,
};

static int slab_debug_trace_open(struct inode *inode, struct file *filep)
{

	struct kmem_cache_node *n;
	enum track_item alloc;
	int node;
	struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
						sizeof(struct loc_track));
	struct kmem_cache *s = file_inode(filep)->i_private;
6071 6072
	unsigned long *obj_map;

6073 6074 6075
	if (!t)
		return -ENOMEM;

6076
	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
6077 6078
	if (!obj_map) {
		seq_release_private(inode, filep);
6079
		return -ENOMEM;
6080
	}
6081 6082 6083 6084 6085 6086

	if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
		alloc = TRACK_ALLOC;
	else
		alloc = TRACK_FREE;

6087 6088
	if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
		bitmap_free(obj_map);
6089
		seq_release_private(inode, filep);
6090
		return -ENOMEM;
6091
	}
6092 6093 6094

	for_each_kmem_cache_node(s, node, n) {
		unsigned long flags;
6095
		struct slab *slab;
6096 6097 6098 6099 6100

		if (!atomic_long_read(&n->nr_slabs))
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
6101 6102 6103 6104
		list_for_each_entry(slab, &n->partial, slab_list)
			process_slab(t, s, slab, alloc, obj_map);
		list_for_each_entry(slab, &n->full, slab_list)
			process_slab(t, s, slab, alloc, obj_map);
6105 6106 6107
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

6108 6109 6110 6111
	/* Sort locations by count */
	sort_r(t->loc, t->count, sizeof(struct location),
		cmp_loc_by_count, NULL, NULL);

6112
	bitmap_free(obj_map);
6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167
	return 0;
}

static int slab_debug_trace_release(struct inode *inode, struct file *file)
{
	struct seq_file *seq = file->private_data;
	struct loc_track *t = seq->private;

	free_loc_track(t);
	return seq_release_private(inode, file);
}

static const struct file_operations slab_debugfs_fops = {
	.open    = slab_debug_trace_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.release = slab_debug_trace_release,
};

static void debugfs_slab_add(struct kmem_cache *s)
{
	struct dentry *slab_cache_dir;

	if (unlikely(!slab_debugfs_root))
		return;

	slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);

	debugfs_create_file("alloc_traces", 0400,
		slab_cache_dir, s, &slab_debugfs_fops);

	debugfs_create_file("free_traces", 0400,
		slab_cache_dir, s, &slab_debugfs_fops);
}

void debugfs_slab_release(struct kmem_cache *s)
{
	debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
}

static int __init slab_debugfs_init(void)
{
	struct kmem_cache *s;

	slab_debugfs_root = debugfs_create_dir("slab", NULL);

	list_for_each_entry(s, &slab_caches, list)
		if (s->flags & SLAB_STORE_USER)
			debugfs_slab_add(s);

	return 0;

}
__initcall(slab_debugfs_init);
#endif
6168 6169 6170
/*
 * The /proc/slabinfo ABI
 */
6171
#ifdef CONFIG_SLUB_DEBUG
6172
void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
6173 6174
{
	unsigned long nr_slabs = 0;
6175 6176
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
6177
	int node;
6178
	struct kmem_cache_node *n;
6179

6180
	for_each_kmem_cache_node(s, node, n) {
6181 6182
		nr_slabs += node_nr_slabs(n);
		nr_objs += node_nr_objs(n);
6183
		nr_free += count_partial(n, count_free);
6184 6185
	}

6186 6187 6188 6189 6190 6191
	sinfo->active_objs = nr_objs - nr_free;
	sinfo->num_objs = nr_objs;
	sinfo->active_slabs = nr_slabs;
	sinfo->num_slabs = nr_slabs;
	sinfo->objects_per_slab = oo_objects(s->oo);
	sinfo->cache_order = oo_order(s->oo);
6192 6193
}

6194
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
6195 6196 6197
{
}

6198 6199
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos)
6200
{
6201
	return -EIO;
6202
}
6203
#endif /* CONFIG_SLUB_DEBUG */
新手
引导
客服 返回
顶部