slab.h 21.9 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4 5 6 7
#ifndef MM_SLAB_H
#define MM_SLAB_H
/*
 * Internal slab definitions
 */

8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
/* Reuses the bits in struct page */
struct slab {
	unsigned long __page_flags;
	union {
		struct list_head slab_list;
		struct {	/* Partial pages */
			struct slab *next;
#ifdef CONFIG_64BIT
			int slabs;	/* Nr of slabs left */
#else
			short int slabs;
#endif
		};
		struct rcu_head rcu_head;
	};
	struct kmem_cache *slab_cache; /* not slob */
	/* Double-word boundary */
	void *freelist;		/* first free object */
	union {
		void *s_mem;	/* slab: first object */
		unsigned long counters;		/* SLUB */
		struct {			/* SLUB */
			unsigned inuse:16;
			unsigned objects:15;
			unsigned frozen:1;
		};
	};

	union {
		unsigned int active;		/* SLAB */
		int units;			/* SLOB */
	};
	atomic_t __page_refcount;
#ifdef CONFIG_MEMCG
	unsigned long memcg_data;
#endif
};

#define SLAB_MATCH(pg, sl)						\
	static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
SLAB_MATCH(flags, __page_flags);
SLAB_MATCH(compound_head, slab_list);	/* Ensure bit 0 is clear */
SLAB_MATCH(slab_list, slab_list);
SLAB_MATCH(rcu_head, rcu_head);
SLAB_MATCH(slab_cache, slab_cache);
SLAB_MATCH(s_mem, s_mem);
SLAB_MATCH(active, active);
SLAB_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
SLAB_MATCH(memcg_data, memcg_data);
#endif
#undef SLAB_MATCH
static_assert(sizeof(struct slab) <= sizeof(struct page));

/**
 * folio_slab - Converts from folio to slab.
 * @folio: The folio.
 *
 * Currently struct slab is a different representation of a folio where
 * folio_test_slab() is true.
 *
 * Return: The slab which contains this folio.
 */
#define folio_slab(folio)	(_Generic((folio),			\
	const struct folio *:	(const struct slab *)(folio),		\
	struct folio *:		(struct slab *)(folio)))

/**
 * slab_folio - The folio allocated for a slab
 * @slab: The slab.
 *
 * Slabs are allocated as folios that contain the individual objects and are
 * using some fields in the first struct page of the folio - those fields are
 * now accessed by struct slab. It is occasionally necessary to convert back to
 * a folio in order to communicate with the rest of the mm.  Please use this
 * helper function instead of casting yourself, as the implementation may change
 * in the future.
 */
#define slab_folio(s)		(_Generic((s),				\
	const struct slab *:	(const struct folio *)s,		\
	struct slab *:		(struct folio *)s))

/**
 * page_slab - Converts from first struct page to slab.
 * @p: The first (either head of compound or single) page of slab.
 *
 * A temporary wrapper to convert struct page to struct slab in situations where
 * we know the page is the compound head, or single order-0 page.
 *
 * Long-term ideally everything would work with struct slab directly or go
 * through folio to struct slab.
 *
 * Return: The slab which contains this page
 */
#define page_slab(p)		(_Generic((p),				\
	const struct page *:	(const struct slab *)(p),		\
	struct page *:		(struct slab *)(p)))

/**
 * slab_page - The first struct page allocated for a slab
 * @slab: The slab.
 *
 * A convenience wrapper for converting slab to the first struct page of the
 * underlying folio, to communicate with code not yet converted to folio or
 * struct slab.
 */
#define slab_page(s) folio_page(slab_folio(s), 0)

/*
 * If network-based swap is enabled, sl*b must keep track of whether pages
 * were allocated from pfmemalloc reserves.
 */
static inline bool slab_test_pfmemalloc(const struct slab *slab)
{
	return folio_test_active((struct folio *)slab_folio(slab));
}

static inline void slab_set_pfmemalloc(struct slab *slab)
{
	folio_set_active(slab_folio(slab));
}

static inline void slab_clear_pfmemalloc(struct slab *slab)
{
	folio_clear_active(slab_folio(slab));
}

static inline void __slab_clear_pfmemalloc(struct slab *slab)
{
	__folio_clear_active(slab_folio(slab));
}

static inline void *slab_address(const struct slab *slab)
{
	return folio_address(slab_folio(slab));
}

static inline int slab_nid(const struct slab *slab)
{
	return folio_nid(slab_folio(slab));
}

static inline pg_data_t *slab_pgdat(const struct slab *slab)
{
	return folio_pgdat(slab_folio(slab));
}

static inline struct slab *virt_to_slab(const void *addr)
{
	struct folio *folio = virt_to_folio(addr);

	if (!folio_test_slab(folio))
		return NULL;

	return folio_slab(folio);
}

static inline int slab_order(const struct slab *slab)
{
	return folio_order((struct folio *)slab_folio(slab));
}

static inline size_t slab_size(const struct slab *slab)
{
	return PAGE_SIZE << slab_order(slab);
}

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
#ifdef CONFIG_SLOB
/*
 * Common fields provided in kmem_cache by all slab allocators
 * This struct is either used directly by the allocator (SLOB)
 * or the allocator must include definitions for all fields
 * provided in kmem_cache_common in their definition of kmem_cache.
 *
 * Once we can do anonymous structs (C11 standard) we could put a
 * anonymous struct definition in these allocators so that the
 * separate allocations in the kmem_cache structure of SLAB and
 * SLUB is no longer needed.
 */
struct kmem_cache {
	unsigned int object_size;/* The original size of the object */
	unsigned int size;	/* The aligned/padded/added on size  */
	unsigned int align;	/* Alignment as calculated */
191
	slab_flags_t flags;	/* Active flags on the slab */
192 193
	unsigned int useroffset;/* Usercopy region offset */
	unsigned int usersize;	/* Usercopy region size */
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
	const char *name;	/* Slab name for sysfs */
	int refcount;		/* Use counter */
	void (*ctor)(void *);	/* Called on object slot creation */
	struct list_head list;	/* List of all slab caches on the system */
};

#endif /* CONFIG_SLOB */

#ifdef CONFIG_SLAB
#include <linux/slab_def.h>
#endif

#ifdef CONFIG_SLUB
#include <linux/slub_def.h>
#endif

#include <linux/memcontrol.h>
211 212 213
#include <linux/fault-inject.h>
#include <linux/kasan.h>
#include <linux/kmemleak.h>
214
#include <linux/random.h>
215
#include <linux/sched/mm.h>
216

217 218 219 220 221 222 223 224 225 226 227
/*
 * State of the slab allocator.
 *
 * This is used to describe the states of the allocator during bootup.
 * Allocators use this to gradually bootstrap themselves. Most allocators
 * have the problem that the structures used for managing slab caches are
 * allocated from slab caches themselves.
 */
enum slab_state {
	DOWN,			/* No slab functionality yet */
	PARTIAL,		/* SLUB: kmem_cache_node available */
228
	PARTIAL_NODE,		/* SLAB: kmalloc size for node struct available */
229 230 231 232 233 234
	UP,			/* Slab caches usable but not all extras yet */
	FULL			/* Everything is working */
};

extern enum slab_state slab_state;

235 236
/* The slab cache mutex protects the management structures during changes */
extern struct mutex slab_mutex;
237 238

/* The list of all slab caches on the system */
239 240
extern struct list_head slab_caches;

241 242 243
/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;

244 245
/* A table of kmalloc cache names and sizes */
extern const struct kmalloc_info_struct {
246
	const char *name[NR_KMALLOC_TYPES];
247
	unsigned int size;
248 249
} kmalloc_info[];

250 251
#ifndef CONFIG_SLOB
/* Kmalloc array related functions */
252
void setup_kmalloc_cache_index_table(void);
253
void create_kmalloc_caches(slab_flags_t);
254 255 256

/* Find the kmalloc slab corresponding for a certain size */
struct kmem_cache *kmalloc_slab(size_t, gfp_t);
257 258
#endif

259
gfp_t kmalloc_fix_flags(gfp_t flags);
260

261
/* Functions provided by the slab allocators */
262
int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
263

264 265 266
struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
			slab_flags_t flags, unsigned int useroffset,
			unsigned int usersize);
267
extern void create_boot_cache(struct kmem_cache *, const char *name,
268 269
			unsigned int size, slab_flags_t flags,
			unsigned int useroffset, unsigned int usersize);
270

271
int slab_unmergeable(struct kmem_cache *s);
272
struct kmem_cache *find_mergeable(unsigned size, unsigned align,
273
		slab_flags_t flags, const char *name, void (*ctor)(void *));
J
Joonsoo Kim 已提交
274
#ifndef CONFIG_SLOB
275
struct kmem_cache *
276
__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
277
		   slab_flags_t flags, void (*ctor)(void *));
278

279
slab_flags_t kmem_cache_flags(unsigned int object_size,
280
	slab_flags_t flags, const char *name);
281
#else
282
static inline struct kmem_cache *
283
__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
284
		   slab_flags_t flags, void (*ctor)(void *))
285
{ return NULL; }
286

287
static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
288
	slab_flags_t flags, const char *name)
289 290 291
{
	return flags;
}
292 293 294
#endif


295
/* Legal flag mask for kmem_cache_create(), for various configurations */
296 297
#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
			 SLAB_CACHE_DMA32 | SLAB_PANIC | \
298
			 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
299 300 301 302 303

#if defined(CONFIG_DEBUG_SLAB)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
#elif defined(CONFIG_SLUB_DEBUG)
#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
304
			  SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
305 306 307 308 309 310
#else
#define SLAB_DEBUG_FLAGS (0)
#endif

#if defined(CONFIG_SLAB)
#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
311
			  SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
312
			  SLAB_ACCOUNT)
313 314
#elif defined(CONFIG_SLUB)
#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
315
			  SLAB_TEMPORARY | SLAB_ACCOUNT)
316
#else
317
#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE)
318 319
#endif

320
/* Common flags available with current configuration */
321 322
#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)

323 324 325 326 327 328 329 330 331 332 333 334 335
/* Common flags permitted for kmem_cache_create */
#define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
			      SLAB_RED_ZONE | \
			      SLAB_POISON | \
			      SLAB_STORE_USER | \
			      SLAB_TRACE | \
			      SLAB_CONSISTENCY_CHECKS | \
			      SLAB_MEM_SPREAD | \
			      SLAB_NOLEAKTRACE | \
			      SLAB_RECLAIM_ACCOUNT | \
			      SLAB_TEMPORARY | \
			      SLAB_ACCOUNT)

336
bool __kmem_cache_empty(struct kmem_cache *);
337
int __kmem_cache_shutdown(struct kmem_cache *);
338
void __kmem_cache_release(struct kmem_cache *);
339
int __kmem_cache_shrink(struct kmem_cache *);
340
void slab_kmem_cache_release(struct kmem_cache *);
341

342 343 344
struct seq_file;
struct file;

345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
struct slabinfo {
	unsigned long active_objs;
	unsigned long num_objs;
	unsigned long active_slabs;
	unsigned long num_slabs;
	unsigned long shared_avail;
	unsigned int limit;
	unsigned int batchcount;
	unsigned int shared;
	unsigned int objects_per_slab;
	unsigned int cache_order;
};

void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
360 361
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos);
362

363 364 365
/*
 * Generic implementation of bulk operations
 * These are useful for situations in which the allocator cannot
366
 * perform optimizations. In that case segments of the object listed
367 368 369
 * may be allocated or freed using these operations.
 */
void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
370
int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
371

372
static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
373 374
{
	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
375
		NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
376 377
}

378 379 380 381 382 383 384
#ifdef CONFIG_SLUB_DEBUG
#ifdef CONFIG_SLUB_DEBUG_ON
DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
#else
DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
#endif
extern void print_tracking(struct kmem_cache *s, void *object);
385
long validate_slab_cache(struct kmem_cache *s);
386 387 388 389
static inline bool __slub_debug_enabled(void)
{
	return static_branch_unlikely(&slub_debug_enabled);
}
390 391 392 393
#else
static inline void print_tracking(struct kmem_cache *s, void *object)
{
}
394 395 396 397
static inline bool __slub_debug_enabled(void)
{
	return false;
}
398 399 400 401 402 403 404 405 406
#endif

/*
 * Returns true if any of the specified slub_debug flags is enabled for the
 * cache. Use only for flags parsed by setup_slub_debug() as it also enables
 * the static key.
 */
static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
{
407 408 409
	if (IS_ENABLED(CONFIG_SLUB_DEBUG))
		VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
	if (__slub_debug_enabled())
410 411 412 413
		return s->flags & flags;
	return false;
}

414
#ifdef CONFIG_MEMCG_KMEM
415
int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
416
				 gfp_t gfp, bool new_page);
417 418
void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,
		     enum node_stat_item idx, int nr);
419 420 421

static inline void memcg_free_page_obj_cgroups(struct page *page)
{
422
	kfree(page_objcgs(page));
423
	page->memcg_data = 0;
424 425
}

426 427 428 429 430 431 432 433 434
static inline size_t obj_full_size(struct kmem_cache *s)
{
	/*
	 * For each accounted object there is an extra space which is used
	 * to store obj_cgroup membership. Charge it too.
	 */
	return s->size + sizeof(struct obj_cgroup *);
}

435 436 437 438 439 440
/*
 * Returns false if the allocation should fail.
 */
static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
					     struct obj_cgroup **objcgp,
					     size_t objects, gfp_t flags)
441
{
442 443
	struct obj_cgroup *objcg;

444 445 446 447 448 449
	if (!memcg_kmem_enabled())
		return true;

	if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
		return true;

450 451
	objcg = get_obj_cgroup_from_current();
	if (!objcg)
452
		return true;
453 454 455

	if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
		obj_cgroup_put(objcg);
456
		return false;
457 458
	}

459 460
	*objcgp = objcg;
	return true;
461 462
}

463 464
static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
					      struct obj_cgroup *objcg,
465 466
					      gfp_t flags, size_t size,
					      void **p)
467 468 469 470 471
{
	struct page *page;
	unsigned long off;
	size_t i;

472
	if (!memcg_kmem_enabled() || !objcg)
473 474
		return;

475 476 477
	for (i = 0; i < size; i++) {
		if (likely(p[i])) {
			page = virt_to_head_page(p[i]);
478

479
			if (!page_objcgs(page) &&
480 481
			    memcg_alloc_page_obj_cgroups(page, s, flags,
							 false)) {
482 483 484 485
				obj_cgroup_uncharge(objcg, obj_full_size(s));
				continue;
			}

486 487
			off = obj_to_index(s, page, p[i]);
			obj_cgroup_get(objcg);
488
			page_objcgs(page)[off] = objcg;
489 490 491 492
			mod_objcg_state(objcg, page_pgdat(page),
					cache_vmstat_idx(s), obj_full_size(s));
		} else {
			obj_cgroup_uncharge(objcg, obj_full_size(s));
493 494 495 496 497
		}
	}
	obj_cgroup_put(objcg);
}

498 499
static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
					void **p, int objects)
500
{
501
	struct kmem_cache *s;
502
	struct obj_cgroup **objcgs;
503
	struct obj_cgroup *objcg;
504
	struct page *page;
505
	unsigned int off;
506
	int i;
507

508 509 510
	if (!memcg_kmem_enabled())
		return;

511 512 513
	for (i = 0; i < objects; i++) {
		if (unlikely(!p[i]))
			continue;
514

515
		page = virt_to_head_page(p[i]);
516
		objcgs = page_objcgs_check(page);
517
		if (!objcgs)
518
			continue;
519

520 521 522 523
		if (!s_orig)
			s = page->slab_cache;
		else
			s = s_orig;
524

525
		off = obj_to_index(s, page, p[i]);
526
		objcg = objcgs[off];
527 528
		if (!objcg)
			continue;
529

530
		objcgs[off] = NULL;
531 532 533 534 535
		obj_cgroup_uncharge(objcg, obj_full_size(s));
		mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),
				-obj_full_size(s));
		obj_cgroup_put(objcg);
	}
536 537
}

538
#else /* CONFIG_MEMCG_KMEM */
539
static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
540 541 542 543
{
	return NULL;
}

544
static inline int memcg_alloc_page_obj_cgroups(struct page *page,
545 546
					       struct kmem_cache *s, gfp_t gfp,
					       bool new_page)
547 548 549 550 551 552 553 554
{
	return 0;
}

static inline void memcg_free_page_obj_cgroups(struct page *page)
{
}

555 556 557
static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
					     struct obj_cgroup **objcgp,
					     size_t objects, gfp_t flags)
558
{
559
	return true;
560 561
}

562 563
static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
					      struct obj_cgroup *objcg,
564 565
					      gfp_t flags, size_t size,
					      void **p)
566 567 568
{
}

569 570
static inline void memcg_slab_free_hook(struct kmem_cache *s,
					void **p, int objects)
571 572
{
}
573
#endif /* CONFIG_MEMCG_KMEM */
574

575 576
static inline struct kmem_cache *virt_to_cache(const void *obj)
{
577
	struct slab *slab;
578

579 580
	slab = virt_to_slab(obj);
	if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n",
581 582
					__func__))
		return NULL;
583
	return slab->slab_cache;
584 585
}

586 587
static __always_inline void account_slab(struct slab *slab, int order,
					 struct kmem_cache *s, gfp_t gfp)
588
{
589
	if (memcg_kmem_enabled() && (s->flags & SLAB_ACCOUNT))
590
		memcg_alloc_page_obj_cgroups(slab_page(slab), s, gfp, true);
591

592
	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
593
			    PAGE_SIZE << order);
594 595
}

596 597
static __always_inline void unaccount_slab(struct slab *slab, int order,
					   struct kmem_cache *s)
598
{
599
	if (memcg_kmem_enabled())
600
		memcg_free_page_obj_cgroups(slab_page(slab));
601

602
	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
603
			    -(PAGE_SIZE << order));
604 605
}

606 607 608 609 610 611 612 613 614
static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
{
	struct kmem_cache *cachep;

	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
	    !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
		return s;

	cachep = virt_to_cache(x);
615
	if (WARN(cachep && cachep != s,
616 617 618 619 620 621
		  "%s: Wrong slab cache. %s but object is from %s\n",
		  __func__, s->name, cachep->name))
		print_tracking(cachep, x);
	return cachep;
}

622 623 624 625 626 627 628 629 630 631 632 633 634 635
static inline size_t slab_ksize(const struct kmem_cache *s)
{
#ifndef CONFIG_SLUB
	return s->object_size;

#else /* CONFIG_SLUB */
# ifdef CONFIG_SLUB_DEBUG
	/*
	 * Debugging requires use of the padding between object
	 * and whatever may come after it.
	 */
	if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
		return s->object_size;
# endif
636 637
	if (s->flags & SLAB_KASAN)
		return s->object_size;
638 639 640 641 642
	/*
	 * If we have the need to store the freelist pointer
	 * back there or track user information then we can
	 * only use the space before that information.
	 */
643
	if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
644 645 646 647 648 649 650 651 652
		return s->inuse;
	/*
	 * Else we can use all the padding etc for the allocation
	 */
	return s->size;
#endif
}

static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
653 654
						     struct obj_cgroup **objcgp,
						     size_t size, gfp_t flags)
655 656
{
	flags &= gfp_allowed_mask;
657

658
	might_alloc(flags);
659

660
	if (should_failslab(s, flags))
661 662
		return NULL;

663 664
	if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))
		return NULL;
665 666

	return s;
667 668
}

669
static inline void slab_post_alloc_hook(struct kmem_cache *s,
670 671
					struct obj_cgroup *objcg, gfp_t flags,
					size_t size, void **p, bool init)
672 673 674 675
{
	size_t i;

	flags &= gfp_allowed_mask;
676 677 678 679 680 681 682 683

	/*
	 * As memory initialization might be integrated into KASAN,
	 * kasan_slab_alloc and initialization memset must be
	 * kept together to avoid discrepancies in behavior.
	 *
	 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
	 */
684
	for (i = 0; i < size; i++) {
685 686 687
		p[i] = kasan_slab_alloc(s, p[i], flags, init);
		if (p[i] && init && !kasan_has_integrated_init())
			memset(p[i], 0, s->object_size);
688
		kmemleak_alloc_recursive(p[i], s->object_size, 1,
689 690
					 s->flags, flags);
	}
691

692
	memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
693 694
}

695
#ifndef CONFIG_SLOB
696 697 698 699 700 701 702 703 704 705
/*
 * The slab lists for all objects.
 */
struct kmem_cache_node {
	spinlock_t list_lock;

#ifdef CONFIG_SLAB
	struct list_head slabs_partial;	/* partial list first, better asm code */
	struct list_head slabs_full;
	struct list_head slabs_free;
706 707
	unsigned long total_slabs;	/* length of all slab lists */
	unsigned long free_slabs;	/* length of free slab list only */
708 709 710 711
	unsigned long free_objects;
	unsigned int free_limit;
	unsigned int colour_next;	/* Per-node cache coloring */
	struct array_cache *shared;	/* shared per node */
J
Joonsoo Kim 已提交
712
	struct alien_cache **alien;	/* on other nodes */
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
	unsigned long next_reap;	/* updated without locking */
	int free_touched;		/* updated without locking */
#endif

#ifdef CONFIG_SLUB
	unsigned long nr_partial;
	struct list_head partial;
#ifdef CONFIG_SLUB_DEBUG
	atomic_long_t nr_slabs;
	atomic_long_t total_objects;
	struct list_head full;
#endif
#endif

};
728

729 730 731 732 733 734 735 736 737 738
static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
	return s->node[node];
}

/*
 * Iterator over all nodes. The body will be executed for each node that has
 * a kmem_cache_node structure allocated (which is true for all online nodes)
 */
#define for_each_kmem_cache_node(__s, __node, __n) \
739 740
	for (__node = 0; __node < nr_node_ids; __node++) \
		 if ((__n = get_node(__s, __node)))
741 742 743

#endif

744
void *slab_start(struct seq_file *m, loff_t *pos);
745 746
void *slab_next(struct seq_file *m, void *p, loff_t *pos);
void slab_stop(struct seq_file *m, void *p);
747
int memcg_slab_show(struct seq_file *m, void *p);
748

749 750 751 752 753 754 755 756
#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
void dump_unreclaimable_slab(void);
#else
static inline void dump_unreclaimable_slab(void)
{
}
#endif

757 758
void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);

759 760 761 762 763 764 765 766 767 768 769 770 771
#ifdef CONFIG_SLAB_FREELIST_RANDOM
int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
			gfp_t gfp);
void cache_random_seq_destroy(struct kmem_cache *cachep);
#else
static inline int cache_random_seq_create(struct kmem_cache *cachep,
					unsigned int count, gfp_t gfp)
{
	return 0;
}
static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
#endif /* CONFIG_SLAB_FREELIST_RANDOM */

772 773
static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
{
774 775
	if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
				&init_on_alloc)) {
776 777 778 779 780 781 782 783 784 785 786
		if (c->ctor)
			return false;
		if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
			return flags & __GFP_ZERO;
		return true;
	}
	return flags & __GFP_ZERO;
}

static inline bool slab_want_init_on_free(struct kmem_cache *c)
{
787 788
	if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
				&init_on_free))
789 790 791 792 793
		return !(c->ctor ||
			 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
	return false;
}

794 795 796 797 798 799
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
void debugfs_slab_release(struct kmem_cache *);
#else
static inline void debugfs_slab_release(struct kmem_cache *s) { }
#endif

800
#ifdef CONFIG_PRINTK
801 802 803
#define KS_ADDRS_COUNT 16
struct kmem_obj_info {
	void *kp_ptr;
804
	struct slab *kp_slab;
805 806 807 808 809
	void *kp_objp;
	unsigned long kp_data_offset;
	struct kmem_cache *kp_slab_cache;
	void *kp_ret;
	void *kp_stack[KS_ADDRS_COUNT];
810
	void *kp_free_stack[KS_ADDRS_COUNT];
811
};
812
void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
813
#endif
814

815 816 817 818 819 820 821 822 823 824 825
#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR
void __check_heap_object(const void *ptr, unsigned long n,
			 const struct slab *slab, bool to_user);
#else
static inline
void __check_heap_object(const void *ptr, unsigned long n,
			 const struct slab *slab, bool to_user)
{
}
#endif

826
#endif /* MM_SLAB_H */
新手
引导
客服 返回
顶部