221 				struct kmem_cache_node *n, struct slab *slab,
241 #define MAKE_LIST(cachep, listp, slab, nodeid)				\  argument
244 		list_splice(&get_node(cachep, nodeid)->slab, listp);	\
375 				 const struct slab *slab, unsigned int idx)  in index_to_obj()  argument
377 	return slab->s_mem + cache->size * idx;  in index_to_obj()
552 					struct slab *slab, void *objp)  in cache_free_pfmemalloc()  argument
558 	slab_node = slab_nid(slab);  in cache_free_pfmemalloc()
1348 static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,  in kmem_getpages()
1352 	struct slab *slab;  in kmem_getpages()  local
1362 	slab = folio_slab(folio);  in kmem_getpages()
1364 	account_slab(slab, cachep->gfporder, cachep, flags);  in kmem_getpages()
1370 		slab_set_pfmemalloc(slab);  in kmem_getpages()
1372 	return slab;  in kmem_getpages()
1378 static void kmem_freepages(struct kmem_cache *cachep, struct slab *slab)  in kmem_freepages()  argument
1381 	struct folio *folio = slab_folio(slab);  in kmem_freepages()
1384 	__slab_clear_pfmemalloc(slab);  in kmem_freepages()
1392 	unaccount_slab(slab, order, cachep);  in kmem_freepages()
1399 	struct slab *slab;  in kmem_rcu_free()  local
1401 	slab = container_of(head, struct slab, rcu_head);  in kmem_rcu_free()
1402 	cachep = slab->slab_cache;  in kmem_rcu_free()
1404 	kmem_freepages(cachep, slab);  in kmem_rcu_free()
1537 		struct slab *slab = virt_to_slab(objp);  in check_poison_obj()  local
1540 		objnr = obj_to_index(cachep, slab, objp);  in check_poison_obj()
1542 			objp = index_to_obj(cachep, slab, objnr - 1);  in check_poison_obj()
1548 			objp = index_to_obj(cachep, slab, objnr + 1);  in check_poison_obj()
1559 						struct slab *slab)  in slab_destroy_debugcheck()  argument
1564 		poison_obj(cachep, slab->freelist - obj_offset(cachep),  in slab_destroy_debugcheck()
1569 		void *objp = index_to_obj(cachep, slab, i);  in slab_destroy_debugcheck()
1585 						struct slab *slab)  in slab_destroy_debugcheck()  argument
1599 static void slab_destroy(struct kmem_cache *cachep, struct slab *slab)  in slab_destroy()  argument
1603 	freelist = slab->freelist;  in slab_destroy()
1604 	slab_destroy_debugcheck(cachep, slab);  in slab_destroy()
1606 		call_rcu(&slab->rcu_head, kmem_rcu_free);  in slab_destroy()
1608 		kmem_freepages(cachep, slab);  in slab_destroy()
1624 	struct slab *slab, *n;  in slabs_destroy()  local
1626 	list_for_each_entry_safe(slab, n, list, slab_list) {  in slabs_destroy()
1627 		list_del(&slab->slab_list);  in slabs_destroy()
1628 		slab_destroy(cachep, slab);  in slabs_destroy()
2175 	struct slab *slab;  in drain_freelist()  local
2187 		slab = list_entry(p, struct slab, slab_list);  in drain_freelist()
2188 		list_del(&slab->slab_list);  in drain_freelist()
2197 		slab_destroy(cache, slab);  in drain_freelist()
2274 				   struct slab *slab, int colour_off,  in alloc_slabmgmt()  argument
2278 	void *addr = slab_address(slab);  in alloc_slabmgmt()
2280 	slab->s_mem = addr + colour_off;  in alloc_slabmgmt()
2281 	slab->active = 0;  in alloc_slabmgmt()
2298 static inline freelist_idx_t get_free_obj(struct slab *slab, unsigned int idx)  in get_free_obj()  argument
2300 	return ((freelist_idx_t *) slab->freelist)[idx];  in get_free_obj()
2303 static inline void set_free_obj(struct slab *slab,  in set_free_obj()  argument
2306 	((freelist_idx_t *)(slab->freelist))[idx] = val;  in set_free_obj()
2309 static void cache_init_objs_debug(struct kmem_cache *cachep, struct slab *slab)  in cache_init_objs_debug()  argument
2315 		void *objp = index_to_obj(cachep, slab, i);  in cache_init_objs_debug()
2389 static void swap_free_obj(struct slab *slab, unsigned int a, unsigned int b)  in swap_free_obj()  argument
2391 	swap(((freelist_idx_t *) slab->freelist)[a],  in swap_free_obj()
2392 		((freelist_idx_t *) slab->freelist)[b]);  in swap_free_obj()
2399 static bool shuffle_freelist(struct kmem_cache *cachep, struct slab *slab)  in shuffle_freelist()  argument
2416 		slab->freelist = index_to_obj(cachep, slab, objfreelist) +  in shuffle_freelist()
2427 			set_free_obj(slab, i, i);  in shuffle_freelist()
2432 			swap_free_obj(slab, i, rand);  in shuffle_freelist()
2436 			set_free_obj(slab, i, next_random_slot(&state));  in shuffle_freelist()
2440 		set_free_obj(slab, cachep->num - 1, objfreelist);  in shuffle_freelist()
2446 				struct slab *slab)  in shuffle_freelist()  argument
2453 			    struct slab *slab)  in cache_init_objs()  argument
2459 	cache_init_objs_debug(cachep, slab);  in cache_init_objs()
2462 	shuffled = shuffle_freelist(cachep, slab);  in cache_init_objs()
2465 		slab->freelist = index_to_obj(cachep, slab, cachep->num - 1) +  in cache_init_objs()
2470 		objp = index_to_obj(cachep, slab, i);  in cache_init_objs()
2481 			set_free_obj(slab, i, i);  in cache_init_objs()
2485 static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slab)  in slab_get_obj()  argument
2489 	objp = index_to_obj(cachep, slab, get_free_obj(slab, slab->active));  in slab_get_obj()
2490 	slab->active++;  in slab_get_obj()
2496 			struct slab *slab, void *objp)  in slab_put_obj()  argument
2498 	unsigned int objnr = obj_to_index(cachep, slab, objp);  in slab_put_obj()
2503 	for (i = slab->active; i < cachep->num; i++) {  in slab_put_obj()
2504 		if (get_free_obj(slab, i) == objnr) {  in slab_put_obj()
2511 	slab->active--;  in slab_put_obj()
2512 	if (!slab->freelist)  in slab_put_obj()
2513 		slab->freelist = objp + obj_offset(cachep);  in slab_put_obj()
2515 	set_free_obj(slab, slab->active, objnr);  in slab_put_obj()
2522 static struct slab *cache_grow_begin(struct kmem_cache *cachep,  in cache_grow_begin()
2530 	struct slab *slab;  in cache_grow_begin()  local
2550 	slab = kmem_getpages(cachep, local_flags, nodeid);  in cache_grow_begin()
2551 	if (!slab)  in cache_grow_begin()
2554 	slab_node = slab_nid(slab);  in cache_grow_begin()
2573 	kasan_poison_slab(slab);  in cache_grow_begin()
2576 	freelist = alloc_slabmgmt(cachep, slab, offset,  in cache_grow_begin()
2581 	slab->slab_cache = cachep;  in cache_grow_begin()
2582 	slab->freelist = freelist;  in cache_grow_begin()
2584 	cache_init_objs(cachep, slab);  in cache_grow_begin()
2589 	return slab;  in cache_grow_begin()
2592 	kmem_freepages(cachep, slab);  in cache_grow_begin()
2599 static void cache_grow_end(struct kmem_cache *cachep, struct slab *slab)  in cache_grow_end()  argument
2606 	if (!slab)  in cache_grow_end()
2609 	INIT_LIST_HEAD(&slab->slab_list);  in cache_grow_end()
2610 	n = get_node(cachep, slab_nid(slab));  in cache_grow_end()
2614 	if (!slab->active) {  in cache_grow_end()
2615 		list_add_tail(&slab->slab_list, &n->slabs_free);  in cache_grow_end()
2618 		fixup_slab_list(cachep, n, slab, &list);  in cache_grow_end()
2621 	n->free_objects += cachep->num - slab->active;  in cache_grow_end()
2669 	struct slab *slab;  in cache_free_debugcheck()  local
2675 	slab = virt_to_slab(objp);  in cache_free_debugcheck()
2685 	objnr = obj_to_index(cachep, slab, objp);  in cache_free_debugcheck()
2688 	BUG_ON(objp != index_to_obj(cachep, slab, objnr));  in cache_free_debugcheck()
2718 				struct kmem_cache_node *n, struct slab *slab,  in fixup_slab_list()  argument
2722 	list_del(&slab->slab_list);  in fixup_slab_list()
2723 	if (slab->active == cachep->num) {  in fixup_slab_list()
2724 		list_add(&slab->slab_list, &n->slabs_full);  in fixup_slab_list()
2729 				void **objp = slab->freelist;  in fixup_slab_list()
2735 			slab->freelist = NULL;  in fixup_slab_list()
2738 		list_add(&slab->slab_list, &n->slabs_partial);  in fixup_slab_list()
2742 static noinline struct slab *get_valid_first_slab(struct kmem_cache_node *n,  in get_valid_first_slab()
2743 					struct slab *slab, bool pfmemalloc)  in get_valid_first_slab()  argument
2745 	if (!slab)  in get_valid_first_slab()
2749 		return slab;  in get_valid_first_slab()
2751 	if (!slab_test_pfmemalloc(slab))  in get_valid_first_slab()
2752 		return slab;  in get_valid_first_slab()
2756 		slab_clear_pfmemalloc(slab);  in get_valid_first_slab()
2757 		return slab;  in get_valid_first_slab()
2761 	list_del(&slab->slab_list);  in get_valid_first_slab()
2762 	if (!slab->active) {  in get_valid_first_slab()
2763 		list_add_tail(&slab->slab_list, &n->slabs_free);  in get_valid_first_slab()
2766 		list_add_tail(&slab->slab_list, &n->slabs_partial);  in get_valid_first_slab()
2768 	list_for_each_entry(slab, &n->slabs_partial, slab_list) {  in get_valid_first_slab()
2769 		if (!slab_test_pfmemalloc(slab))  in get_valid_first_slab()
2770 			return slab;  in get_valid_first_slab()
2774 	list_for_each_entry(slab, &n->slabs_free, slab_list) {  in get_valid_first_slab()
2775 		if (!slab_test_pfmemalloc(slab)) {  in get_valid_first_slab()
2777 			return slab;  in get_valid_first_slab()
2784 static struct slab *get_first_slab(struct kmem_cache_node *n, bool pfmemalloc)  in get_first_slab()
2786 	struct slab *slab;  in get_first_slab()  local
2789 	slab = list_first_entry_or_null(&n->slabs_partial, struct slab,  in get_first_slab()
2791 	if (!slab) {  in get_first_slab()
2793 		slab = list_first_entry_or_null(&n->slabs_free, struct slab,  in get_first_slab()
2795 		if (slab)  in get_first_slab()
2800 		slab = get_valid_first_slab(n, slab, pfmemalloc);  in get_first_slab()
2802 	return slab;  in get_first_slab()
2808 	struct slab *slab;  in cache_alloc_pfmemalloc()  local
2816 	slab = get_first_slab(n, true);  in cache_alloc_pfmemalloc()
2817 	if (!slab) {  in cache_alloc_pfmemalloc()
2822 	obj = slab_get_obj(cachep, slab);  in cache_alloc_pfmemalloc()
2825 	fixup_slab_list(cachep, n, slab, &list);  in cache_alloc_pfmemalloc()
2838 		struct array_cache *ac, struct slab *slab, int batchcount)  in alloc_block()  argument
2844 	BUG_ON(slab->active >= cachep->num);  in alloc_block()
2846 	while (slab->active < cachep->num && batchcount--) {  in alloc_block()
2851 		ac->entry[ac->avail++] = slab_get_obj(cachep, slab);  in alloc_block()
2864 	struct slab *slab;  in cache_alloc_refill()  local
2897 		slab = get_first_slab(n, false);  in cache_alloc_refill()
2898 		if (!slab)  in cache_alloc_refill()
2903 		batchcount = alloc_block(cachep, ac, slab, batchcount);  in cache_alloc_refill()
2904 		fixup_slab_list(cachep, n, slab, &list);  in cache_alloc_refill()
2923 		slab = cache_grow_begin(cachep, gfp_exact_node(flags), node);  in cache_alloc_refill()
2930 		if (!ac->avail && slab)  in cache_alloc_refill()
2931 			alloc_block(cachep, ac, slab, batchcount);  in cache_alloc_refill()
2932 		cache_grow_end(cachep, slab);  in cache_alloc_refill()
3057 	struct slab *slab;  in fallback_alloc()  local
3093 		slab = cache_grow_begin(cache, flags, numa_mem_id());  in fallback_alloc()
3094 		cache_grow_end(cache, slab);  in fallback_alloc()
3095 		if (slab) {  in fallback_alloc()
3096 			nid = slab_nid(slab);  in fallback_alloc()
3120 	struct slab *slab;  in ____cache_alloc_node()  local
3131 	slab = get_first_slab(n, false);  in ____cache_alloc_node()
3132 	if (!slab)  in ____cache_alloc_node()
3141 	BUG_ON(slab->active == cachep->num);  in ____cache_alloc_node()
3143 	obj = slab_get_obj(cachep, slab);  in ____cache_alloc_node()
3146 	fixup_slab_list(cachep, n, slab, &list);  in ____cache_alloc_node()
3154 	slab = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid);  in ____cache_alloc_node()
3155 	if (slab) {  in ____cache_alloc_node()
3157 		obj = slab_get_obj(cachep, slab);  in ____cache_alloc_node()
3159 	cache_grow_end(cachep, slab);  in ____cache_alloc_node()
3259 	struct slab *slab;  in free_block()  local
3265 		struct slab *slab;  in free_block()  local
3269 		slab = virt_to_slab(objp);  in free_block()
3270 		list_del(&slab->slab_list);  in free_block()
3272 		slab_put_obj(cachep, slab, objp);  in free_block()
3276 		if (slab->active == 0) {  in free_block()
3277 			list_add(&slab->slab_list, &n->slabs_free);  in free_block()
3284 			list_add_tail(&slab->slab_list, &n->slabs_partial);  in free_block()
3291 		slab = list_last_entry(&n->slabs_free, struct slab, slab_list);  in free_block()
3292 		list_move(&slab->slab_list, list);  in free_block()
3328 		struct slab *slab;  in cache_flusharray()  local
3330 		list_for_each_entry(slab, &n->slabs_free, slab_list) {  in cache_flusharray()
3331 			BUG_ON(slab->active);  in cache_flusharray()
3408 		struct slab *slab = virt_to_slab(objp);  in ___cache_free()  local
3410 		if (unlikely(slab_test_pfmemalloc(slab))) {  in ___cache_free()
3411 			cache_free_pfmemalloc(cachep, slab, objp);  in ___cache_free()
3526 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)  in __kmem_obj_info()  argument
3533 	kpp->kp_slab = slab;  in __kmem_obj_info()
3534 	cachep = slab->slab_cache;  in __kmem_obj_info()
3538 	slab = virt_to_slab(objp);  in __kmem_obj_info()
3539 	objnr = obj_to_index(cachep, slab, objp);  in __kmem_obj_info()
3540 	objp = index_to_obj(cachep, slab, objnr);  in __kmem_obj_info()
3999 			 const struct slab *slab, bool to_user)  in __check_heap_object()  argument
4008 	cachep = slab->slab_cache;  in __check_heap_object()
4009 	objnr = obj_to_index(cachep, slab, (void *)ptr);  in __check_heap_object()
4016 		offset = ptr - index_to_obj(cachep, slab, objnr) - obj_offset(cachep);  in __check_heap_object()