24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
60 	if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {  in check_element()
61 		__check_element(pool, element, ksize(element));  in check_element()
62 	} else if (pool->free == mempool_free_pages) {  in check_element()
64 		int order = (int)(long)pool->pool_data;  in check_element()
67 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
80 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
83 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {  in poison_element()
85 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
87 		int order = (int)(long)pool->pool_data;  in poison_element()
95 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
98 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
103 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
105 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
107 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
108 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
112 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
114 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
116 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
117 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
121 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
123 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
124 	poison_element(pool, element);  in add_element()
125 	kasan_poison_element(pool, element);  in add_element()
126 	pool->elements[pool->curr_nr++] = element;  in add_element()
129 static void *remove_element(mempool_t *pool)  in remove_element()  argument
131 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
133 	BUG_ON(pool->curr_nr < 0);  in remove_element()
134 	kasan_unpoison_element(pool, element);  in remove_element()
135 	check_element(pool, element);  in remove_element()
150 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
152 	while (pool->curr_nr) {  in mempool_exit()
153 		void *element = remove_element(pool);  in mempool_exit()
154 		pool->free(element, pool->pool_data);  in mempool_exit()
156 	kfree(pool->elements);  in mempool_exit()
157 	pool->elements = NULL;  in mempool_exit()
169 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
171 	if (unlikely(!pool))  in mempool_destroy()
174 	mempool_exit(pool);  in mempool_destroy()
175 	kfree(pool);  in mempool_destroy()
179 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
183 	spin_lock_init(&pool->lock);  in mempool_init_node()
184 	pool->min_nr	= min_nr;  in mempool_init_node()
185 	pool->pool_data = pool_data;  in mempool_init_node()
186 	pool->alloc	= alloc_fn;  in mempool_init_node()
187 	pool->free	= free_fn;  in mempool_init_node()
188 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
190 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
192 	if (!pool->elements)  in mempool_init_node()
198 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
201 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
203 			mempool_exit(pool);  in mempool_init_node()
206 		add_element(pool, element);  in mempool_init_node()
227 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init()  argument
230 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init()
264 	mempool_t *pool;  in mempool_create_node()  local
266 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
267 	if (!pool)  in mempool_create_node()
270 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node()
272 		kfree(pool);  in mempool_create_node()
276 	return pool;  in mempool_create_node()
298 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
307 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
308 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
309 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
310 			element = remove_element(pool);  in mempool_resize()
311 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
312 			pool->free(element, pool->pool_data);  in mempool_resize()
313 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
315 		pool->min_nr = new_min_nr;  in mempool_resize()
318 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
326 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
327 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
329 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
333 	memcpy(new_elements, pool->elements,  in mempool_resize()
334 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
335 	kfree(pool->elements);  in mempool_resize()
336 	pool->elements = new_elements;  in mempool_resize()
337 	pool->min_nr = new_min_nr;  in mempool_resize()
339 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
340 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
341 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
344 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
345 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
346 			add_element(pool, element);  in mempool_resize()
348 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
349 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
354 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
374 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
392 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
396 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
397 	if (likely(pool->curr_nr)) {  in mempool_alloc()
398 		element = remove_element(pool);  in mempool_alloc()
399 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
415 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
422 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
428 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
430 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
438 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
451 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
491 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
492 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
493 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
494 			add_element(pool, element);  in mempool_free()
495 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
496 			wake_up(&pool->wait);  in mempool_free()
499 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
501 	pool->free(element, pool->pool_data);  in mempool_free()