108 	struct sbitmap sb;  member
165 int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
169 static inline unsigned int __map_depth(const struct sbitmap *sb, int index)  in __map_depth()  argument
171 	if (index == sb->map_nr - 1)  in __map_depth()
172 		return sb->depth - (index << sb->shift);  in __map_depth()
173 	return 1U << sb->shift;  in __map_depth()
180 static inline void sbitmap_free(struct sbitmap *sb)  in sbitmap_free()  argument
182 	free_percpu(sb->alloc_hint);  in sbitmap_free()
183 	kvfree(sb->map);  in sbitmap_free()
184 	sb->map = NULL;  in sbitmap_free()
195 void sbitmap_resize(struct sbitmap *sb, unsigned int depth);
205 int sbitmap_get(struct sbitmap *sb);
222 int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth);
230 bool sbitmap_any_bit_set(const struct sbitmap *sb);
232 #define SB_NR_TO_INDEX(sb, bitnr) ((bitnr) >> (sb)->shift)  argument
233 #define SB_NR_TO_BIT(sb, bitnr) ((bitnr) & ((1U << (sb)->shift) - 1U))  argument
247 static inline void __sbitmap_for_each_set(struct sbitmap *sb,  in __sbitmap_for_each_set()  argument
255 	if (start >= sb->depth)  in __sbitmap_for_each_set()
257 	index = SB_NR_TO_INDEX(sb, start);  in __sbitmap_for_each_set()
258 	nr = SB_NR_TO_BIT(sb, start);  in __sbitmap_for_each_set()
260 	while (scanned < sb->depth) {  in __sbitmap_for_each_set()
263 					   __map_depth(sb, index) - nr,  in __sbitmap_for_each_set()
264 					   sb->depth - scanned);  in __sbitmap_for_each_set()
267 		word = sb->map[index].word & ~sb->map[index].cleared;  in __sbitmap_for_each_set()
281 			if (!fn(sb, (index << sb->shift) + nr, data))  in __sbitmap_for_each_set()
288 		if (++index >= sb->map_nr)  in __sbitmap_for_each_set()
299 static inline void sbitmap_for_each_set(struct sbitmap *sb, sb_for_each_fn fn,  in sbitmap_for_each_set()  argument
302 	__sbitmap_for_each_set(sb, 0, fn, data);  in sbitmap_for_each_set()
305 static inline unsigned long *__sbitmap_word(struct sbitmap *sb,  in __sbitmap_word()  argument
308 	return &sb->map[SB_NR_TO_INDEX(sb, bitnr)].word;  in __sbitmap_word()
313 static inline void sbitmap_set_bit(struct sbitmap *sb, unsigned int bitnr)  in sbitmap_set_bit()  argument
315 	set_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));  in sbitmap_set_bit()
318 static inline void sbitmap_clear_bit(struct sbitmap *sb, unsigned int bitnr)  in sbitmap_clear_bit()  argument
320 	clear_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));  in sbitmap_clear_bit()
329 static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int bitnr)  in sbitmap_deferred_clear_bit()  argument
331 	unsigned long *addr = &sb->map[SB_NR_TO_INDEX(sb, bitnr)].cleared;  in sbitmap_deferred_clear_bit()
333 	set_bit(SB_NR_TO_BIT(sb, bitnr), addr);  in sbitmap_deferred_clear_bit()
340 static inline void sbitmap_put(struct sbitmap *sb, unsigned int bitnr)  in sbitmap_put()  argument
342 	sbitmap_deferred_clear_bit(sb, bitnr);  in sbitmap_put()
344 	if (likely(sb->alloc_hint && !sb->round_robin && bitnr < sb->depth))  in sbitmap_put()
345 		*raw_cpu_ptr(sb->alloc_hint) = bitnr;  in sbitmap_put()
348 static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr)  in sbitmap_test_bit()  argument
350 	return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr));  in sbitmap_test_bit()
378 void sbitmap_show(struct sbitmap *sb, struct seq_file *m);
388 unsigned int sbitmap_weight(const struct sbitmap *sb);
399 void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m);
424 	sbitmap_free(&sbq->sb);  in sbitmap_queue_free()