44 int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,  in sbitmap_init_node()  argument
58 		if (depth >= 4) {  in sbitmap_init_node()
59 			while ((4U << shift) > depth)  in sbitmap_init_node()
68 	sb->depth = depth;  in sbitmap_init_node()
69 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_init_node()
71 	if (depth == 0) {  in sbitmap_init_node()
81 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_init_node()
82 		depth -= sb->map[i].depth;  in sbitmap_init_node()
89 void sbitmap_resize(struct sbitmap *sb, unsigned int depth)  in sbitmap_resize()  argument
97 	sb->depth = depth;  in sbitmap_resize()
98 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_resize()
101 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_resize()
102 		depth -= sb->map[i].depth;  in sbitmap_resize()
107 static int __sbitmap_get_word(unsigned long *word, unsigned long depth,  in __sbitmap_get_word()  argument
114 		nr = find_next_zero_bit(word, depth, hint);  in __sbitmap_get_word()
115 		if (unlikely(nr >= depth)) {  in __sbitmap_get_word()
132 		if (hint >= depth - 1)  in __sbitmap_get_word()
146 					sb->map[index].depth, alloc_hint,  in sbitmap_find_bit_in_index()
203 					min(sb->map[index].depth, shallow_depth),  in sbitmap_get_shallow()
248 		ret = find_first_zero_bit(&mask, word->depth);  in sbitmap_any_bit_clear()
249 		if (ret < word->depth)  in sbitmap_any_bit_clear()
264 			weight += bitmap_weight(&word->word, word->depth);  in __sbitmap_weight()
266 			weight += bitmap_weight(&word->cleared, word->depth);  in __sbitmap_weight()
283 	seq_printf(m, "depth=%u\n", sb->depth);  in sbitmap_show()
312 		unsigned int word_bits = READ_ONCE(sb->map[i].depth);  in sbitmap_bitmap_show()
339 					unsigned int depth)  in sbq_calc_wake_batch()  argument
361 	depth = ((depth >> sbq->sb.shift) * shallow_depth +  in sbq_calc_wake_batch()
362 		 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));  in sbq_calc_wake_batch()
363 	wake_batch = clamp_t(unsigned int, depth / SBQ_WAIT_QUEUES, 1,  in sbq_calc_wake_batch()
369 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,  in sbitmap_queue_init_node()  argument
375 	ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node);  in sbitmap_queue_init_node()
385 	if (depth && !round_robin) {  in sbitmap_queue_init_node()
387 			*per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth;  in sbitmap_queue_init_node()
391 	sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);  in sbitmap_queue_init_node()
413 					    unsigned int depth)  in sbitmap_queue_update_wake_batch()  argument
415 	unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth);  in sbitmap_queue_update_wake_batch()
431 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)  in sbitmap_queue_resize()  argument
433 	sbitmap_queue_update_wake_batch(sbq, depth);  in sbitmap_queue_resize()
434 	sbitmap_resize(&sbq->sb, depth);  in sbitmap_queue_resize()
440 	unsigned int hint, depth;  in __sbitmap_queue_get()  local
444 	depth = READ_ONCE(sbq->sb.depth);  in __sbitmap_queue_get()
445 	if (unlikely(hint >= depth)) {  in __sbitmap_queue_get()
446 		hint = depth ? prandom_u32() % depth : 0;  in __sbitmap_queue_get()
457 		if (hint >= depth - 1)  in __sbitmap_queue_get()
469 	unsigned int hint, depth;  in __sbitmap_queue_get_shallow()  local
475 	depth = READ_ONCE(sbq->sb.depth);  in __sbitmap_queue_get_shallow()
476 	if (unlikely(hint >= depth)) {  in __sbitmap_queue_get_shallow()
477 		hint = depth ? prandom_u32() % depth : 0;  in __sbitmap_queue_get_shallow()
488 		if (hint >= depth - 1)  in __sbitmap_queue_get_shallow()
501 	sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);  in sbitmap_queue_min_shallow_depth()
601 	if (likely(!sbq->round_robin && nr < sbq->sb.depth))  in sbitmap_queue_clear()