23 int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,  in sbitmap_init_node()  argument
46 	sb->shift = shift;  in sbitmap_init_node()
47 	sb->depth = depth;  in sbitmap_init_node()
48 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_init_node()
51 		sb->map = NULL;  in sbitmap_init_node()
55 	sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node);  in sbitmap_init_node()
56 	if (!sb->map)  in sbitmap_init_node()
59 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_init_node()
60 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_init_node()
61 		depth -= sb->map[i].depth;  in sbitmap_init_node()
67 void sbitmap_resize(struct sbitmap *sb, unsigned int depth)  in sbitmap_resize()  argument
69 	unsigned int bits_per_word = 1U << sb->shift;  in sbitmap_resize()
72 	sb->depth = depth;  in sbitmap_resize()
73 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_resize()
75 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_resize()
76 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_resize()
77 		depth -= sb->map[i].depth;  in sbitmap_resize()
114 int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin)  in sbitmap_get()  argument
119 	index = SB_NR_TO_INDEX(sb, alloc_hint);  in sbitmap_get()
121 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_get()
122 		nr = __sbitmap_get_word(&sb->map[index].word,  in sbitmap_get()
123 					sb->map[index].depth,  in sbitmap_get()
124 					SB_NR_TO_BIT(sb, alloc_hint),  in sbitmap_get()
127 			nr += index << sb->shift;  in sbitmap_get()
133 		alloc_hint = index << sb->shift;  in sbitmap_get()
135 		if (index >= sb->map_nr) {  in sbitmap_get()
145 int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,  in sbitmap_get_shallow()  argument
151 	index = SB_NR_TO_INDEX(sb, alloc_hint);  in sbitmap_get_shallow()
153 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_get_shallow()
154 		nr = __sbitmap_get_word(&sb->map[index].word,  in sbitmap_get_shallow()
155 					min(sb->map[index].depth, shallow_depth),  in sbitmap_get_shallow()
156 					SB_NR_TO_BIT(sb, alloc_hint), true);  in sbitmap_get_shallow()
158 			nr += index << sb->shift;  in sbitmap_get_shallow()
164 		alloc_hint = index << sb->shift;  in sbitmap_get_shallow()
166 		if (index >= sb->map_nr) {  in sbitmap_get_shallow()
176 bool sbitmap_any_bit_set(const struct sbitmap *sb)  in sbitmap_any_bit_set()  argument
180 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_any_bit_set()
181 		if (sb->map[i].word)  in sbitmap_any_bit_set()
188 bool sbitmap_any_bit_clear(const struct sbitmap *sb)  in sbitmap_any_bit_clear()  argument
192 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_any_bit_clear()
193 		const struct sbitmap_word *word = &sb->map[i];  in sbitmap_any_bit_clear()
204 unsigned int sbitmap_weight(const struct sbitmap *sb)  in sbitmap_weight()  argument
208 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_weight()
209 		const struct sbitmap_word *word = &sb->map[i];  in sbitmap_weight()
217 void sbitmap_show(struct sbitmap *sb, struct seq_file *m)  in sbitmap_show()  argument
219 	seq_printf(m, "depth=%u\n", sb->depth);  in sbitmap_show()
220 	seq_printf(m, "busy=%u\n", sbitmap_weight(sb));  in sbitmap_show()
221 	seq_printf(m, "bits_per_word=%u\n", 1U << sb->shift);  in sbitmap_show()
222 	seq_printf(m, "map_nr=%u\n", sb->map_nr);  in sbitmap_show()
238 void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m)  in sbitmap_bitmap_show()  argument
245 	for (i = 0; i < sb->map_nr; i++) {  in sbitmap_bitmap_show()
246 		unsigned long word = READ_ONCE(sb->map[i].word);  in sbitmap_bitmap_show()
247 		unsigned int word_bits = READ_ONCE(sb->map[i].depth);  in sbitmap_bitmap_show()
295 	shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth);  in sbq_calc_wake_batch()
296 	depth = ((depth >> sbq->sb.shift) * shallow_depth +  in sbq_calc_wake_batch()
297 		 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));  in sbq_calc_wake_batch()
310 	ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node);  in sbitmap_queue_init_node()
316 		sbitmap_free(&sbq->sb);  in sbitmap_queue_init_node()
332 		sbitmap_free(&sbq->sb);  in sbitmap_queue_init_node()
368 	sbitmap_resize(&sbq->sb, depth);  in sbitmap_queue_resize()
378 	depth = READ_ONCE(sbq->sb.depth);  in __sbitmap_queue_get()
383 	nr = sbitmap_get(&sbq->sb, hint, sbq->round_robin);  in __sbitmap_queue_get()
409 	depth = READ_ONCE(sbq->sb.depth);  in __sbitmap_queue_get_shallow()
414 	nr = sbitmap_get_shallow(&sbq->sb, hint, shallow_depth);  in __sbitmap_queue_get_shallow()
435 	sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);  in sbitmap_queue_min_shallow_depth()
512 	sbitmap_clear_bit_unlock(&sbq->sb, nr);  in sbitmap_queue_clear()
522 	if (likely(!sbq->round_robin && nr < sbq->sb.depth))  in sbitmap_queue_clear()
553 	sbitmap_show(&sbq->sb, m);  in sbitmap_queue_show()