14 	unsigned depth = sb->depth;  in init_alloc_hint()  local
20 	if (depth && !sb->round_robin) {  in init_alloc_hint()
24 			*per_cpu_ptr(sb->alloc_hint, i) = prandom_u32() % depth;  in init_alloc_hint()
30 						    unsigned int depth)  in update_alloc_hint_before_get()  argument
35 	if (unlikely(hint >= depth)) {  in update_alloc_hint_before_get()
36 		hint = depth ? prandom_u32() % depth : 0;  in update_alloc_hint_before_get()
44 					       unsigned int depth,  in update_alloc_hint_after_get()  argument
54 		if (hint >= depth - 1)  in update_alloc_hint_after_get()
83 int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,  in sbitmap_init_node()  argument
91 		shift = sbitmap_calculate_shift(depth);  in sbitmap_init_node()
98 	sb->depth = depth;  in sbitmap_init_node()
99 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_init_node()
102 	if (depth == 0) {  in sbitmap_init_node()
121 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_init_node()
122 		depth -= sb->map[i].depth;  in sbitmap_init_node()
128 void sbitmap_resize(struct sbitmap *sb, unsigned int depth)  in sbitmap_resize()  argument
136 	sb->depth = depth;  in sbitmap_resize()
137 	sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);  in sbitmap_resize()
140 		sb->map[i].depth = min(depth, bits_per_word);  in sbitmap_resize()
141 		depth -= sb->map[i].depth;  in sbitmap_resize()
146 static int __sbitmap_get_word(unsigned long *word, unsigned long depth,  in __sbitmap_get_word()  argument
155 		nr = find_next_zero_bit(word, depth, hint);  in __sbitmap_get_word()
156 		if (unlikely(nr >= depth)) {  in __sbitmap_get_word()
173 		if (hint >= depth - 1)  in __sbitmap_get_word()
187 		nr = __sbitmap_get_word(&map->word, map->depth, alloc_hint,  in sbitmap_find_bit_in_index()
234 	unsigned int hint, depth;  in sbitmap_get()  local
239 	depth = READ_ONCE(sb->depth);  in sbitmap_get()
240 	hint = update_alloc_hint_before_get(sb, depth);  in sbitmap_get()
242 	update_alloc_hint_after_get(sb, depth, hint, nr);  in sbitmap_get()
260 					min(sb->map[index].depth, shallow_depth),  in __sbitmap_get_shallow()
286 	unsigned int hint, depth;  in sbitmap_get_shallow()  local
291 	depth = READ_ONCE(sb->depth);  in sbitmap_get_shallow()
292 	hint = update_alloc_hint_before_get(sb, depth);  in sbitmap_get_shallow()
294 	update_alloc_hint_after_get(sb, depth, hint, nr);  in sbitmap_get_shallow()
320 			weight += bitmap_weight(&word->word, word->depth);  in __sbitmap_weight()
322 			weight += bitmap_weight(&word->cleared, word->depth);  in __sbitmap_weight()
340 	seq_printf(m, "depth=%u\n", sb->depth);  in sbitmap_show()
370 		unsigned int word_bits = READ_ONCE(sb->map[i].depth);  in sbitmap_bitmap_show()
399 					unsigned int depth)  in sbq_calc_wake_batch()  argument
406 	 * batch size is small enough that the full depth of the bitmap,  in sbq_calc_wake_batch()
407 	 * potentially limited by a shallow depth, is enough to wake up all of  in sbq_calc_wake_batch()
411 	 * be a partial word. There are depth / bits_per_word full words and  in sbq_calc_wake_batch()
412 	 * depth % bits_per_word bits left over. In bitwise arithmetic:  in sbq_calc_wake_batch()
415 	 * depth / bits_per_word = depth >> shift  in sbq_calc_wake_batch()
416 	 * depth % bits_per_word = depth & ((1 << shift) - 1)  in sbq_calc_wake_batch()
421 	depth = ((depth >> sbq->sb.shift) * shallow_depth +  in sbq_calc_wake_batch()
422 		 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));  in sbq_calc_wake_batch()
423 	wake_batch = clamp_t(unsigned int, depth / SBQ_WAIT_QUEUES, 1,  in sbq_calc_wake_batch()
429 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,  in sbitmap_queue_init_node()  argument
435 	ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node,  in sbitmap_queue_init_node()
441 	sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);  in sbitmap_queue_init_node()
461 					    unsigned int depth)  in sbitmap_queue_update_wake_batch()  argument
463 	unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth);  in sbitmap_queue_update_wake_batch()
479 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)  in sbitmap_queue_resize()  argument
481 	sbitmap_queue_update_wake_batch(sbq, depth);  in sbitmap_queue_resize()
482 	sbitmap_resize(&sbq->sb, depth);  in sbitmap_queue_resize()
505 	sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);  in sbitmap_queue_min_shallow_depth()
605 	if (likely(!sbq->sb.round_robin && nr < sbq->sb.depth))  in sbitmap_queue_clear()