116 	struct swap_slots_cache *cache;  in alloc_swap_slot_cache()  local
137 	cache = &per_cpu(swp_slots, cpu);  in alloc_swap_slot_cache()
138 	if (cache->slots || cache->slots_ret) {  in alloc_swap_slot_cache()
148 	if (!cache->lock_initialized) {  in alloc_swap_slot_cache()
149 		mutex_init(&cache->alloc_lock);  in alloc_swap_slot_cache()
150 		spin_lock_init(&cache->free_lock);  in alloc_swap_slot_cache()
151 		cache->lock_initialized = true;  in alloc_swap_slot_cache()
153 	cache->nr = 0;  in alloc_swap_slot_cache()
154 	cache->cur = 0;  in alloc_swap_slot_cache()
155 	cache->n_ret = 0;  in alloc_swap_slot_cache()
163 	cache->slots = slots;  in alloc_swap_slot_cache()
164 	cache->slots_ret = slots_ret;  in alloc_swap_slot_cache()
172 	struct swap_slots_cache *cache;  in drain_slots_cache_cpu()  local
175 	cache = &per_cpu(swp_slots, cpu);  in drain_slots_cache_cpu()
176 	if ((type & SLOTS_CACHE) && cache->slots) {  in drain_slots_cache_cpu()
177 		mutex_lock(&cache->alloc_lock);  in drain_slots_cache_cpu()
178 		swapcache_free_entries(cache->slots + cache->cur, cache->nr);  in drain_slots_cache_cpu()
179 		cache->cur = 0;  in drain_slots_cache_cpu()
180 		cache->nr = 0;  in drain_slots_cache_cpu()
181 		if (free_slots && cache->slots) {  in drain_slots_cache_cpu()
182 			kvfree(cache->slots);  in drain_slots_cache_cpu()
183 			cache->slots = NULL;  in drain_slots_cache_cpu()
185 		mutex_unlock(&cache->alloc_lock);  in drain_slots_cache_cpu()
187 	if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {  in drain_slots_cache_cpu()
188 		spin_lock_irq(&cache->free_lock);  in drain_slots_cache_cpu()
189 		swapcache_free_entries(cache->slots_ret, cache->n_ret);  in drain_slots_cache_cpu()
190 		cache->n_ret = 0;  in drain_slots_cache_cpu()
191 		if (free_slots && cache->slots_ret) {  in drain_slots_cache_cpu()
192 			slots = cache->slots_ret;  in drain_slots_cache_cpu()
193 			cache->slots_ret = NULL;  in drain_slots_cache_cpu()
195 		spin_unlock_irq(&cache->free_lock);  in drain_slots_cache_cpu()
261 static int refill_swap_slots_cache(struct swap_slots_cache *cache)  in refill_swap_slots_cache()  argument
263 	if (!use_swap_slot_cache || cache->nr)  in refill_swap_slots_cache()
266 	cache->cur = 0;  in refill_swap_slots_cache()
268 		cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,  in refill_swap_slots_cache()
269 					   cache->slots, 1);  in refill_swap_slots_cache()
271 	return cache->nr;  in refill_swap_slots_cache()
276 	struct swap_slots_cache *cache;  in free_swap_slot()  local
278 	cache = raw_cpu_ptr(&swp_slots);  in free_swap_slot()
279 	if (likely(use_swap_slot_cache && cache->slots_ret)) {  in free_swap_slot()
280 		spin_lock_irq(&cache->free_lock);  in free_swap_slot()
282 		if (!use_swap_slot_cache || !cache->slots_ret) {  in free_swap_slot()
283 			spin_unlock_irq(&cache->free_lock);  in free_swap_slot()
286 		if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {  in free_swap_slot()
293 			swapcache_free_entries(cache->slots_ret, cache->n_ret);  in free_swap_slot()
294 			cache->n_ret = 0;  in free_swap_slot()
296 		cache->slots_ret[cache->n_ret++] = entry;  in free_swap_slot()
297 		spin_unlock_irq(&cache->free_lock);  in free_swap_slot()
309 	struct swap_slots_cache *cache;  in get_swap_page()  local
328 	cache = raw_cpu_ptr(&swp_slots);  in get_swap_page()
330 	if (likely(check_cache_active() && cache->slots)) {  in get_swap_page()
331 		mutex_lock(&cache->alloc_lock);  in get_swap_page()
332 		if (cache->slots) {  in get_swap_page()
334 			if (cache->nr) {  in get_swap_page()
335 				entry = cache->slots[cache->cur];  in get_swap_page()
336 				cache->slots[cache->cur++].val = 0;  in get_swap_page()
337 				cache->nr--;  in get_swap_page()
338 			} else if (refill_swap_slots_cache(cache)) {  in get_swap_page()
342 		mutex_unlock(&cache->alloc_lock);  in get_swap_page()