88 int add_to_swap_cache(struct folio *folio, swp_entry_t entry,  in add_to_swap_cache()  argument
93 	XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio));  in add_to_swap_cache()
94 	unsigned long i, nr = folio_nr_pages(folio);  in add_to_swap_cache()
97 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in add_to_swap_cache()
98 	VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);  in add_to_swap_cache()
99 	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);  in add_to_swap_cache()
101 	folio_ref_add(folio, nr);  in add_to_swap_cache()
102 	folio_set_swapcache(folio);  in add_to_swap_cache()
110 			VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio);  in add_to_swap_cache()
116 			set_page_private(folio_page(folio, i), entry.val + i);  in add_to_swap_cache()
117 			xas_store(&xas, folio);  in add_to_swap_cache()
121 		__node_stat_mod_folio(folio, NR_FILE_PAGES, nr);  in add_to_swap_cache()
122 		__lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr);  in add_to_swap_cache()
130 	folio_clear_swapcache(folio);  in add_to_swap_cache()
131 	folio_ref_sub(folio, nr);  in add_to_swap_cache()
139 void __delete_from_swap_cache(struct folio *folio,  in __delete_from_swap_cache()  argument
144 	long nr = folio_nr_pages(folio);  in __delete_from_swap_cache()
148 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in __delete_from_swap_cache()
149 	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);  in __delete_from_swap_cache()
150 	VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);  in __delete_from_swap_cache()
154 		VM_BUG_ON_PAGE(entry != folio, entry);  in __delete_from_swap_cache()
155 		set_page_private(folio_page(folio, i), 0);  in __delete_from_swap_cache()
158 	folio_clear_swapcache(folio);  in __delete_from_swap_cache()
160 	__node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);  in __delete_from_swap_cache()
161 	__lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr);  in __delete_from_swap_cache()
174 bool add_to_swap(struct folio *folio)  in add_to_swap()  argument
179 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);  in add_to_swap()
180 	VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio);  in add_to_swap()
182 	entry = folio_alloc_swap(folio);  in add_to_swap()
197 	err = add_to_swap_cache(folio, entry,  in add_to_swap()
216 	folio_mark_dirty(folio);  in add_to_swap()
221 	put_swap_folio(folio, entry);  in add_to_swap()
231 void delete_from_swap_cache(struct folio *folio)  in delete_from_swap_cache()  argument
233 	swp_entry_t entry = folio_swap_entry(folio);  in delete_from_swap_cache()
237 	__delete_from_swap_cache(folio, entry, NULL);  in delete_from_swap_cache()
240 	put_swap_folio(folio, entry);  in delete_from_swap_cache()
241 	folio_ref_sub(folio, folio_nr_pages(folio));  in delete_from_swap_cache()
282 	struct folio *folio = page_folio(page);  in free_swap_cache()  local
284 	if (folio_test_swapcache(folio) && !folio_mapped(folio) &&  in free_swap_cache()
285 	    folio_trylock(folio)) {  in free_swap_cache()
286 		folio_free_swap(folio);  in free_swap_cache()
287 		folio_unlock(folio);  in free_swap_cache()
328 struct folio *swap_cache_get_folio(swp_entry_t entry,  in swap_cache_get_folio()
331 	struct folio *folio;  in swap_cache_get_folio()  local
337 	folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry));  in swap_cache_get_folio()
340 	if (folio) {  in swap_cache_get_folio()
348 		if (unlikely(folio_test_large(folio)))  in swap_cache_get_folio()
349 			return folio;  in swap_cache_get_folio()
351 		readahead = folio_test_clear_readahead(folio);  in swap_cache_get_folio()
372 	return folio;  in swap_cache_get_folio()
417 	struct folio *folio;  in __read_swap_cache_async()  local
432 		folio = filemap_get_folio(swap_address_space(entry),  in __read_swap_cache_async()
435 		if (folio)  in __read_swap_cache_async()
436 			return folio_file_page(folio, swp_offset(entry));  in __read_swap_cache_async()
454 		folio = vma_alloc_folio(gfp_mask, 0, vma, addr, false);  in __read_swap_cache_async()
455 		if (!folio)  in __read_swap_cache_async()
465 		folio_put(folio);  in __read_swap_cache_async()
483 	__folio_set_locked(folio);  in __read_swap_cache_async()
484 	__folio_set_swapbacked(folio);  in __read_swap_cache_async()
486 	if (mem_cgroup_swapin_charge_folio(folio, NULL, gfp_mask, entry))  in __read_swap_cache_async()
490 	if (add_to_swap_cache(folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))  in __read_swap_cache_async()
496 		workingset_refault(folio, shadow);  in __read_swap_cache_async()
499 	folio_add_lru(folio);  in __read_swap_cache_async()
501 	return &folio->page;  in __read_swap_cache_async()
504 	put_swap_folio(folio, entry);  in __read_swap_cache_async()
505 	folio_unlock(folio);  in __read_swap_cache_async()
506 	folio_put(folio);  in __read_swap_cache_async()