126 static void page_cache_delete(struct address_space *mapping,  in page_cache_delete()  argument
129 	XA_STATE(xas, &mapping->i_pages, folio->index);  in page_cache_delete()
132 	mapping_set_update(&xas, mapping);  in page_cache_delete()
145 	folio->mapping = NULL;  in page_cache_delete()
147 	mapping->nrpages -= nr;  in page_cache_delete()
150 static void filemap_unaccount_folio(struct address_space *mapping,  in filemap_unaccount_folio()  argument
163 		if (mapping_exiting(mapping) && !folio_test_large(folio)) {  in filemap_unaccount_folio()
192 		filemap_nr_thps_dec(mapping);  in filemap_unaccount_folio()
210 			 mapping_can_writeback(mapping)))  in filemap_unaccount_folio()
211 		folio_account_cleaned(folio, inode_to_wb(mapping->host));  in filemap_unaccount_folio()
221 	struct address_space *mapping = folio->mapping;  in __filemap_remove_folio()  local
224 	filemap_unaccount_folio(mapping, folio);  in __filemap_remove_folio()
225 	page_cache_delete(mapping, folio, shadow);  in __filemap_remove_folio()
228 void filemap_free_folio(struct address_space *mapping, struct folio *folio)  in filemap_free_folio()  argument
233 	free_folio = mapping->a_ops->free_folio;  in filemap_free_folio()
252 	struct address_space *mapping = folio->mapping;  in filemap_remove_folio()  local
255 	spin_lock(&mapping->host->i_lock);  in filemap_remove_folio()
256 	xa_lock_irq(&mapping->i_pages);  in filemap_remove_folio()
258 	xa_unlock_irq(&mapping->i_pages);  in filemap_remove_folio()
259 	if (mapping_shrinkable(mapping))  in filemap_remove_folio()
260 		inode_add_lru(mapping->host);  in filemap_remove_folio()
261 	spin_unlock(&mapping->host->i_lock);  in filemap_remove_folio()
263 	filemap_free_folio(mapping, folio);  in filemap_remove_folio()
279 static void page_cache_delete_batch(struct address_space *mapping,  in page_cache_delete_batch()  argument
282 	XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);  in page_cache_delete_batch()
287 	mapping_set_update(&xas, mapping);  in page_cache_delete_batch()
310 		folio->mapping = NULL;  in page_cache_delete_batch()
317 	mapping->nrpages -= total_pages;  in page_cache_delete_batch()
320 void delete_from_page_cache_batch(struct address_space *mapping,  in delete_from_page_cache_batch()  argument
328 	spin_lock(&mapping->host->i_lock);  in delete_from_page_cache_batch()
329 	xa_lock_irq(&mapping->i_pages);  in delete_from_page_cache_batch()
334 		filemap_unaccount_folio(mapping, folio);  in delete_from_page_cache_batch()
336 	page_cache_delete_batch(mapping, fbatch);  in delete_from_page_cache_batch()
337 	xa_unlock_irq(&mapping->i_pages);  in delete_from_page_cache_batch()
338 	if (mapping_shrinkable(mapping))  in delete_from_page_cache_batch()
339 		inode_add_lru(mapping->host);  in delete_from_page_cache_batch()
340 	spin_unlock(&mapping->host->i_lock);  in delete_from_page_cache_batch()
343 		filemap_free_folio(mapping, fbatch->folios[i]);  in delete_from_page_cache_batch()
346 int filemap_check_errors(struct address_space *mapping)  in filemap_check_errors()  argument
350 	if (test_bit(AS_ENOSPC, &mapping->flags) &&  in filemap_check_errors()
351 	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))  in filemap_check_errors()
353 	if (test_bit(AS_EIO, &mapping->flags) &&  in filemap_check_errors()
354 	    test_and_clear_bit(AS_EIO, &mapping->flags))  in filemap_check_errors()
360 static int filemap_check_and_keep_errors(struct address_space *mapping)  in filemap_check_and_keep_errors()  argument
363 	if (test_bit(AS_EIO, &mapping->flags))  in filemap_check_and_keep_errors()
365 	if (test_bit(AS_ENOSPC, &mapping->flags))  in filemap_check_and_keep_errors()
380 int filemap_fdatawrite_wbc(struct address_space *mapping,  in filemap_fdatawrite_wbc()  argument
385 	if (!mapping_can_writeback(mapping) ||  in filemap_fdatawrite_wbc()
386 	    !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))  in filemap_fdatawrite_wbc()
389 	wbc_attach_fdatawrite_inode(wbc, mapping->host);  in filemap_fdatawrite_wbc()
390 	ret = do_writepages(mapping, wbc);  in filemap_fdatawrite_wbc()
413 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in __filemap_fdatawrite_range()  argument
423 	return filemap_fdatawrite_wbc(mapping, &wbc);  in __filemap_fdatawrite_range()
426 static inline int __filemap_fdatawrite(struct address_space *mapping,  in __filemap_fdatawrite()  argument
429 	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);  in __filemap_fdatawrite()
432 int filemap_fdatawrite(struct address_space *mapping)  in filemap_fdatawrite()  argument
434 	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);  in filemap_fdatawrite()
438 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in filemap_fdatawrite_range()  argument
441 	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);  in filemap_fdatawrite_range()
454 int filemap_flush(struct address_space *mapping)  in filemap_flush()  argument
456 	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);  in filemap_flush()
472 bool filemap_range_has_page(struct address_space *mapping,  in filemap_range_has_page()  argument
476 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);  in filemap_range_has_page()
503 static void __filemap_fdatawait_range(struct address_space *mapping,  in __filemap_fdatawait_range()  argument
516 		nr_folios = filemap_get_folios_tag(mapping, &index, end,  in __filemap_fdatawait_range()
549 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,  in filemap_fdatawait_range()  argument
552 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in filemap_fdatawait_range()
553 	return filemap_check_errors(mapping);  in filemap_fdatawait_range()
571 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,  in filemap_fdatawait_range_keep_errors()  argument
574 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in filemap_fdatawait_range_keep_errors()
575 	return filemap_check_and_keep_errors(mapping);  in filemap_fdatawait_range_keep_errors()
597 	struct address_space *mapping = file->f_mapping;  in file_fdatawait_range()  local
599 	__filemap_fdatawait_range(mapping, start_byte, end_byte);  in file_fdatawait_range()
618 int filemap_fdatawait_keep_errors(struct address_space *mapping)  in filemap_fdatawait_keep_errors()  argument
620 	__filemap_fdatawait_range(mapping, 0, LLONG_MAX);  in filemap_fdatawait_keep_errors()
621 	return filemap_check_and_keep_errors(mapping);  in filemap_fdatawait_keep_errors()
626 static bool mapping_needs_writeback(struct address_space *mapping)  in mapping_needs_writeback()  argument
628 	return mapping->nrpages;  in mapping_needs_writeback()
631 bool filemap_range_has_writeback(struct address_space *mapping,  in filemap_range_has_writeback()  argument
634 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);  in filemap_range_has_writeback()
669 int filemap_write_and_wait_range(struct address_space *mapping,  in filemap_write_and_wait_range()  argument
677 	if (mapping_needs_writeback(mapping)) {  in filemap_write_and_wait_range()
678 		err = __filemap_fdatawrite_range(mapping, lstart, lend,  in filemap_write_and_wait_range()
687 			__filemap_fdatawait_range(mapping, lstart, lend);  in filemap_write_and_wait_range()
689 	err2 = filemap_check_errors(mapping);  in filemap_write_and_wait_range()
696 void __filemap_set_wb_err(struct address_space *mapping, int err)  in __filemap_set_wb_err()  argument
698 	errseq_t eseq = errseq_set(&mapping->wb_err, err);  in __filemap_set_wb_err()
700 	trace_filemap_set_wb_err(mapping, eseq);  in __filemap_set_wb_err()
732 	struct address_space *mapping = file->f_mapping;  in file_check_and_advance_wb_err()  local
735 	if (errseq_check(&mapping->wb_err, old)) {  in file_check_and_advance_wb_err()
739 		err = errseq_check_and_advance(&mapping->wb_err,  in file_check_and_advance_wb_err()
750 	clear_bit(AS_EIO, &mapping->flags);  in file_check_and_advance_wb_err()
751 	clear_bit(AS_ENOSPC, &mapping->flags);  in file_check_and_advance_wb_err()
775 	struct address_space *mapping = file->f_mapping;  in file_write_and_wait_range()  local
780 	if (mapping_needs_writeback(mapping)) {  in file_write_and_wait_range()
781 		err = __filemap_fdatawrite_range(mapping, lstart, lend,  in file_write_and_wait_range()
785 			__filemap_fdatawait_range(mapping, lstart, lend);  in file_write_and_wait_range()
809 	struct address_space *mapping = old->mapping;  in replace_page_cache_folio()  local
810 	void (*free_folio)(struct folio *) = mapping->a_ops->free_folio;  in replace_page_cache_folio()
812 	XA_STATE(xas, &mapping->i_pages, offset);  in replace_page_cache_folio()
816 	VM_BUG_ON_FOLIO(new->mapping, new);  in replace_page_cache_folio()
819 	new->mapping = mapping;  in replace_page_cache_folio()
827 	old->mapping = NULL;  in replace_page_cache_folio()
844 noinline int __filemap_add_folio(struct address_space *mapping,  in __filemap_add_folio()  argument
847 	XA_STATE(xas, &mapping->i_pages, index);  in __filemap_add_folio()
854 	mapping_set_update(&xas, mapping);  in __filemap_add_folio()
868 	folio->mapping = mapping;  in __filemap_add_folio()
894 				BUG_ON(shmem_mapping(mapping));  in __filemap_add_folio()
904 		mapping->nrpages += nr;  in __filemap_add_folio()
925 	folio->mapping = NULL;  in __filemap_add_folio()
932 int filemap_add_folio(struct address_space *mapping, struct folio *folio,  in filemap_add_folio()  argument
939 	ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow);  in filemap_add_folio()
1731 pgoff_t page_cache_next_miss(struct address_space *mapping,  in page_cache_next_miss()  argument
1734 	XA_STATE(xas, &mapping->i_pages, index);  in page_cache_next_miss()
1767 pgoff_t page_cache_prev_miss(struct address_space *mapping,  in page_cache_prev_miss()  argument
1770 	XA_STATE(xas, &mapping->i_pages, index);  in page_cache_prev_miss()
1816 void *filemap_get_entry(struct address_space *mapping, pgoff_t index)  in filemap_get_entry()  argument
1818 	XA_STATE(xas, &mapping->i_pages, index);  in filemap_get_entry()
1863 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,  in __filemap_get_folio()  argument
1869 	folio = filemap_get_entry(mapping, index);  in __filemap_get_folio()
1886 		if (unlikely(folio->mapping != mapping)) {  in __filemap_get_folio()
1909 		if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))  in __filemap_get_folio()
1920 		if (!mapping_large_folio_support(mapping))  in __filemap_get_folio()
1944 			err = filemap_add_folio(mapping, folio, index, gfp);  in __filemap_get_folio()
2024 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,  in find_get_entries()  argument
2027 	XA_STATE(xas, &mapping->i_pages, *start);  in find_get_entries()
2070 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,  in find_lock_entries()  argument
2073 	XA_STATE(xas, &mapping->i_pages, *start);  in find_lock_entries()
2085 			if (folio->mapping != mapping ||  in find_lock_entries()
2135 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,  in filemap_get_folios()  argument
2138 	XA_STATE(xas, &mapping->i_pages, *start);  in filemap_get_folios()
2188 unsigned filemap_get_folios_contig(struct address_space *mapping,  in filemap_get_folios_contig()  argument
2191 	XA_STATE(xas, &mapping->i_pages, *start);  in filemap_get_folios_contig()
2259 unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,  in filemap_get_folios_tag()  argument
2262 	XA_STATE(xas, &mapping->i_pages, *start);  in filemap_get_folios_tag()
2329 static void filemap_get_read_batch(struct address_space *mapping,  in filemap_get_read_batch()  argument
2332 	XA_STATE(xas, &mapping->i_pages, index);  in filemap_get_read_batch()
2398 static bool filemap_range_uptodate(struct address_space *mapping,  in filemap_range_uptodate()  argument
2407 	if (!mapping->a_ops->is_partially_uptodate)  in filemap_range_uptodate()
2409 	if (mapping->host->i_blkbits >= folio_shift(folio))  in filemap_range_uptodate()
2419 	return mapping->a_ops->is_partially_uptodate(folio, pos, count);  in filemap_range_uptodate()
2423 		struct address_space *mapping, size_t count,  in filemap_update_page()  argument
2429 		if (!filemap_invalidate_trylock_shared(mapping))  in filemap_update_page()
2432 		filemap_invalidate_lock_shared(mapping);  in filemap_update_page()
2440 			filemap_invalidate_unlock_shared(mapping);  in filemap_update_page()
2454 	if (!folio->mapping)  in filemap_update_page()
2458 	if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio,  in filemap_update_page()
2466 	error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,  in filemap_update_page()
2472 	filemap_invalidate_unlock_shared(mapping);  in filemap_update_page()
2479 		struct address_space *mapping, pgoff_t index,  in filemap_create_folio()  argument
2485 	folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0);  in filemap_create_folio()
2502 	filemap_invalidate_lock_shared(mapping);  in filemap_create_folio()
2503 	error = filemap_add_folio(mapping, folio, index,  in filemap_create_folio()
2504 			mapping_gfp_constraint(mapping, GFP_KERNEL));  in filemap_create_folio()
2510 	error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);  in filemap_create_folio()
2514 	filemap_invalidate_unlock_shared(mapping);  in filemap_create_folio()
2518 	filemap_invalidate_unlock_shared(mapping);  in filemap_create_folio()
2524 		struct address_space *mapping, struct folio *folio,  in filemap_readahead()  argument
2527 	DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index);  in filemap_readahead()
2539 	struct address_space *mapping = filp->f_mapping;  in filemap_get_pages()  local
2552 	filemap_get_read_batch(mapping, index, last_index - 1, fbatch);  in filemap_get_pages()
2556 		page_cache_sync_readahead(mapping, ra, filp, index,  in filemap_get_pages()
2558 		filemap_get_read_batch(mapping, index, last_index - 1, fbatch);  in filemap_get_pages()
2563 		err = filemap_create_folio(filp, mapping,  in filemap_get_pages()
2572 		err = filemap_readahead(iocb, filp, mapping, folio, last_index);  in filemap_get_pages()
2580 		err = filemap_update_page(iocb, mapping, count, folio,  in filemap_get_pages()
2622 	struct address_space *mapping = filp->f_mapping;  in filemap_read()  local
2623 	struct inode *inode = mapping->host;  in filemap_read()
2673 		writably_mapped = mapping_writably_mapped(mapping);  in filemap_read()
2728 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_write_and_wait()  local
2733 		if (filemap_range_needs_writeback(mapping, pos, end))  in kiocb_write_and_wait()
2738 	return filemap_write_and_wait_range(mapping, pos, end);  in kiocb_write_and_wait()
2743 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_invalidate_pages()  local
2750 		if (filemap_range_has_page(mapping, pos, end))  in kiocb_invalidate_pages()
2753 		ret = filemap_write_and_wait_range(mapping, pos, end);  in kiocb_invalidate_pages()
2764 	return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT,  in kiocb_invalidate_pages()
2800 		struct address_space *mapping = file->f_mapping;  in generic_file_read_iter()  local
2801 		struct inode *inode = mapping->host;  in generic_file_read_iter()
2808 		retval = mapping->a_ops->direct_IO(iocb, iter);  in generic_file_read_iter()
2982 		struct address_space *mapping, struct folio *folio,  in folio_seek_hole_data()  argument
2985 	const struct address_space_operations *ops = mapping->a_ops;  in folio_seek_hole_data()
2986 	size_t offset, bsz = i_blocksize(mapping->host);  in folio_seek_hole_data()
2996 	if (unlikely(folio->mapping != mapping))  in folio_seek_hole_data()
3039 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,  in mapping_seek_hole_data()  argument
3042 	XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);  in mapping_seek_hole_data()
3063 		start = folio_seek_hole_data(&xas, mapping, folio, start, pos,  in mapping_seek_hole_data()
3143 	struct address_space *mapping = file->f_mapping;  in do_sync_mmap_readahead()  local
3144 	DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff);  in do_sync_mmap_readahead()
3260 	struct address_space *mapping = file->f_mapping;  in filemap_fault()  local
3261 	struct inode *inode = mapping->host;  in filemap_fault()
3274 	folio = filemap_get_folio(mapping, index);  in filemap_fault()
3283 			filemap_invalidate_lock_shared(mapping);  in filemap_fault()
3298 			filemap_invalidate_lock_shared(mapping);  in filemap_fault()
3301 		folio = __filemap_get_folio(mapping, index,  in filemap_fault()
3307 			filemap_invalidate_unlock_shared(mapping);  in filemap_fault()
3316 	if (unlikely(folio->mapping != mapping)) {  in filemap_fault()
3352 		filemap_invalidate_unlock_shared(mapping);  in filemap_fault()
3376 	error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);  in filemap_fault()
3383 	filemap_invalidate_unlock_shared(mapping);  in filemap_fault()
3396 		filemap_invalidate_unlock_shared(mapping);  in filemap_fault()
3432 		struct address_space *mapping, pgoff_t end_pgoff)  in next_uptodate_folio()  argument
3455 		if (folio->mapping != mapping)  in next_uptodate_folio()
3459 		max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);  in next_uptodate_folio()
3563 	struct address_space *mapping = file->f_mapping;  in filemap_map_pages()  local
3566 	XA_STATE(xas, &mapping->i_pages, start_pgoff);  in filemap_map_pages()
3572 	folio = next_uptodate_folio(&xas, mapping, end_pgoff);  in filemap_map_pages()
3607 	} while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL);  in filemap_map_pages()
3624 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;  in filemap_page_mkwrite()  local
3628 	sb_start_pagefault(mapping->host->i_sb);  in filemap_page_mkwrite()
3631 	if (folio->mapping != mapping) {  in filemap_page_mkwrite()
3644 	sb_end_pagefault(mapping->host->i_sb);  in filemap_page_mkwrite()
3658 	struct address_space *mapping = file->f_mapping;  in generic_file_mmap()  local
3660 	if (!mapping->a_ops->read_folio)  in generic_file_mmap()
3695 static struct folio *do_read_cache_folio(struct address_space *mapping,  in do_read_cache_folio()  argument
3702 		filler = mapping->a_ops->read_folio;  in do_read_cache_folio()
3704 	folio = filemap_get_folio(mapping, index);  in do_read_cache_folio()
3709 		err = filemap_add_folio(mapping, folio, index, gfp);  in do_read_cache_folio()
3729 	if (!folio->mapping) {  in do_read_cache_folio()
3771 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index,  in read_cache_folio()  argument
3774 	return do_read_cache_folio(mapping, index, filler, file,  in read_cache_folio()
3775 			mapping_gfp_mask(mapping));  in read_cache_folio()
3796 struct folio *mapping_read_folio_gfp(struct address_space *mapping,  in mapping_read_folio_gfp()  argument
3799 	return do_read_cache_folio(mapping, index, NULL, NULL, gfp);  in mapping_read_folio_gfp()
3803 static struct page *do_read_cache_page(struct address_space *mapping,  in do_read_cache_page()  argument
3808 	folio = do_read_cache_folio(mapping, index, filler, file, gfp);  in do_read_cache_page()
3814 struct page *read_cache_page(struct address_space *mapping,  in read_cache_page()  argument
3817 	return do_read_cache_page(mapping, index, filler, file,  in read_cache_page()
3818 			mapping_gfp_mask(mapping));  in read_cache_page()
3837 struct page *read_cache_page_gfp(struct address_space *mapping,  in read_cache_page_gfp()  argument
3841 	return do_read_cache_page(mapping, index, NULL, NULL, gfp);  in read_cache_page_gfp()
3867 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_invalidate_post_direct_write()  local
3869 	if (mapping->nrpages &&  in kiocb_invalidate_post_direct_write()
3870 	    invalidate_inode_pages2_range(mapping,  in kiocb_invalidate_post_direct_write()
3879 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in generic_file_direct_write()  local
3894 	written = mapping->a_ops->direct_IO(iocb, from);  in generic_file_direct_write()
3914 		struct inode *inode = mapping->host;  in generic_file_direct_write()
3936 	struct address_space *mapping = file->f_mapping;  in generic_perform_write()  local
3937 	const struct address_space_operations *a_ops = mapping->a_ops;  in generic_perform_write()
3969 		status = a_ops->write_begin(file, mapping, pos, bytes,  in generic_perform_write()
3974 		if (mapping_writably_mapped(mapping))  in generic_perform_write()
3980 		status = a_ops->write_end(file, mapping, pos, bytes, copied,  in generic_perform_write()
4003 		balance_dirty_pages_ratelimited(mapping);  in generic_perform_write()
4037 	struct address_space *mapping = file->f_mapping;  in __generic_file_write_iter()  local
4038 	struct inode *inode = mapping->host;  in __generic_file_write_iter()
4118 	struct address_space * const mapping = folio->mapping;  in filemap_release_folio()  local
4126 	if (mapping && mapping->a_ops->release_folio)  in filemap_release_folio()
4127 		return mapping->a_ops->release_folio(folio, gfp);  in filemap_release_folio()
4145 static void filemap_cachestat(struct address_space *mapping,  in filemap_cachestat()  argument
4148 	XA_STATE(xas, &mapping->i_pages, first_index);  in filemap_cachestat()
4179 			if (shmem_mapping(mapping)) {  in filemap_cachestat()
4260 	struct address_space *mapping;  in SYSCALL_DEFINE4()  local
4289 	mapping = f.file->f_mapping;  in SYSCALL_DEFINE4()
4290 	filemap_cachestat(mapping, first_index, last_index, &cs);  in SYSCALL_DEFINE4()