1 /* SPDX-License-Identifier: GPL-2.0 */
21 unsigned long invalidate_mapping_pages(struct address_space *mapping,
26 	if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||  in invalidate_remote_inode()
27 	    S_ISLNK(inode->i_mode))  in invalidate_remote_inode()
28 		invalidate_mapping_pages(inode->i_mapping, 0, -1);  in invalidate_remote_inode()
30 int invalidate_inode_pages2(struct address_space *mapping);
31 int invalidate_inode_pages2_range(struct address_space *mapping,
36 int filemap_fdatawait_keep_errors(struct address_space *mapping);
38 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
41 static inline int filemap_fdatawait(struct address_space *mapping)  in filemap_fdatawait()  argument
43 	return filemap_fdatawait_range(mapping, 0, LLONG_MAX);  in filemap_fdatawait()
47 int filemap_write_and_wait_range(struct address_space *mapping,
49 int __filemap_fdatawrite_range(struct address_space *mapping,
51 int filemap_fdatawrite_range(struct address_space *mapping,
53 int filemap_check_errors(struct address_space *mapping);
54 void __filemap_set_wb_err(struct address_space *mapping, int err);
55 int filemap_fdatawrite_wbc(struct address_space *mapping,
58 static inline int filemap_write_and_wait(struct address_space *mapping)  in filemap_write_and_wait()  argument
60 	return filemap_write_and_wait_range(mapping, 0, LLONG_MAX);  in filemap_write_and_wait()
64  * filemap_set_wb_err - set a writeback error on an address_space
65  * @mapping: mapping in which to set writeback error
66  * @err: error to be set in mapping
74  * filemap_set_wb_err to record the error in the mapping so that it will be
77 static inline void filemap_set_wb_err(struct address_space *mapping, int err)  in filemap_set_wb_err()  argument
81 		__filemap_set_wb_err(mapping, err);  in filemap_set_wb_err()
85  * filemap_check_wb_err - has an error occurred since the mark was sampled?
86  * @mapping: mapping to check for writeback errors
87  * @since: previously-sampled errseq_t
89  * Grab the errseq_t value from the mapping, and see if it has changed "since"
94 static inline int filemap_check_wb_err(struct address_space *mapping,  in filemap_check_wb_err()  argument
97 	return errseq_check(&mapping->wb_err, since);  in filemap_check_wb_err()
101  * filemap_sample_wb_err - sample the current errseq_t to test for later errors
102  * @mapping: mapping to be sampled
107 static inline errseq_t filemap_sample_wb_err(struct address_space *mapping)  in filemap_sample_wb_err()  argument
109 	return errseq_sample(&mapping->wb_err);  in filemap_sample_wb_err()
113  * file_sample_sb_err - sample the current errseq_t to test for later errors
116  * Grab the most current superblock-level errseq_t value for the given
121 	return errseq_sample(&file->f_path.dentry->d_sb->s_wb_err);  in file_sample_sb_err()
125  * Flush file data before changing attributes.  Caller must hold any locks
132 	return filemap_write_and_wait(inode->i_mapping);  in inode_drain_writes()
135 static inline bool mapping_empty(struct address_space *mapping)  in mapping_empty()  argument
137 	return xa_empty(&mapping->i_pages);  in mapping_empty()
141  * mapping_shrinkable - test if page cache state allows inode reclaim
142  * @mapping: the page cache mapping
144  * This checks the mapping's cache state for the pupose of inode
150  * its LRU state don't nest inside the irq-safe i_pages lock.
161 static inline bool mapping_shrinkable(struct address_space *mapping)  in mapping_shrinkable()  argument
174 	head = rcu_access_pointer(mapping->i_pages.xa_head);  in mapping_shrinkable()
179 	 * The xarray stores single offset-0 entries directly in the  in mapping_shrinkable()
180 	 * head pointer, which allows non-resident page cache entries  in mapping_shrinkable()
191  * Bits in mapping->flags.
205  * mapping_set_error - record a writeback error in the address_space
206  * @mapping: the mapping in which an error should be set
207  * @error: the error to set in the mapping
215  * mapping_set_error to record the error in the mapping so that it can be
218 static inline void mapping_set_error(struct address_space *mapping, int error)  in mapping_set_error()  argument
224 	__filemap_set_wb_err(mapping, error);  in mapping_set_error()
227 	if (mapping->host)  in mapping_set_error()
228 		errseq_set(&mapping->host->i_sb->s_wb_err, error);  in mapping_set_error()
231 	if (error == -ENOSPC)  in mapping_set_error()
232 		set_bit(AS_ENOSPC, &mapping->flags);  in mapping_set_error()
234 		set_bit(AS_EIO, &mapping->flags);  in mapping_set_error()
237 static inline void mapping_set_unevictable(struct address_space *mapping)  in mapping_set_unevictable()  argument
239 	set_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_set_unevictable()
242 static inline void mapping_clear_unevictable(struct address_space *mapping)  in mapping_clear_unevictable()  argument
244 	clear_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_clear_unevictable()
247 static inline bool mapping_unevictable(struct address_space *mapping)  in mapping_unevictable()  argument
249 	return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_unevictable()
252 static inline void mapping_set_exiting(struct address_space *mapping)  in mapping_set_exiting()  argument
254 	set_bit(AS_EXITING, &mapping->flags);  in mapping_set_exiting()
257 static inline int mapping_exiting(struct address_space *mapping)  in mapping_exiting()  argument
259 	return test_bit(AS_EXITING, &mapping->flags);  in mapping_exiting()
262 static inline void mapping_set_no_writeback_tags(struct address_space *mapping)  in mapping_set_no_writeback_tags()  argument
264 	set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_set_no_writeback_tags()
267 static inline int mapping_use_writeback_tags(struct address_space *mapping)  in mapping_use_writeback_tags()  argument
269 	return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_use_writeback_tags()
272 static inline gfp_t mapping_gfp_mask(struct address_space * mapping)  in mapping_gfp_mask()  argument
274 	return mapping->gfp_mask;  in mapping_gfp_mask()
277 /* Restricts the given gfp_mask to what the mapping allows. */
278 static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,  in mapping_gfp_constraint()  argument
281 	return mapping_gfp_mask(mapping) & gfp_mask;  in mapping_gfp_constraint()
285  * This is non-atomic.  Only to be used before the mapping is activated.
290 	m->gfp_mask = mask;  in mapping_set_gfp_mask()
294  * mapping_set_large_folios() - Indicate the file supports large folios.
295  * @mapping: The file.
302  * is non-atomic.
304 static inline void mapping_set_large_folios(struct address_space *mapping)  in mapping_set_large_folios()  argument
306 	__set_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);  in mapping_set_large_folios()
313 static inline bool mapping_large_folio_support(struct address_space *mapping)  in mapping_large_folio_support()  argument
316 		test_bit(AS_LARGE_FOLIO_SUPPORT, &mapping->flags);  in mapping_large_folio_support()
319 static inline int filemap_nr_thps(struct address_space *mapping)  in filemap_nr_thps()  argument
322 	return atomic_read(&mapping->nr_thps);  in filemap_nr_thps()
328 static inline void filemap_nr_thps_inc(struct address_space *mapping)  in filemap_nr_thps_inc()  argument
331 	if (!mapping_large_folio_support(mapping))  in filemap_nr_thps_inc()
332 		atomic_inc(&mapping->nr_thps);  in filemap_nr_thps_inc()
334 	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);  in filemap_nr_thps_inc()
338 static inline void filemap_nr_thps_dec(struct address_space *mapping)  in filemap_nr_thps_dec()  argument
341 	if (!mapping_large_folio_support(mapping))  in filemap_nr_thps_dec()
342 		atomic_dec(&mapping->nr_thps);  in filemap_nr_thps_dec()
344 	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);  in filemap_nr_thps_dec()
353  * folio_file_mapping - Find the mapping this folio belongs to.
356  * For folios which are in the page cache, return the mapping that this
357  * page belongs to.  Folios in the swap cache return the mapping of the
358  * swap file or swap device where the data is stored.  This is different
359  * from the mapping returned by folio_mapping().  The only reason to
360  * use it is if, like NFS, you return 0 from ->activate_swapfile.
369 	return folio->mapping;  in folio_file_mapping()
390  * folio_inode - Get the host inode for this folio.
400 	return folio->mapping->host;  in folio_inode()
404  * folio_attach_private - Attach private data to a folio.
405  * @folio: Folio to attach data to.
406  * @data: Data to attach to folio.
408  * Attaching private data to a folio increments the page's reference count.
409  * The data must be detached before the folio will be freed.
411 static inline void folio_attach_private(struct folio *folio, void *data)  in folio_attach_private()  argument
414 	folio->private = data;  in folio_attach_private()
419  * folio_change_private - Change private data on a folio.
420  * @folio: Folio to change the data on.
421  * @data: Data to set on the folio.
423  * Change the private data attached to a folio and return the old
424  * data.  The page must previously have had data attached and the data
427  * Return: Data that was previously attached to the folio.
429 static inline void *folio_change_private(struct folio *folio, void *data)  in folio_change_private()  argument
433 	folio->private = data;  in folio_change_private()
438  * folio_detach_private - Detach private data from a folio.
439  * @folio: Folio to detach data from.
441  * Removes the data that was previously attached to the folio and decrements
444  * Return: Data that was attached to the folio.
448 	void *data = folio_get_private(folio);  in folio_detach_private()  local
453 	folio->private = NULL;  in folio_detach_private()
456 	return data;  in folio_detach_private()
459 static inline void attach_page_private(struct page *page, void *data)  in attach_page_private()  argument
461 	folio_attach_private(page_folio(page), data);  in attach_page_private()
480 	return &filemap_alloc_folio(gfp, 0)->page;  in __page_cache_alloc()
495 pgoff_t page_cache_next_miss(struct address_space *mapping,
497 pgoff_t page_cache_prev_miss(struct address_space *mapping,
511 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
513 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
517  * filemap_get_folio - Find and get a folio.
518  * @mapping: The address_space to search.
521  * Looks up the page cache entry at @mapping & @index.  If a folio is
526 static inline struct folio *filemap_get_folio(struct address_space *mapping,  in filemap_get_folio()  argument
529 	return __filemap_get_folio(mapping, index, 0, 0);  in filemap_get_folio()
533  * filemap_lock_folio - Find and lock a folio.
534  * @mapping: The address_space to search.
537  * Looks up the page cache entry at @mapping & @index.  If a folio is
544 static inline struct folio *filemap_lock_folio(struct address_space *mapping,  in filemap_lock_folio()  argument
547 	return __filemap_get_folio(mapping, index, FGP_LOCK, 0);  in filemap_lock_folio()
551  * find_get_page - find and get a page reference
552  * @mapping: the address_space to search
555  * Looks up the page cache slot at @mapping & @offset.  If there is a
560 static inline struct page *find_get_page(struct address_space *mapping,  in find_get_page()  argument
563 	return pagecache_get_page(mapping, offset, 0, 0);  in find_get_page()
566 static inline struct page *find_get_page_flags(struct address_space *mapping,  in find_get_page_flags()  argument
569 	return pagecache_get_page(mapping, offset, fgp_flags, 0);  in find_get_page_flags()
573  * find_lock_page - locate, pin and lock a pagecache page
574  * @mapping: the address_space to search
577  * Looks up the page cache entry at @mapping & @index.  If there is a
585 static inline struct page *find_lock_page(struct address_space *mapping,  in find_lock_page()  argument
588 	return pagecache_get_page(mapping, index, FGP_LOCK, 0);  in find_lock_page()
592  * find_or_create_page - locate or add a pagecache page
593  * @mapping: the page's address_space
594  * @index: the page's index into the mapping
597  * Looks up the page cache slot at @mapping & @offset.  If there is a
610 static inline struct page *find_or_create_page(struct address_space *mapping,  in find_or_create_page()  argument
613 	return pagecache_get_page(mapping, index,  in find_or_create_page()
619  * grab_cache_page_nowait - returns locked page at given index in given cache
620  * @mapping: target address_space
624  * This is intended for speculative data generators, where the data can
631 static inline struct page *grab_cache_page_nowait(struct address_space *mapping,  in grab_cache_page_nowait()  argument
634 	return pagecache_get_page(mapping, index,  in grab_cache_page_nowait()
636 			mapping_gfp_mask(mapping));  in grab_cache_page_nowait()
639 #define swapcache_index(folio)	__page_file_index(&(folio)->page)
642  * folio_index - File index of a folio.
656         return folio->index;  in folio_index()
660  * folio_next_index - Get the index of the next folio.
667 	return folio->index + folio_nr_pages(folio);  in folio_next_index()
671  * folio_file_page - The page for a particular index.
678  * Return: The page containing the file data for this index.
684 		return &folio->page;  in folio_file_page()
685 	return folio_page(folio, index & (folio_nr_pages(folio) - 1));  in folio_file_page()
689  * folio_contains - Does this folio contain this index?
702 		return folio->index == index;  in folio_contains()
703 	return index - folio_index(folio) < folio_nr_pages(folio);  in folio_contains()
716 	return head + (index & (thp_nr_pages(head) - 1));  in find_subpage()
719 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
721 unsigned filemap_get_folios_contig(struct address_space *mapping,
723 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
726 static inline unsigned find_get_pages_tag(struct address_space *mapping,  in find_get_pages_tag()  argument
730 	return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag,  in find_get_pages_tag()
734 struct page *grab_cache_page_write_begin(struct address_space *mapping,
740 static inline struct page *grab_cache_page(struct address_space *mapping,  in grab_cache_page()  argument
743 	return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));  in grab_cache_page()
750 extern struct page * read_cache_page_gfp(struct address_space *mapping,
753 static inline struct page *read_mapping_page(struct address_space *mapping,  in read_mapping_page()  argument
756 	return read_cache_page(mapping, index, NULL, file);  in read_mapping_page()
759 static inline struct folio *read_mapping_folio(struct address_space *mapping,  in read_mapping_folio()  argument
762 	return read_cache_folio(mapping, index, NULL, file);  in read_mapping_folio()
766  * Get index of the page within radix-tree (but not for hugetlb pages).
767  * (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE)
774 		return page->index;  in page_to_index()
778 	 *  We don't initialize ->index for tail pages: calculate based on  in page_to_index()
781 	return head->index + page - head;  in page_to_index()
788  * (TODO: hugetlb pages should have ->index in PAGE_SIZE)
798  * Return byte-offset into filesystem object for page.
802 	return ((loff_t)page->index) << PAGE_SHIFT;  in page_offset()
811  * folio_pos - Returns the byte position of this folio in its file.
816 	return page_offset(&folio->page);  in folio_pos()
820  * folio_file_pos - Returns the byte position of this folio in its file.
828 	return page_file_offset(&folio->page);  in folio_file_pos()
833  * (TODO: hugetlb folios should have ->index in PAGE_SIZE)
838 		return hugetlb_basepage_index(&folio->page);  in folio_pgoff()
839 	return folio->index;  in folio_pgoff()
851 	pgoff = (address - vma->vm_start) >> PAGE_SHIFT;  in linear_page_index()
852 	pgoff += vma->vm_pgoff;  in linear_page_index()
871 	if (wait_page->folio != key->folio)  in wake_page_match()
873 	key->page_match = 1;  in wake_page_match()
875 	if (wait_page->bit_nr != key->bit_nr)  in wake_page_match()
889  * folio_trylock() - Attempt to lock a folio.
914  * folio_lock() - Lock this folio.
921  * is sufficient to keep folio->mapping stable.
925  * cross a page boundary).  Other modifications to the data in the folio
943  * lock_page() - Lock the folio containing this page.
964  * folio_lock_killable() - Lock this folio, interruptible by a fatal signal.
971  * Return: 0 if the lock was acquired; -EINTR if a fatal signal was received.
983  * signals.  It returns 0 if it locked the page and -EINTR if it was
992  * folio_lock_or_retry - Lock the folio, unless this would block and the
1051 		struct address_space *mapping, int warn)  in __set_page_dirty()  argument
1053 	__folio_mark_dirty(page_folio(page), mapping, warn);  in __set_page_dirty()
1073 bool noop_dirty_folio(struct address_space *mapping, struct folio *folio);
1076 int filemap_migrate_folio(struct address_space *mapping, struct folio *dst,
1100 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
1102 int filemap_add_folio(struct address_space *mapping, struct folio *folio,
1108 void delete_from_page_cache_batch(struct address_space *mapping,
1115 /* Must be non-static for BPF error injection */
1116 int __filemap_add_folio(struct address_space *mapping, struct folio *folio,
1119 bool filemap_range_has_writeback(struct address_space *mapping,
1123  * filemap_range_needs_writeback - check if range potentially needs writeback
1124  * @mapping:           address space within which to check
1136 static inline bool filemap_range_needs_writeback(struct address_space *mapping,  in filemap_range_needs_writeback()  argument
1140 	if (!mapping->nrpages)  in filemap_range_needs_writeback()
1142 	if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&  in filemap_range_needs_writeback()
1143 	    !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))  in filemap_range_needs_writeback()
1145 	return filemap_range_has_writeback(mapping, start_byte, end_byte);  in filemap_range_needs_writeback()
1149  * struct readahead_control - Describes a readahead request.
1152  * implement the ->readahead method should call readahead_page() or
1161  * @mapping: Readahead this filesystem object.
1166 	struct address_space *mapping;  member
1179 		.mapping = m,						\
1195  * page_cache_sync_readahead - generic file readahead
1196  * @mapping: address_space which holds the pagecache and I/O vectors
1208 void page_cache_sync_readahead(struct address_space *mapping,  in page_cache_sync_readahead()  argument
1212 	DEFINE_READAHEAD(ractl, file, ra, mapping, index);  in page_cache_sync_readahead()
1217  * page_cache_async_readahead - file readahead for marked pages
1218  * @mapping: address_space which holds the pagecache and I/O vectors
1231 void page_cache_async_readahead(struct address_space *mapping,  in page_cache_async_readahead()  argument
1235 	DEFINE_READAHEAD(ractl, file, ra, mapping, index);  in page_cache_async_readahead()
1243 	BUG_ON(ractl->_batch_count > ractl->_nr_pages);  in __readahead_folio()
1244 	ractl->_nr_pages -= ractl->_batch_count;  in __readahead_folio()
1245 	ractl->_index += ractl->_batch_count;  in __readahead_folio()
1247 	if (!ractl->_nr_pages) {  in __readahead_folio()
1248 		ractl->_batch_count = 0;  in __readahead_folio()
1252 	folio = xa_load(&ractl->mapping->i_pages, ractl->_index);  in __readahead_folio()
1254 	ractl->_batch_count = folio_nr_pages(folio);  in __readahead_folio()
1260  * readahead_page - Get the next page to read.
1272 	return &folio->page;  in readahead_page()
1276  * readahead_folio - Get the next folio to read.
1296 	XA_STATE(xas, &rac->mapping->i_pages, 0);  in __readahead_batch()
1299 	BUG_ON(rac->_batch_count > rac->_nr_pages);  in __readahead_batch()
1300 	rac->_nr_pages -= rac->_batch_count;  in __readahead_batch()
1301 	rac->_index += rac->_batch_count;  in __readahead_batch()
1302 	rac->_batch_count = 0;  in __readahead_batch()
1304 	xas_set(&xas, rac->_index);  in __readahead_batch()
1306 	xas_for_each(&xas, page, rac->_index + rac->_nr_pages - 1) {  in __readahead_batch()
1312 		rac->_batch_count += thp_nr_pages(page);  in __readahead_batch()
1322  * readahead_page_batch - Get a batch of pages to read.
1336  * readahead_pos - The byte offset into the file of this readahead request.
1341 	return (loff_t)rac->_index * PAGE_SIZE;  in readahead_pos()
1345  * readahead_length - The number of bytes in this readahead request.
1350 	return rac->_nr_pages * PAGE_SIZE;  in readahead_length()
1354  * readahead_index - The index of the first page in this readahead request.
1359 	return rac->_index;  in readahead_index()
1363  * readahead_count - The number of pages in this readahead request.
1368 	return rac->_nr_pages;  in readahead_count()
1372  * readahead_batch_length - The number of bytes in the current batch.
1377 	return rac->_batch_count * PAGE_SIZE;  in readahead_batch_length()
1382 	return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>  in dir_pages()
1387  * folio_mkwrite_check_truncate - check if folio was truncated
1392  * or -EFAULT if the folio was truncated.
1401 	if (!folio->mapping)  in folio_mkwrite_check_truncate()
1402 		return -EFAULT;  in folio_mkwrite_check_truncate()
1405 	if (folio_next_index(folio) - 1 < index)  in folio_mkwrite_check_truncate()
1408 	if (folio->index > index || !offset)  in folio_mkwrite_check_truncate()
1409 		return -EFAULT;  in folio_mkwrite_check_truncate()
1415  * page_mkwrite_check_truncate - check if page was truncated
1420  * or -EFAULT if the page was truncated.
1429 	if (page->mapping != inode->i_mapping)  in page_mkwrite_check_truncate()
1430 		return -EFAULT;  in page_mkwrite_check_truncate()
1433 	if (page->index < index)  in page_mkwrite_check_truncate()
1436 	if (page->index > index || !offset)  in page_mkwrite_check_truncate()
1437 		return -EFAULT;  in page_mkwrite_check_truncate()
1443  * i_blocks_per_folio - How many blocks fit in this folio.
1456 	return folio_size(folio) >> inode->i_blkbits;  in i_blocks_per_folio()