1 /* SPDX-License-Identifier: GPL-2.0 */
21 static inline bool mapping_empty(struct address_space *mapping)  in mapping_empty()  argument
23 	return xa_empty(&mapping->i_pages);  in mapping_empty()
27  * Bits in mapping->flags.
41  * mapping_set_error - record a writeback error in the address_space
42  * @mapping: the mapping in which an error should be set
43  * @error: the error to set in the mapping
51  * mapping_set_error to record the error in the mapping so that it can be
54 static inline void mapping_set_error(struct address_space *mapping, int error)  in mapping_set_error()  argument
60 	__filemap_set_wb_err(mapping, error);  in mapping_set_error()
63 	if (mapping->host)  in mapping_set_error()
64 		errseq_set(&mapping->host->i_sb->s_wb_err, error);  in mapping_set_error()
67 	if (error == -ENOSPC)  in mapping_set_error()
68 		set_bit(AS_ENOSPC, &mapping->flags);  in mapping_set_error()
70 		set_bit(AS_EIO, &mapping->flags);  in mapping_set_error()
73 static inline void mapping_set_unevictable(struct address_space *mapping)  in mapping_set_unevictable()  argument
75 	set_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_set_unevictable()
78 static inline void mapping_clear_unevictable(struct address_space *mapping)  in mapping_clear_unevictable()  argument
80 	clear_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_clear_unevictable()
83 static inline bool mapping_unevictable(struct address_space *mapping)  in mapping_unevictable()  argument
85 	return mapping && test_bit(AS_UNEVICTABLE, &mapping->flags);  in mapping_unevictable()
88 static inline void mapping_set_exiting(struct address_space *mapping)  in mapping_set_exiting()  argument
90 	set_bit(AS_EXITING, &mapping->flags);  in mapping_set_exiting()
93 static inline int mapping_exiting(struct address_space *mapping)  in mapping_exiting()  argument
95 	return test_bit(AS_EXITING, &mapping->flags);  in mapping_exiting()
98 static inline void mapping_set_no_writeback_tags(struct address_space *mapping)  in mapping_set_no_writeback_tags()  argument
100 	set_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_set_no_writeback_tags()
103 static inline int mapping_use_writeback_tags(struct address_space *mapping)  in mapping_use_writeback_tags()  argument
105 	return !test_bit(AS_NO_WRITEBACK_TAGS, &mapping->flags);  in mapping_use_writeback_tags()
108 static inline gfp_t mapping_gfp_mask(struct address_space * mapping)  in mapping_gfp_mask()  argument
110 	return mapping->gfp_mask;  in mapping_gfp_mask()
113 /* Restricts the given gfp_mask to what the mapping allows. */
114 static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,  in mapping_gfp_constraint()  argument
117 	return mapping_gfp_mask(mapping) & gfp_mask;  in mapping_gfp_constraint()
121  * This is non-atomic.  Only to be used before the mapping is activated.
126 	m->gfp_mask = mask;  in mapping_set_gfp_mask()
129 static inline bool mapping_thp_support(struct address_space *mapping)  in mapping_thp_support()  argument
131 	return test_bit(AS_THP_SUPPORT, &mapping->flags);  in mapping_thp_support()
134 static inline int filemap_nr_thps(struct address_space *mapping)  in filemap_nr_thps()  argument
137 	return atomic_read(&mapping->nr_thps);  in filemap_nr_thps()
143 static inline void filemap_nr_thps_inc(struct address_space *mapping)  in filemap_nr_thps_inc()  argument
146 	if (!mapping_thp_support(mapping))  in filemap_nr_thps_inc()
147 		atomic_inc(&mapping->nr_thps);  in filemap_nr_thps_inc()
153 static inline void filemap_nr_thps_dec(struct address_space *mapping)  in filemap_nr_thps_dec()  argument
156 	if (!mapping_thp_support(mapping))  in filemap_nr_thps_dec()
157 		atomic_dec(&mapping->nr_thps);  in filemap_nr_thps_dec()
181  * been used to lookup the page in the pagecache radix-tree (or page table):
192  * get_user_pages) locking protocol, where the lookup-side (eg. find_get_page)
198  * Remove-side that cares about stability of _refcount (eg. reclaim) has the
205  * - 2 runs before A: in this case, A sees elevated refcount and bails out
206  * - A runs before 2: in this case, 2 sees zero refcount and retries;
213  * such a re-insertion, depending on order that locks are granted.
226 	 * Preempt must be disabled here - we rely on rcu_read_lock doing  in __page_cache_add_speculative()
263  * attach_page_private - Attach private data to a page.
264  * @page: Page to attach data to.
265  * @data: Data to attach to page.
267  * Attaching private data to a page increments the page's reference count.
268  * The data must be detached before the page will be freed.
270 static inline void attach_page_private(struct page *page, void *data)  in attach_page_private()  argument
273 	set_page_private(page, (unsigned long)data);  in attach_page_private()
278  * detach_page_private - Detach private data from a page.
279  * @page: Page to detach data from.
281  * Removes the data that was previously attached to the page and decrements
284  * Return: Data that was attached to the page.
288 	void *data = (void *)page_private(page);  in detach_page_private()  local
296 	return data;  in detach_page_private()
320 pgoff_t page_cache_next_miss(struct address_space *mapping,
322 pgoff_t page_cache_prev_miss(struct address_space *mapping,
335 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
339  * find_get_page - find and get a page reference
340  * @mapping: the address_space to search
343  * Looks up the page cache slot at @mapping & @offset.  If there is a
348 static inline struct page *find_get_page(struct address_space *mapping,  in find_get_page()  argument
351 	return pagecache_get_page(mapping, offset, 0, 0);  in find_get_page()
354 static inline struct page *find_get_page_flags(struct address_space *mapping,  in find_get_page_flags()  argument
357 	return pagecache_get_page(mapping, offset, fgp_flags, 0);  in find_get_page_flags()
361  * find_lock_page - locate, pin and lock a pagecache page
362  * @mapping: the address_space to search
365  * Looks up the page cache entry at @mapping & @index.  If there is a
373 static inline struct page *find_lock_page(struct address_space *mapping,  in find_lock_page()  argument
376 	return pagecache_get_page(mapping, index, FGP_LOCK, 0);  in find_lock_page()
380  * find_lock_head - Locate, pin and lock a pagecache page.
381  * @mapping: The address_space to search.
384  * Looks up the page cache entry at @mapping & @index.  If there is a
392 static inline struct page *find_lock_head(struct address_space *mapping,  in find_lock_head()  argument
395 	return pagecache_get_page(mapping, index, FGP_LOCK | FGP_HEAD, 0);  in find_lock_head()
399  * find_or_create_page - locate or add a pagecache page
400  * @mapping: the page's address_space
401  * @index: the page's index into the mapping
404  * Looks up the page cache slot at @mapping & @offset.  If there is a
417 static inline struct page *find_or_create_page(struct address_space *mapping,  in find_or_create_page()  argument
420 	return pagecache_get_page(mapping, index,  in find_or_create_page()
426  * grab_cache_page_nowait - returns locked page at given index in given cache
427  * @mapping: target address_space
431  * This is intended for speculative data generators, where the data can
438 static inline struct page *grab_cache_page_nowait(struct address_space *mapping,  in grab_cache_page_nowait()  argument
441 	return pagecache_get_page(mapping, index,  in grab_cache_page_nowait()
443 			mapping_gfp_mask(mapping));  in grab_cache_page_nowait()
451 		return head->index == index;  in thp_contains()
452 	return page_index(head) == (index & ~(thp_nr_pages(head) - 1UL));  in thp_contains()
465 	return head + (index & (thp_nr_pages(head) - 1));  in find_subpage()
468 unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
470 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
473 static inline unsigned find_get_pages(struct address_space *mapping,  in find_get_pages()  argument
477 	return find_get_pages_range(mapping, start, (pgoff_t)-1, nr_pages,  in find_get_pages()
480 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
482 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
485 static inline unsigned find_get_pages_tag(struct address_space *mapping,  in find_get_pages_tag()  argument
489 	return find_get_pages_range_tag(mapping, index, (pgoff_t)-1, tag,  in find_get_pages_tag()
493 struct page *grab_cache_page_write_begin(struct address_space *mapping,
499 static inline struct page *grab_cache_page(struct address_space *mapping,  in grab_cache_page()  argument
502 	return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));  in grab_cache_page()
505 extern struct page * read_cache_page(struct address_space *mapping,
506 				pgoff_t index, filler_t *filler, void *data);
507 extern struct page * read_cache_page_gfp(struct address_space *mapping,
509 extern int read_cache_pages(struct address_space *mapping,
510 		struct list_head *pages, filler_t *filler, void *data);
512 static inline struct page *read_mapping_page(struct address_space *mapping,  in read_mapping_page()  argument
513 				pgoff_t index, void *data)  in read_mapping_page()  argument
515 	return read_cache_page(mapping, index, NULL, data);  in read_mapping_page()
519  * Get index of the page within radix-tree (but not for hugetlb pages).
520  * (TODO: remove once hugetlb pages will have ->index in PAGE_SIZE)
527 		return page->index;  in page_to_index()
531 	 *  We don't initialize ->index for tail pages: calculate based on  in page_to_index()
534 	return head->index + page - head;  in page_to_index()
541  * (TODO: hugetlb pages should have ->index in PAGE_SIZE)
551  * Return byte-offset into filesystem object for page.
555 	return ((loff_t)page->index) << PAGE_SHIFT;  in page_offset()
572 	pgoff = (address - vma->vm_start) >> PAGE_SHIFT;  in linear_page_index()
573 	pgoff += vma->vm_pgoff;  in linear_page_index()
592 	if (wait_page->page != key->page)  in wake_page_match()
594 	key->page_match = 1;  in wake_page_match()
596 	if (wait_page->bit_nr != key->bit_nr)  in wake_page_match()
615 	return (likely(!test_and_set_bit_lock(PG_locked, &page->flags)));  in trylock_page()
630  * signals.  It returns 0 if it locked the page and -EINTR if it was
642  * lock_page_async - Lock the page, unless this would block. If the page
646  * Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page
658  * lock_page_or_retry - Lock the page, unless this would block and the
682  * ie with increased "page->count" so that the page won't
711  * set_page_private_2 - Set PG_private_2 on a page and take a ref
740 	char __user *end = uaddr + size - 1;  in fault_in_pages_writeable()
746 		return -EFAULT;  in fault_in_pages_writeable()
753 			return -EFAULT;  in fault_in_pages_writeable()
768 	const char __user *end = uaddr + size - 1;  in fault_in_pages_readable()
774 		return -EFAULT;  in fault_in_pages_readable()
778 			return -EFAULT;  in fault_in_pages_readable()
792 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
794 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
799 void delete_from_page_cache_batch(struct address_space *mapping,
809 		struct address_space *mapping, pgoff_t offset, gfp_t gfp_mask)  in add_to_page_cache()  argument
814 	error = add_to_page_cache_locked(page, mapping, offset, gfp_mask);  in add_to_page_cache()
821  * struct readahead_control - Describes a readahead request.
824  * implement the ->readahead method should call readahead_page() or
833  * @mapping: Readahead this filesystem object.
838 	struct address_space *mapping;  member
849 		.mapping = m,						\
865  * page_cache_sync_readahead - generic file readahead
866  * @mapping: address_space which holds the pagecache and I/O vectors
878 void page_cache_sync_readahead(struct address_space *mapping,  in page_cache_sync_readahead()  argument
882 	DEFINE_READAHEAD(ractl, file, ra, mapping, index);  in page_cache_sync_readahead()
887  * page_cache_async_readahead - file readahead for marked pages
888  * @mapping: address_space which holds the pagecache and I/O vectors
901 void page_cache_async_readahead(struct address_space *mapping,  in page_cache_async_readahead()  argument
905 	DEFINE_READAHEAD(ractl, file, ra, mapping, index);  in page_cache_async_readahead()
910  * readahead_page - Get the next page to read.
922 	BUG_ON(rac->_batch_count > rac->_nr_pages);  in readahead_page()
923 	rac->_nr_pages -= rac->_batch_count;  in readahead_page()
924 	rac->_index += rac->_batch_count;  in readahead_page()
926 	if (!rac->_nr_pages) {  in readahead_page()
927 		rac->_batch_count = 0;  in readahead_page()
931 	page = xa_load(&rac->mapping->i_pages, rac->_index);  in readahead_page()
933 	rac->_batch_count = thp_nr_pages(page);  in readahead_page()
942 	XA_STATE(xas, &rac->mapping->i_pages, 0);  in __readahead_batch()
945 	BUG_ON(rac->_batch_count > rac->_nr_pages);  in __readahead_batch()
946 	rac->_nr_pages -= rac->_batch_count;  in __readahead_batch()
947 	rac->_index += rac->_batch_count;  in __readahead_batch()
948 	rac->_batch_count = 0;  in __readahead_batch()
950 	xas_set(&xas, rac->_index);  in __readahead_batch()
952 	xas_for_each(&xas, page, rac->_index + rac->_nr_pages - 1) {  in __readahead_batch()
958 		rac->_batch_count += thp_nr_pages(page);  in __readahead_batch()
961 		 * The page cache isn't using multi-index entries yet,  in __readahead_batch()
967 			xas_set(&xas, rac->_index + rac->_batch_count);  in __readahead_batch()
978  * readahead_page_batch - Get a batch of pages to read.
992  * readahead_pos - The byte offset into the file of this readahead request.
997 	return (loff_t)rac->_index * PAGE_SIZE;  in readahead_pos()
1001  * readahead_length - The number of bytes in this readahead request.
1006 	return rac->_nr_pages * PAGE_SIZE;  in readahead_length()
1010  * readahead_index - The index of the first page in this readahead request.
1015 	return rac->_index;  in readahead_index()
1019  * readahead_count - The number of pages in this readahead request.
1024 	return rac->_nr_pages;  in readahead_count()
1028  * readahead_batch_length - The number of bytes in the current batch.
1033 	return rac->_batch_count * PAGE_SIZE;  in readahead_batch_length()
1038 	return (unsigned long)(inode->i_size + PAGE_SIZE - 1) >>  in dir_pages()
1043  * page_mkwrite_check_truncate - check if page was truncated
1048  * or -EFAULT if the page was truncated.
1057 	if (page->mapping != inode->i_mapping)  in page_mkwrite_check_truncate()
1058 		return -EFAULT;  in page_mkwrite_check_truncate()
1061 	if (page->index < index)  in page_mkwrite_check_truncate()
1064 	if (page->index > index || !offset)  in page_mkwrite_check_truncate()
1065 		return -EFAULT;  in page_mkwrite_check_truncate()
1071  * i_blocks_per_page - How many blocks fit in this page.
1084 	return thp_size(page) >> inode->i_blkbits;  in i_blocks_per_page()