1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
17  * folio is present but not up-to-date, readahead will not try to read
18  * it. In that case a simple ->read_folio() will be requested.
29  * contains ->size being the total number of pages, and ->async_size
37  * to be determined: the start of the region to read, the size of the
40  * The start of the region is simply the first page address at or after
46  * this would be less than zero - then zero is used.  NOTE THIS
47  * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED
57  * page to the start of the new readahead.  In these cases, the size of
65  * larger than the current request, and it is not scaled up, unless it
66  * is at the start of file.
68  * In general readahead is accelerated at the start of the file, as
77  * Readahead requests are sent to the filesystem using the ->readahead()
79  * implementation.  ->readahead() should normally initiate reads on all
81  * error.  The page cache reading code will issue a ->read_folio() request
82  * for any folio which ->readahead() did not read, and only an error
85  * ->readahead() will generally call readahead_folio() repeatedly to get
103  * considered to be important and ->readahead() should not fail them due
113  * ->read_folio() which may be less efficient.
121 #include <linux/backing-dev.h>
129 #include <linux/blk-cgroup.h>
142 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
143 	ra->prev_pos = -1;  in file_ra_state_init()
149 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
156 	if (unlikely(rac->_workingset))  in read_pages()
157 		psi_memstall_enter(&rac->_pflags);  in read_pages()
160 	if (aops->readahead) {  in read_pages()
161 		aops->readahead(rac);  in read_pages()
163 		 * Clean up the remaining folios.  The sizes in ->ra  in read_pages()
171 			rac->ra->size -= nr;  in read_pages()
172 			if (rac->ra->async_size >= nr) {  in read_pages()
173 				rac->ra->async_size -= nr;  in read_pages()
181 			aops->read_folio(rac->file, folio);  in read_pages()
185 	if (unlikely(rac->_workingset))  in read_pages()
186 		psi_memstall_leave(&rac->_pflags);  in read_pages()
187 	rac->_workingset = false;  in read_pages()
193  * page_cache_ra_unbounded - Start unchecked readahead.
196  * @lookahead_size: Where to start the next readahead.
198  * This function is for filesystems to call when they want to start
209 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
220 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
231 		struct folio *folio = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()
243 			ractl->_index++;  in page_cache_ra_unbounded()
244 			i = ractl->_index + ractl->_nr_pages - index - 1;  in page_cache_ra_unbounded()
255 			ractl->_index++;  in page_cache_ra_unbounded()
256 			i = ractl->_index + ractl->_nr_pages - index - 1;  in page_cache_ra_unbounded()
259 		if (i == nr_to_read - lookahead_size)  in page_cache_ra_unbounded()
261 		ractl->_workingset |= folio_test_workingset(folio);  in page_cache_ra_unbounded()
262 		ractl->_nr_pages++;  in page_cache_ra_unbounded()
266 	 * Now start the IO.  We ignore I/O errors - if the folio is not  in page_cache_ra_unbounded()
285 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
293 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
297 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
298 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
310 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
311 	struct file_ra_state *ra = ractl->ra;  in force_page_cache_ra()
312 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
315 	if (unlikely(!mapping->a_ops->read_folio && !mapping->a_ops->readahead))  in force_page_cache_ra()
320 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
323 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
330 		ractl->_index = index;  in force_page_cache_ra()
334 		nr_to_read -= this_chunk;  in force_page_cache_ra()
342  * 1-2 page = 16k, 3-4 page 32k, 5-8 page = 64k, > 8 page = 128k initial
359  *  Get the previous window size, ramp it up, and
365 	unsigned long cur = ra->size;  in get_next_ra_size()
375  * On-demand readahead design.
377  * The fields in struct file_ra_state represent the most-recently-executed
380  *                        |<----- async_size ---------|
381  *     |------------------- size -------------------->|
383  *     ^start             ^page marked with PG_readahead
394  * page at (start+size-async_size) with PG_readahead, and use it as readahead
396  * readahead-for-nothing fuss, saving pointless page cache lookups.
404  * There is a special-case: if the first page which the application tries to
409  * The code ramps up the readahead size aggressively at first, but slow down as
414  * Count contiguously cached pages from @index-1 to @index-@max,
416  * 	- length of the sequential read sequence, or
417  * 	- thrashing threshold in memory tight systems
425 	head = page_cache_prev_miss(mapping, index - 1, max);  in count_history_pages()
428 	return index - 1 - head;  in count_history_pages()
453 	 * it is a strong indication of long-run stream (or whole-file-read)  in try_context_readahead()
458 	ra->start = index;  in try_context_readahead()
459 	ra->size = min(size + req_size, max);  in try_context_readahead()
460 	ra->async_size = 1;  in try_context_readahead()
485 		return -ENOMEM;  in ra_alloc_folio()
489 	err = filemap_add_folio(ractl->mapping, folio, index, gfp);  in ra_alloc_folio()
495 	ractl->_nr_pages += 1UL << order;  in ra_alloc_folio()
496 	ractl->_workingset |= folio_test_workingset(folio);  in ra_alloc_folio()
503 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_order()
505 	pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;  in page_cache_ra_order()
506 	pgoff_t mark = index + ra->size - ra->async_size;  in page_cache_ra_order()
510 	if (!mapping_large_folio_support(mapping) || ra->size < 4)  in page_cache_ra_order()
513 	limit = min(limit, index + ra->size - 1);  in page_cache_ra_order()
519 		while ((1 << new_order) > ra->size)  in page_cache_ra_order()
520 			new_order--;  in page_cache_ra_order()
528 		if (index & ((1UL << order) - 1)) {  in page_cache_ra_order()
534 		while (index + (1UL << order) - 1 > limit) {  in page_cache_ra_order()
535 			if (--order == 1)  in page_cache_ra_order()
545 		ra->size += index - limit - 1;  in page_cache_ra_order()
546 		ra->async_size += index - limit - 1;  in page_cache_ra_order()
560 	do_page_cache_ra(ractl, ra->size, ra->async_size);  in page_cache_ra_order()
569 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ondemand_readahead()
570 	struct file_ra_state *ra = ractl->ra;  in ondemand_readahead()
571 	unsigned long max_pages = ra->ra_pages;  in ondemand_readahead()
579 	 * be up to the optimal hardware IO size  in ondemand_readahead()
581 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ondemand_readahead()
582 		max_pages = min(req_size, bdi->io_pages);  in ondemand_readahead()
585 	 * start of file  in ondemand_readahead()
592 	 * Ramp up sizes, and push forward the readahead window.  in ondemand_readahead()
594 	expected = round_up(ra->start + ra->size - ra->async_size,  in ondemand_readahead()
596 	if (index == expected || index == (ra->start + ra->size)) {  in ondemand_readahead()
597 		ra->start += ra->size;  in ondemand_readahead()
598 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
599 		ra->async_size = ra->size;  in ondemand_readahead()
607 	 * readahead size. Ramp it up and use it as the new readahead size.  in ondemand_readahead()
610 		pgoff_t start;  in ondemand_readahead()  local
613 		start = page_cache_next_miss(ractl->mapping, index + 1,  in ondemand_readahead()
617 		if (!start || start - index > max_pages)  in ondemand_readahead()
620 		ra->start = start;  in ondemand_readahead()
621 		ra->size = start - index;	/* old async_size */  in ondemand_readahead()
622 		ra->size += req_size;  in ondemand_readahead()
623 		ra->size = get_next_ra_size(ra, max_pages);  in ondemand_readahead()
624 		ra->async_size = ra->size;  in ondemand_readahead()
636 	 * trivial case: (index - prev_index) == 1  in ondemand_readahead()
637 	 * unaligned reads: (index - prev_index) == 0  in ondemand_readahead()
639 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in ondemand_readahead()
640 	if (index - prev_index <= 1UL)  in ondemand_readahead()
647 	if (try_context_readahead(ractl->mapping, ra, index, req_size,  in ondemand_readahead()
659 	ra->start = index;  in ondemand_readahead()
660 	ra->size = get_init_ra_size(req_size, max_pages);  in ondemand_readahead()
661 	ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;  in ondemand_readahead()
670 	if (index == ra->start && ra->size == ra->async_size) {  in ondemand_readahead()
672 		if (ra->size + add_pages <= max_pages) {  in ondemand_readahead()
673 			ra->async_size = add_pages;  in ondemand_readahead()
674 			ra->size += add_pages;  in ondemand_readahead()
676 			ra->size = max_pages;  in ondemand_readahead()
677 			ra->async_size = max_pages >> 1;  in ondemand_readahead()
681 	ractl->_index = ra->start;  in ondemand_readahead()
688 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
696 	if (!ractl->ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
697 		if (!ractl->file)  in page_cache_sync_ra()
717 	if (!ractl->ra->ra_pages)  in page_cache_async_ra()
740 	ret = -EBADF;  in ksys_readahead()
742 	if (!f.file || !(f.file->f_mode & FMODE_READ))  in ksys_readahead()
748 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
750 	ret = -EINVAL;  in ksys_readahead()
751 	if (!f.file->f_mapping || !f.file->f_mapping->a_ops ||  in ksys_readahead()
752 	    !S_ISREG(file_inode(f.file)->i_mode))  in ksys_readahead()
774  * readahead_expand - Expand a readahead request
776  * @new_start: The revised start
794 	struct address_space *mapping = ractl->mapping;  in readahead_expand()
795 	struct file_ra_state *ra = ractl->ra;  in readahead_expand()
802 	while (ractl->_index > new_index) {  in readahead_expand()
803 		unsigned long index = ractl->_index - 1;  in readahead_expand()
804 		struct page *page = xa_load(&mapping->i_pages, index);  in readahead_expand()
817 		ractl->_nr_pages++;  in readahead_expand()
818 		ractl->_index = page->index;  in readahead_expand()
821 	new_len += new_start - readahead_pos(ractl);  in readahead_expand()
825 	while (ractl->_nr_pages < new_nr_pages) {  in readahead_expand()
826 		unsigned long index = ractl->_index + ractl->_nr_pages;  in readahead_expand()
827 		struct page *page = xa_load(&mapping->i_pages, index);  in readahead_expand()
839 		if (unlikely(PageWorkingset(page)) && !ractl->_workingset) {  in readahead_expand()
840 			ractl->_workingset = true;  in readahead_expand()
841 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
843 		ractl->_nr_pages++;  in readahead_expand()
845 			ra->size++;  in readahead_expand()
846 			ra->async_size++;  in readahead_expand()