1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *	linux/mm/filemap.c
4  *
5  * Copyright (C) 1994-1999  Linus Torvalds
6  */
7 
8 /*
9  * This file handles the generic file mmap semantics used by
10  * most "normal" filesystems (but you don't /have/ to use this:
11  * the NFS filesystem used to do this differently, for example)
12  */
13 #include <linux/export.h>
14 #include <linux/compiler.h>
15 #include <linux/dax.h>
16 #include <linux/fs.h>
17 #include <linux/sched/signal.h>
18 #include <linux/uaccess.h>
19 #include <linux/capability.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/gfp.h>
22 #include <linux/mm.h>
23 #include <linux/swap.h>
24 #include <linux/mman.h>
25 #include <linux/pagemap.h>
26 #include <linux/file.h>
27 #include <linux/uio.h>
28 #include <linux/error-injection.h>
29 #include <linux/hash.h>
30 #include <linux/writeback.h>
31 #include <linux/backing-dev.h>
32 #include <linux/pagevec.h>
33 #include <linux/blkdev.h>
34 #include <linux/security.h>
35 #include <linux/cpuset.h>
36 #include <linux/hugetlb.h>
37 #include <linux/memcontrol.h>
38 #include <linux/cleancache.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/rmap.h>
41 #include <linux/delayacct.h>
42 #include <linux/psi.h>
43 #include <linux/ramfs.h>
44 #include <linux/page_idle.h>
45 #include <asm/pgalloc.h>
46 #include <asm/tlbflush.h>
47 #include "internal.h"
48 
49 #define CREATE_TRACE_POINTS
50 #include <trace/events/filemap.h>
51 
52 /*
53  * FIXME: remove all knowledge of the buffer layer from the core VM
54  */
55 #include <linux/buffer_head.h> /* for try_to_free_buffers */
56 
57 #include <asm/mman.h>
58 
59 /*
60  * Shared mappings implemented 30.11.1994. It's not fully working yet,
61  * though.
62  *
63  * Shared mappings now work. 15.8.1995  Bruno.
64  *
65  * finished 'unifying' the page and buffer cache and SMP-threaded the
66  * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
67  *
68  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
69  */
70 
71 /*
72  * Lock ordering:
73  *
74  *  ->i_mmap_rwsem		(truncate_pagecache)
75  *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
76  *      ->swap_lock		(exclusive_swap_page, others)
77  *        ->i_pages lock
78  *
79  *  ->i_rwsem
80  *    ->invalidate_lock		(acquired by fs in truncate path)
81  *      ->i_mmap_rwsem		(truncate->unmap_mapping_range)
82  *
83  *  ->mmap_lock
84  *    ->i_mmap_rwsem
85  *      ->page_table_lock or pte_lock	(various, mainly in memory.c)
86  *        ->i_pages lock	(arch-dependent flush_dcache_mmap_lock)
87  *
88  *  ->mmap_lock
89  *    ->invalidate_lock		(filemap_fault)
90  *      ->lock_page		(filemap_fault, access_process_vm)
91  *
92  *  ->i_rwsem			(generic_perform_write)
93  *    ->mmap_lock		(fault_in_pages_readable->do_page_fault)
94  *
95  *  bdi->wb.list_lock
96  *    sb_lock			(fs/fs-writeback.c)
97  *    ->i_pages lock		(__sync_single_inode)
98  *
99  *  ->i_mmap_rwsem
100  *    ->anon_vma.lock		(vma_adjust)
101  *
102  *  ->anon_vma.lock
103  *    ->page_table_lock or pte_lock	(anon_vma_prepare and various)
104  *
105  *  ->page_table_lock or pte_lock
106  *    ->swap_lock		(try_to_unmap_one)
107  *    ->private_lock		(try_to_unmap_one)
108  *    ->i_pages lock		(try_to_unmap_one)
109  *    ->lruvec->lru_lock	(follow_page->mark_page_accessed)
110  *    ->lruvec->lru_lock	(check_pte_range->isolate_lru_page)
111  *    ->private_lock		(page_remove_rmap->set_page_dirty)
112  *    ->i_pages lock		(page_remove_rmap->set_page_dirty)
113  *    bdi.wb->list_lock		(page_remove_rmap->set_page_dirty)
114  *    ->inode->i_lock		(page_remove_rmap->set_page_dirty)
115  *    ->memcg->move_lock	(page_remove_rmap->lock_page_memcg)
116  *    bdi.wb->list_lock		(zap_pte_range->set_page_dirty)
117  *    ->inode->i_lock		(zap_pte_range->set_page_dirty)
118  *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers)
119  *
120  * ->i_mmap_rwsem
121  *   ->tasklist_lock            (memory_failure, collect_procs_ao)
122  */
123 
page_cache_delete(struct address_space * mapping,struct page * page,void * shadow)124 static void page_cache_delete(struct address_space *mapping,
125 				   struct page *page, void *shadow)
126 {
127 	XA_STATE(xas, &mapping->i_pages, page->index);
128 	unsigned int nr = 1;
129 
130 	mapping_set_update(&xas, mapping);
131 
132 	/* hugetlb pages are represented by a single entry in the xarray */
133 	if (!PageHuge(page)) {
134 		xas_set_order(&xas, page->index, compound_order(page));
135 		nr = compound_nr(page);
136 	}
137 
138 	VM_BUG_ON_PAGE(!PageLocked(page), page);
139 	VM_BUG_ON_PAGE(PageTail(page), page);
140 	VM_BUG_ON_PAGE(nr != 1 && shadow, page);
141 
142 	xas_store(&xas, shadow);
143 	xas_init_marks(&xas);
144 
145 	page->mapping = NULL;
146 	/* Leave page->index set: truncation lookup relies upon it */
147 	mapping->nrpages -= nr;
148 }
149 
unaccount_page_cache_page(struct address_space * mapping,struct page * page)150 static void unaccount_page_cache_page(struct address_space *mapping,
151 				      struct page *page)
152 {
153 	int nr;
154 
155 	/*
156 	 * if we're uptodate, flush out into the cleancache, otherwise
157 	 * invalidate any existing cleancache entries.  We can't leave
158 	 * stale data around in the cleancache once our page is gone
159 	 */
160 	if (PageUptodate(page) && PageMappedToDisk(page))
161 		cleancache_put_page(page);
162 	else
163 		cleancache_invalidate_page(mapping, page);
164 
165 	VM_BUG_ON_PAGE(PageTail(page), page);
166 	VM_BUG_ON_PAGE(page_mapped(page), page);
167 	if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
168 		int mapcount;
169 
170 		pr_alert("BUG: Bad page cache in process %s  pfn:%05lx\n",
171 			 current->comm, page_to_pfn(page));
172 		dump_page(page, "still mapped when deleted");
173 		dump_stack();
174 		add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
175 
176 		mapcount = page_mapcount(page);
177 		if (mapping_exiting(mapping) &&
178 		    page_count(page) >= mapcount + 2) {
179 			/*
180 			 * All vmas have already been torn down, so it's
181 			 * a good bet that actually the page is unmapped,
182 			 * and we'd prefer not to leak it: if we're wrong,
183 			 * some other bad page check should catch it later.
184 			 */
185 			page_mapcount_reset(page);
186 			page_ref_sub(page, mapcount);
187 		}
188 	}
189 
190 	/* hugetlb pages do not participate in page cache accounting. */
191 	if (PageHuge(page))
192 		return;
193 
194 	nr = thp_nr_pages(page);
195 
196 	__mod_lruvec_page_state(page, NR_FILE_PAGES, -nr);
197 	if (PageSwapBacked(page)) {
198 		__mod_lruvec_page_state(page, NR_SHMEM, -nr);
199 		if (PageTransHuge(page))
200 			__mod_lruvec_page_state(page, NR_SHMEM_THPS, -nr);
201 	} else if (PageTransHuge(page)) {
202 		__mod_lruvec_page_state(page, NR_FILE_THPS, -nr);
203 		filemap_nr_thps_dec(mapping);
204 	}
205 
206 	/*
207 	 * At this point page must be either written or cleaned by
208 	 * truncate.  Dirty page here signals a bug and loss of
209 	 * unwritten data.
210 	 *
211 	 * This fixes dirty accounting after removing the page entirely
212 	 * but leaves PageDirty set: it has no effect for truncated
213 	 * page and anyway will be cleared before returning page into
214 	 * buddy allocator.
215 	 */
216 	if (WARN_ON_ONCE(PageDirty(page)))
217 		account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
218 }
219 
220 /*
221  * Delete a page from the page cache and free it. Caller has to make
222  * sure the page is locked and that nobody else uses it - or that usage
223  * is safe.  The caller must hold the i_pages lock.
224  */
__delete_from_page_cache(struct page * page,void * shadow)225 void __delete_from_page_cache(struct page *page, void *shadow)
226 {
227 	struct address_space *mapping = page->mapping;
228 
229 	trace_mm_filemap_delete_from_page_cache(page);
230 
231 	unaccount_page_cache_page(mapping, page);
232 	page_cache_delete(mapping, page, shadow);
233 }
234 
page_cache_free_page(struct address_space * mapping,struct page * page)235 static void page_cache_free_page(struct address_space *mapping,
236 				struct page *page)
237 {
238 	void (*freepage)(struct page *);
239 
240 	freepage = mapping->a_ops->freepage;
241 	if (freepage)
242 		freepage(page);
243 
244 	if (PageTransHuge(page) && !PageHuge(page)) {
245 		page_ref_sub(page, thp_nr_pages(page));
246 		VM_BUG_ON_PAGE(page_count(page) <= 0, page);
247 	} else {
248 		put_page(page);
249 	}
250 }
251 
252 /**
253  * delete_from_page_cache - delete page from page cache
254  * @page: the page which the kernel is trying to remove from page cache
255  *
256  * This must be called only on pages that have been verified to be in the page
257  * cache and locked.  It will never put the page into the free list, the caller
258  * has a reference on the page.
259  */
delete_from_page_cache(struct page * page)260 void delete_from_page_cache(struct page *page)
261 {
262 	struct address_space *mapping = page_mapping(page);
263 
264 	BUG_ON(!PageLocked(page));
265 	xa_lock_irq(&mapping->i_pages);
266 	__delete_from_page_cache(page, NULL);
267 	xa_unlock_irq(&mapping->i_pages);
268 
269 	page_cache_free_page(mapping, page);
270 }
271 EXPORT_SYMBOL(delete_from_page_cache);
272 
273 /*
274  * page_cache_delete_batch - delete several pages from page cache
275  * @mapping: the mapping to which pages belong
276  * @pvec: pagevec with pages to delete
277  *
278  * The function walks over mapping->i_pages and removes pages passed in @pvec
279  * from the mapping. The function expects @pvec to be sorted by page index
280  * and is optimised for it to be dense.
281  * It tolerates holes in @pvec (mapping entries at those indices are not
282  * modified). The function expects only THP head pages to be present in the
283  * @pvec.
284  *
285  * The function expects the i_pages lock to be held.
286  */
page_cache_delete_batch(struct address_space * mapping,struct pagevec * pvec)287 static void page_cache_delete_batch(struct address_space *mapping,
288 			     struct pagevec *pvec)
289 {
290 	XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index);
291 	int total_pages = 0;
292 	int i = 0;
293 	struct page *page;
294 
295 	mapping_set_update(&xas, mapping);
296 	xas_for_each(&xas, page, ULONG_MAX) {
297 		if (i >= pagevec_count(pvec))
298 			break;
299 
300 		/* A swap/dax/shadow entry got inserted? Skip it. */
301 		if (xa_is_value(page))
302 			continue;
303 		/*
304 		 * A page got inserted in our range? Skip it. We have our
305 		 * pages locked so they are protected from being removed.
306 		 * If we see a page whose index is higher than ours, it
307 		 * means our page has been removed, which shouldn't be
308 		 * possible because we're holding the PageLock.
309 		 */
310 		if (page != pvec->pages[i]) {
311 			VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index,
312 					page);
313 			continue;
314 		}
315 
316 		WARN_ON_ONCE(!PageLocked(page));
317 
318 		if (page->index == xas.xa_index)
319 			page->mapping = NULL;
320 		/* Leave page->index set: truncation lookup relies on it */
321 
322 		/*
323 		 * Move to the next page in the vector if this is a regular
324 		 * page or the index is of the last sub-page of this compound
325 		 * page.
326 		 */
327 		if (page->index + compound_nr(page) - 1 == xas.xa_index)
328 			i++;
329 		xas_store(&xas, NULL);
330 		total_pages++;
331 	}
332 	mapping->nrpages -= total_pages;
333 }
334 
delete_from_page_cache_batch(struct address_space * mapping,struct pagevec * pvec)335 void delete_from_page_cache_batch(struct address_space *mapping,
336 				  struct pagevec *pvec)
337 {
338 	int i;
339 
340 	if (!pagevec_count(pvec))
341 		return;
342 
343 	xa_lock_irq(&mapping->i_pages);
344 	for (i = 0; i < pagevec_count(pvec); i++) {
345 		trace_mm_filemap_delete_from_page_cache(pvec->pages[i]);
346 
347 		unaccount_page_cache_page(mapping, pvec->pages[i]);
348 	}
349 	page_cache_delete_batch(mapping, pvec);
350 	xa_unlock_irq(&mapping->i_pages);
351 
352 	for (i = 0; i < pagevec_count(pvec); i++)
353 		page_cache_free_page(mapping, pvec->pages[i]);
354 }
355 
filemap_check_errors(struct address_space * mapping)356 int filemap_check_errors(struct address_space *mapping)
357 {
358 	int ret = 0;
359 	/* Check for outstanding write errors */
360 	if (test_bit(AS_ENOSPC, &mapping->flags) &&
361 	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))
362 		ret = -ENOSPC;
363 	if (test_bit(AS_EIO, &mapping->flags) &&
364 	    test_and_clear_bit(AS_EIO, &mapping->flags))
365 		ret = -EIO;
366 	return ret;
367 }
368 EXPORT_SYMBOL(filemap_check_errors);
369 
filemap_check_and_keep_errors(struct address_space * mapping)370 static int filemap_check_and_keep_errors(struct address_space *mapping)
371 {
372 	/* Check for outstanding write errors */
373 	if (test_bit(AS_EIO, &mapping->flags))
374 		return -EIO;
375 	if (test_bit(AS_ENOSPC, &mapping->flags))
376 		return -ENOSPC;
377 	return 0;
378 }
379 
380 /**
381  * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
382  * @mapping:	address space structure to write
383  * @wbc:	the writeback_control controlling the writeout
384  *
385  * Call writepages on the mapping using the provided wbc to control the
386  * writeout.
387  *
388  * Return: %0 on success, negative error code otherwise.
389  */
filemap_fdatawrite_wbc(struct address_space * mapping,struct writeback_control * wbc)390 int filemap_fdatawrite_wbc(struct address_space *mapping,
391 			   struct writeback_control *wbc)
392 {
393 	int ret;
394 
395 	if (!mapping_can_writeback(mapping) ||
396 	    !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
397 		return 0;
398 
399 	wbc_attach_fdatawrite_inode(wbc, mapping->host);
400 	ret = do_writepages(mapping, wbc);
401 	wbc_detach_inode(wbc);
402 	return ret;
403 }
404 EXPORT_SYMBOL(filemap_fdatawrite_wbc);
405 
406 /**
407  * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
408  * @mapping:	address space structure to write
409  * @start:	offset in bytes where the range starts
410  * @end:	offset in bytes where the range ends (inclusive)
411  * @sync_mode:	enable synchronous operation
412  *
413  * Start writeback against all of a mapping's dirty pages that lie
414  * within the byte offsets <start, end> inclusive.
415  *
416  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
417  * opposed to a regular memory cleansing writeback.  The difference between
418  * these two operations is that if a dirty page/buffer is encountered, it must
419  * be waited upon, and not just skipped over.
420  *
421  * Return: %0 on success, negative error code otherwise.
422  */
__filemap_fdatawrite_range(struct address_space * mapping,loff_t start,loff_t end,int sync_mode)423 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
424 				loff_t end, int sync_mode)
425 {
426 	struct writeback_control wbc = {
427 		.sync_mode = sync_mode,
428 		.nr_to_write = LONG_MAX,
429 		.range_start = start,
430 		.range_end = end,
431 	};
432 
433 	return filemap_fdatawrite_wbc(mapping, &wbc);
434 }
435 
__filemap_fdatawrite(struct address_space * mapping,int sync_mode)436 static inline int __filemap_fdatawrite(struct address_space *mapping,
437 	int sync_mode)
438 {
439 	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
440 }
441 
filemap_fdatawrite(struct address_space * mapping)442 int filemap_fdatawrite(struct address_space *mapping)
443 {
444 	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
445 }
446 EXPORT_SYMBOL(filemap_fdatawrite);
447 
filemap_fdatawrite_range(struct address_space * mapping,loff_t start,loff_t end)448 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
449 				loff_t end)
450 {
451 	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
452 }
453 EXPORT_SYMBOL(filemap_fdatawrite_range);
454 
455 /**
456  * filemap_flush - mostly a non-blocking flush
457  * @mapping:	target address_space
458  *
459  * This is a mostly non-blocking flush.  Not suitable for data-integrity
460  * purposes - I/O may not be started against all dirty pages.
461  *
462  * Return: %0 on success, negative error code otherwise.
463  */
filemap_flush(struct address_space * mapping)464 int filemap_flush(struct address_space *mapping)
465 {
466 	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
467 }
468 EXPORT_SYMBOL(filemap_flush);
469 
470 /**
471  * filemap_range_has_page - check if a page exists in range.
472  * @mapping:           address space within which to check
473  * @start_byte:        offset in bytes where the range starts
474  * @end_byte:          offset in bytes where the range ends (inclusive)
475  *
476  * Find at least one page in the range supplied, usually used to check if
477  * direct writing in this range will trigger a writeback.
478  *
479  * Return: %true if at least one page exists in the specified range,
480  * %false otherwise.
481  */
filemap_range_has_page(struct address_space * mapping,loff_t start_byte,loff_t end_byte)482 bool filemap_range_has_page(struct address_space *mapping,
483 			   loff_t start_byte, loff_t end_byte)
484 {
485 	struct page *page;
486 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
487 	pgoff_t max = end_byte >> PAGE_SHIFT;
488 
489 	if (end_byte < start_byte)
490 		return false;
491 
492 	rcu_read_lock();
493 	for (;;) {
494 		page = xas_find(&xas, max);
495 		if (xas_retry(&xas, page))
496 			continue;
497 		/* Shadow entries don't count */
498 		if (xa_is_value(page))
499 			continue;
500 		/*
501 		 * We don't need to try to pin this page; we're about to
502 		 * release the RCU lock anyway.  It is enough to know that
503 		 * there was a page here recently.
504 		 */
505 		break;
506 	}
507 	rcu_read_unlock();
508 
509 	return page != NULL;
510 }
511 EXPORT_SYMBOL(filemap_range_has_page);
512 
__filemap_fdatawait_range(struct address_space * mapping,loff_t start_byte,loff_t end_byte)513 static void __filemap_fdatawait_range(struct address_space *mapping,
514 				     loff_t start_byte, loff_t end_byte)
515 {
516 	pgoff_t index = start_byte >> PAGE_SHIFT;
517 	pgoff_t end = end_byte >> PAGE_SHIFT;
518 	struct pagevec pvec;
519 	int nr_pages;
520 
521 	if (end_byte < start_byte)
522 		return;
523 
524 	pagevec_init(&pvec);
525 	while (index <= end) {
526 		unsigned i;
527 
528 		nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
529 				end, PAGECACHE_TAG_WRITEBACK);
530 		if (!nr_pages)
531 			break;
532 
533 		for (i = 0; i < nr_pages; i++) {
534 			struct page *page = pvec.pages[i];
535 
536 			wait_on_page_writeback(page);
537 			ClearPageError(page);
538 		}
539 		pagevec_release(&pvec);
540 		cond_resched();
541 	}
542 }
543 
544 /**
545  * filemap_fdatawait_range - wait for writeback to complete
546  * @mapping:		address space structure to wait for
547  * @start_byte:		offset in bytes where the range starts
548  * @end_byte:		offset in bytes where the range ends (inclusive)
549  *
550  * Walk the list of under-writeback pages of the given address space
551  * in the given range and wait for all of them.  Check error status of
552  * the address space and return it.
553  *
554  * Since the error status of the address space is cleared by this function,
555  * callers are responsible for checking the return value and handling and/or
556  * reporting the error.
557  *
558  * Return: error status of the address space.
559  */
filemap_fdatawait_range(struct address_space * mapping,loff_t start_byte,loff_t end_byte)560 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
561 			    loff_t end_byte)
562 {
563 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
564 	return filemap_check_errors(mapping);
565 }
566 EXPORT_SYMBOL(filemap_fdatawait_range);
567 
568 /**
569  * filemap_fdatawait_range_keep_errors - wait for writeback to complete
570  * @mapping:		address space structure to wait for
571  * @start_byte:		offset in bytes where the range starts
572  * @end_byte:		offset in bytes where the range ends (inclusive)
573  *
574  * Walk the list of under-writeback pages of the given address space in the
575  * given range and wait for all of them.  Unlike filemap_fdatawait_range(),
576  * this function does not clear error status of the address space.
577  *
578  * Use this function if callers don't handle errors themselves.  Expected
579  * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
580  * fsfreeze(8)
581  */
filemap_fdatawait_range_keep_errors(struct address_space * mapping,loff_t start_byte,loff_t end_byte)582 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
583 		loff_t start_byte, loff_t end_byte)
584 {
585 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
586 	return filemap_check_and_keep_errors(mapping);
587 }
588 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors);
589 
590 /**
591  * file_fdatawait_range - wait for writeback to complete
592  * @file:		file pointing to address space structure to wait for
593  * @start_byte:		offset in bytes where the range starts
594  * @end_byte:		offset in bytes where the range ends (inclusive)
595  *
596  * Walk the list of under-writeback pages of the address space that file
597  * refers to, in the given range and wait for all of them.  Check error
598  * status of the address space vs. the file->f_wb_err cursor and return it.
599  *
600  * Since the error status of the file is advanced by this function,
601  * callers are responsible for checking the return value and handling and/or
602  * reporting the error.
603  *
604  * Return: error status of the address space vs. the file->f_wb_err cursor.
605  */
file_fdatawait_range(struct file * file,loff_t start_byte,loff_t end_byte)606 int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
607 {
608 	struct address_space *mapping = file->f_mapping;
609 
610 	__filemap_fdatawait_range(mapping, start_byte, end_byte);
611 	return file_check_and_advance_wb_err(file);
612 }
613 EXPORT_SYMBOL(file_fdatawait_range);
614 
615 /**
616  * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
617  * @mapping: address space structure to wait for
618  *
619  * Walk the list of under-writeback pages of the given address space
620  * and wait for all of them.  Unlike filemap_fdatawait(), this function
621  * does not clear error status of the address space.
622  *
623  * Use this function if callers don't handle errors themselves.  Expected
624  * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
625  * fsfreeze(8)
626  *
627  * Return: error status of the address space.
628  */
filemap_fdatawait_keep_errors(struct address_space * mapping)629 int filemap_fdatawait_keep_errors(struct address_space *mapping)
630 {
631 	__filemap_fdatawait_range(mapping, 0, LLONG_MAX);
632 	return filemap_check_and_keep_errors(mapping);
633 }
634 EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
635 
636 /* Returns true if writeback might be needed or already in progress. */
mapping_needs_writeback(struct address_space * mapping)637 static bool mapping_needs_writeback(struct address_space *mapping)
638 {
639 	return mapping->nrpages;
640 }
641 
642 /**
643  * filemap_range_needs_writeback - check if range potentially needs writeback
644  * @mapping:           address space within which to check
645  * @start_byte:        offset in bytes where the range starts
646  * @end_byte:          offset in bytes where the range ends (inclusive)
647  *
648  * Find at least one page in the range supplied, usually used to check if
649  * direct writing in this range will trigger a writeback. Used by O_DIRECT
650  * read/write with IOCB_NOWAIT, to see if the caller needs to do
651  * filemap_write_and_wait_range() before proceeding.
652  *
653  * Return: %true if the caller should do filemap_write_and_wait_range() before
654  * doing O_DIRECT to a page in this range, %false otherwise.
655  */
filemap_range_needs_writeback(struct address_space * mapping,loff_t start_byte,loff_t end_byte)656 bool filemap_range_needs_writeback(struct address_space *mapping,
657 				   loff_t start_byte, loff_t end_byte)
658 {
659 	XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
660 	pgoff_t max = end_byte >> PAGE_SHIFT;
661 	struct page *page;
662 
663 	if (!mapping_needs_writeback(mapping))
664 		return false;
665 	if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
666 	    !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
667 		return false;
668 	if (end_byte < start_byte)
669 		return false;
670 
671 	rcu_read_lock();
672 	xas_for_each(&xas, page, max) {
673 		if (xas_retry(&xas, page))
674 			continue;
675 		if (xa_is_value(page))
676 			continue;
677 		if (PageDirty(page) || PageLocked(page) || PageWriteback(page))
678 			break;
679 	}
680 	rcu_read_unlock();
681 	return page != NULL;
682 }
683 EXPORT_SYMBOL_GPL(filemap_range_needs_writeback);
684 
685 /**
686  * filemap_write_and_wait_range - write out & wait on a file range
687  * @mapping:	the address_space for the pages
688  * @lstart:	offset in bytes where the range starts
689  * @lend:	offset in bytes where the range ends (inclusive)
690  *
691  * Write out and wait upon file offsets lstart->lend, inclusive.
692  *
693  * Note that @lend is inclusive (describes the last byte to be written) so
694  * that this function can be used to write to the very end-of-file (end = -1).
695  *
696  * Return: error status of the address space.
697  */
filemap_write_and_wait_range(struct address_space * mapping,loff_t lstart,loff_t lend)698 int filemap_write_and_wait_range(struct address_space *mapping,
699 				 loff_t lstart, loff_t lend)
700 {
701 	int err = 0;
702 
703 	if (mapping_needs_writeback(mapping)) {
704 		err = __filemap_fdatawrite_range(mapping, lstart, lend,
705 						 WB_SYNC_ALL);
706 		/*
707 		 * Even if the above returned error, the pages may be
708 		 * written partially (e.g. -ENOSPC), so we wait for it.
709 		 * But the -EIO is special case, it may indicate the worst
710 		 * thing (e.g. bug) happened, so we avoid waiting for it.
711 		 */
712 		if (err != -EIO) {
713 			int err2 = filemap_fdatawait_range(mapping,
714 						lstart, lend);
715 			if (!err)
716 				err = err2;
717 		} else {
718 			/* Clear any previously stored errors */
719 			filemap_check_errors(mapping);
720 		}
721 	} else {
722 		err = filemap_check_errors(mapping);
723 	}
724 	return err;
725 }
726 EXPORT_SYMBOL(filemap_write_and_wait_range);
727 
__filemap_set_wb_err(struct address_space * mapping,int err)728 void __filemap_set_wb_err(struct address_space *mapping, int err)
729 {
730 	errseq_t eseq = errseq_set(&mapping->wb_err, err);
731 
732 	trace_filemap_set_wb_err(mapping, eseq);
733 }
734 EXPORT_SYMBOL(__filemap_set_wb_err);
735 
736 /**
737  * file_check_and_advance_wb_err - report wb error (if any) that was previously
738  * 				   and advance wb_err to current one
739  * @file: struct file on which the error is being reported
740  *
741  * When userland calls fsync (or something like nfsd does the equivalent), we
742  * want to report any writeback errors that occurred since the last fsync (or
743  * since the file was opened if there haven't been any).
744  *
745  * Grab the wb_err from the mapping. If it matches what we have in the file,
746  * then just quickly return 0. The file is all caught up.
747  *
748  * If it doesn't match, then take the mapping value, set the "seen" flag in
749  * it and try to swap it into place. If it works, or another task beat us
750  * to it with the new value, then update the f_wb_err and return the error
751  * portion. The error at this point must be reported via proper channels
752  * (a'la fsync, or NFS COMMIT operation, etc.).
753  *
754  * While we handle mapping->wb_err with atomic operations, the f_wb_err
755  * value is protected by the f_lock since we must ensure that it reflects
756  * the latest value swapped in for this file descriptor.
757  *
758  * Return: %0 on success, negative error code otherwise.
759  */
file_check_and_advance_wb_err(struct file * file)760 int file_check_and_advance_wb_err(struct file *file)
761 {
762 	int err = 0;
763 	errseq_t old = READ_ONCE(file->f_wb_err);
764 	struct address_space *mapping = file->f_mapping;
765 
766 	/* Locklessly handle the common case where nothing has changed */
767 	if (errseq_check(&mapping->wb_err, old)) {
768 		/* Something changed, must use slow path */
769 		spin_lock(&file->f_lock);
770 		old = file->f_wb_err;
771 		err = errseq_check_and_advance(&mapping->wb_err,
772 						&file->f_wb_err);
773 		trace_file_check_and_advance_wb_err(file, old);
774 		spin_unlock(&file->f_lock);
775 	}
776 
777 	/*
778 	 * We're mostly using this function as a drop in replacement for
779 	 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
780 	 * that the legacy code would have had on these flags.
781 	 */
782 	clear_bit(AS_EIO, &mapping->flags);
783 	clear_bit(AS_ENOSPC, &mapping->flags);
784 	return err;
785 }
786 EXPORT_SYMBOL(file_check_and_advance_wb_err);
787 
788 /**
789  * file_write_and_wait_range - write out & wait on a file range
790  * @file:	file pointing to address_space with pages
791  * @lstart:	offset in bytes where the range starts
792  * @lend:	offset in bytes where the range ends (inclusive)
793  *
794  * Write out and wait upon file offsets lstart->lend, inclusive.
795  *
796  * Note that @lend is inclusive (describes the last byte to be written) so
797  * that this function can be used to write to the very end-of-file (end = -1).
798  *
799  * After writing out and waiting on the data, we check and advance the
800  * f_wb_err cursor to the latest value, and return any errors detected there.
801  *
802  * Return: %0 on success, negative error code otherwise.
803  */
file_write_and_wait_range(struct file * file,loff_t lstart,loff_t lend)804 int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
805 {
806 	int err = 0, err2;
807 	struct address_space *mapping = file->f_mapping;
808 
809 	if (mapping_needs_writeback(mapping)) {
810 		err = __filemap_fdatawrite_range(mapping, lstart, lend,
811 						 WB_SYNC_ALL);
812 		/* See comment of filemap_write_and_wait() */
813 		if (err != -EIO)
814 			__filemap_fdatawait_range(mapping, lstart, lend);
815 	}
816 	err2 = file_check_and_advance_wb_err(file);
817 	if (!err)
818 		err = err2;
819 	return err;
820 }
821 EXPORT_SYMBOL(file_write_and_wait_range);
822 
823 /**
824  * replace_page_cache_page - replace a pagecache page with a new one
825  * @old:	page to be replaced
826  * @new:	page to replace with
827  *
828  * This function replaces a page in the pagecache with a new one.  On
829  * success it acquires the pagecache reference for the new page and
830  * drops it for the old page.  Both the old and new pages must be
831  * locked.  This function does not add the new page to the LRU, the
832  * caller must do that.
833  *
834  * The remove + add is atomic.  This function cannot fail.
835  */
replace_page_cache_page(struct page * old,struct page * new)836 void replace_page_cache_page(struct page *old, struct page *new)
837 {
838 	struct address_space *mapping = old->mapping;
839 	void (*freepage)(struct page *) = mapping->a_ops->freepage;
840 	pgoff_t offset = old->index;
841 	XA_STATE(xas, &mapping->i_pages, offset);
842 
843 	VM_BUG_ON_PAGE(!PageLocked(old), old);
844 	VM_BUG_ON_PAGE(!PageLocked(new), new);
845 	VM_BUG_ON_PAGE(new->mapping, new);
846 
847 	get_page(new);
848 	new->mapping = mapping;
849 	new->index = offset;
850 
851 	mem_cgroup_migrate(old, new);
852 
853 	xas_lock_irq(&xas);
854 	xas_store(&xas, new);
855 
856 	old->mapping = NULL;
857 	/* hugetlb pages do not participate in page cache accounting. */
858 	if (!PageHuge(old))
859 		__dec_lruvec_page_state(old, NR_FILE_PAGES);
860 	if (!PageHuge(new))
861 		__inc_lruvec_page_state(new, NR_FILE_PAGES);
862 	if (PageSwapBacked(old))
863 		__dec_lruvec_page_state(old, NR_SHMEM);
864 	if (PageSwapBacked(new))
865 		__inc_lruvec_page_state(new, NR_SHMEM);
866 	xas_unlock_irq(&xas);
867 	if (freepage)
868 		freepage(old);
869 	put_page(old);
870 }
871 EXPORT_SYMBOL_GPL(replace_page_cache_page);
872 
__add_to_page_cache_locked(struct page * page,struct address_space * mapping,pgoff_t offset,gfp_t gfp,void ** shadowp)873 noinline int __add_to_page_cache_locked(struct page *page,
874 					struct address_space *mapping,
875 					pgoff_t offset, gfp_t gfp,
876 					void **shadowp)
877 {
878 	XA_STATE(xas, &mapping->i_pages, offset);
879 	int huge = PageHuge(page);
880 	int error;
881 	bool charged = false;
882 
883 	VM_BUG_ON_PAGE(!PageLocked(page), page);
884 	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
885 	mapping_set_update(&xas, mapping);
886 
887 	get_page(page);
888 	page->mapping = mapping;
889 	page->index = offset;
890 
891 	if (!huge) {
892 		error = mem_cgroup_charge(page, NULL, gfp);
893 		if (error)
894 			goto error;
895 		charged = true;
896 	}
897 
898 	gfp &= GFP_RECLAIM_MASK;
899 
900 	do {
901 		unsigned int order = xa_get_order(xas.xa, xas.xa_index);
902 		void *entry, *old = NULL;
903 
904 		if (order > thp_order(page))
905 			xas_split_alloc(&xas, xa_load(xas.xa, xas.xa_index),
906 					order, gfp);
907 		xas_lock_irq(&xas);
908 		xas_for_each_conflict(&xas, entry) {
909 			old = entry;
910 			if (!xa_is_value(entry)) {
911 				xas_set_err(&xas, -EEXIST);
912 				goto unlock;
913 			}
914 		}
915 
916 		if (old) {
917 			if (shadowp)
918 				*shadowp = old;
919 			/* entry may have been split before we acquired lock */
920 			order = xa_get_order(xas.xa, xas.xa_index);
921 			if (order > thp_order(page)) {
922 				xas_split(&xas, old, order);
923 				xas_reset(&xas);
924 			}
925 		}
926 
927 		xas_store(&xas, page);
928 		if (xas_error(&xas))
929 			goto unlock;
930 
931 		mapping->nrpages++;
932 
933 		/* hugetlb pages do not participate in page cache accounting */
934 		if (!huge)
935 			__inc_lruvec_page_state(page, NR_FILE_PAGES);
936 unlock:
937 		xas_unlock_irq(&xas);
938 	} while (xas_nomem(&xas, gfp));
939 
940 	if (xas_error(&xas)) {
941 		error = xas_error(&xas);
942 		if (charged)
943 			mem_cgroup_uncharge(page);
944 		goto error;
945 	}
946 
947 	trace_mm_filemap_add_to_page_cache(page);
948 	return 0;
949 error:
950 	page->mapping = NULL;
951 	/* Leave page->index set: truncation relies upon it */
952 	put_page(page);
953 	return error;
954 }
955 ALLOW_ERROR_INJECTION(__add_to_page_cache_locked, ERRNO);
956 
957 /**
958  * add_to_page_cache_locked - add a locked page to the pagecache
959  * @page:	page to add
960  * @mapping:	the page's address_space
961  * @offset:	page index
962  * @gfp_mask:	page allocation mode
963  *
964  * This function is used to add a page to the pagecache. It must be locked.
965  * This function does not add the page to the LRU.  The caller must do that.
966  *
967  * Return: %0 on success, negative error code otherwise.
968  */
add_to_page_cache_locked(struct page * page,struct address_space * mapping,pgoff_t offset,gfp_t gfp_mask)969 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
970 		pgoff_t offset, gfp_t gfp_mask)
971 {
972 	return __add_to_page_cache_locked(page, mapping, offset,
973 					  gfp_mask, NULL);
974 }
975 EXPORT_SYMBOL(add_to_page_cache_locked);
976 
add_to_page_cache_lru(struct page * page,struct address_space * mapping,pgoff_t offset,gfp_t gfp_mask)977 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
978 				pgoff_t offset, gfp_t gfp_mask)
979 {
980 	void *shadow = NULL;
981 	int ret;
982 
983 	__SetPageLocked(page);
984 	ret = __add_to_page_cache_locked(page, mapping, offset,
985 					 gfp_mask, &shadow);
986 	if (unlikely(ret))
987 		__ClearPageLocked(page);
988 	else {
989 		/*
990 		 * The page might have been evicted from cache only
991 		 * recently, in which case it should be activated like
992 		 * any other repeatedly accessed page.
993 		 * The exception is pages getting rewritten; evicting other
994 		 * data from the working set, only to cache data that will
995 		 * get overwritten with something else, is a waste of memory.
996 		 */
997 		WARN_ON_ONCE(PageActive(page));
998 		if (!(gfp_mask & __GFP_WRITE) && shadow)
999 			workingset_refault(page, shadow);
1000 		lru_cache_add(page);
1001 	}
1002 	return ret;
1003 }
1004 EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
1005 
1006 #ifdef CONFIG_NUMA
__page_cache_alloc(gfp_t gfp)1007 struct page *__page_cache_alloc(gfp_t gfp)
1008 {
1009 	int n;
1010 	struct page *page;
1011 
1012 	if (cpuset_do_page_mem_spread()) {
1013 		unsigned int cpuset_mems_cookie;
1014 		do {
1015 			cpuset_mems_cookie = read_mems_allowed_begin();
1016 			n = cpuset_mem_spread_node();
1017 			page = __alloc_pages_node(n, gfp, 0);
1018 		} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
1019 
1020 		return page;
1021 	}
1022 	return alloc_pages(gfp, 0);
1023 }
1024 EXPORT_SYMBOL(__page_cache_alloc);
1025 #endif
1026 
1027 /*
1028  * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
1029  *
1030  * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
1031  *
1032  * @mapping1: the first mapping to lock
1033  * @mapping2: the second mapping to lock
1034  */
filemap_invalidate_lock_two(struct address_space * mapping1,struct address_space * mapping2)1035 void filemap_invalidate_lock_two(struct address_space *mapping1,
1036 				 struct address_space *mapping2)
1037 {
1038 	if (mapping1 > mapping2)
1039 		swap(mapping1, mapping2);
1040 	if (mapping1)
1041 		down_write(&mapping1->invalidate_lock);
1042 	if (mapping2 && mapping1 != mapping2)
1043 		down_write_nested(&mapping2->invalidate_lock, 1);
1044 }
1045 EXPORT_SYMBOL(filemap_invalidate_lock_two);
1046 
1047 /*
1048  * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
1049  *
1050  * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1051  *
1052  * @mapping1: the first mapping to unlock
1053  * @mapping2: the second mapping to unlock
1054  */
filemap_invalidate_unlock_two(struct address_space * mapping1,struct address_space * mapping2)1055 void filemap_invalidate_unlock_two(struct address_space *mapping1,
1056 				   struct address_space *mapping2)
1057 {
1058 	if (mapping1)
1059 		up_write(&mapping1->invalidate_lock);
1060 	if (mapping2 && mapping1 != mapping2)
1061 		up_write(&mapping2->invalidate_lock);
1062 }
1063 EXPORT_SYMBOL(filemap_invalidate_unlock_two);
1064 
1065 /*
1066  * In order to wait for pages to become available there must be
1067  * waitqueues associated with pages. By using a hash table of
1068  * waitqueues where the bucket discipline is to maintain all
1069  * waiters on the same queue and wake all when any of the pages
1070  * become available, and for the woken contexts to check to be
1071  * sure the appropriate page became available, this saves space
1072  * at a cost of "thundering herd" phenomena during rare hash
1073  * collisions.
1074  */
1075 #define PAGE_WAIT_TABLE_BITS 8
1076 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
1077 static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
1078 
page_waitqueue(struct page * page)1079 static wait_queue_head_t *page_waitqueue(struct page *page)
1080 {
1081 	return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
1082 }
1083 
pagecache_init(void)1084 void __init pagecache_init(void)
1085 {
1086 	int i;
1087 
1088 	for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
1089 		init_waitqueue_head(&page_wait_table[i]);
1090 
1091 	page_writeback_init();
1092 }
1093 
1094 /*
1095  * The page wait code treats the "wait->flags" somewhat unusually, because
1096  * we have multiple different kinds of waits, not just the usual "exclusive"
1097  * one.
1098  *
1099  * We have:
1100  *
1101  *  (a) no special bits set:
1102  *
1103  *	We're just waiting for the bit to be released, and when a waker
1104  *	calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up,
1105  *	and remove it from the wait queue.
1106  *
1107  *	Simple and straightforward.
1108  *
1109  *  (b) WQ_FLAG_EXCLUSIVE:
1110  *
1111  *	The waiter is waiting to get the lock, and only one waiter should
1112  *	be woken up to avoid any thundering herd behavior. We'll set the
1113  *	WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue.
1114  *
1115  *	This is the traditional exclusive wait.
1116  *
1117  *  (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM:
1118  *
1119  *	The waiter is waiting to get the bit, and additionally wants the
1120  *	lock to be transferred to it for fair lock behavior. If the lock
1121  *	cannot be taken, we stop walking the wait queue without waking
1122  *	the waiter.
1123  *
1124  *	This is the "fair lock handoff" case, and in addition to setting
1125  *	WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see
1126  *	that it now has the lock.
1127  */
wake_page_function(wait_queue_entry_t * wait,unsigned mode,int sync,void * arg)1128 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
1129 {
1130 	unsigned int flags;
1131 	struct wait_page_key *key = arg;
1132 	struct wait_page_queue *wait_page
1133 		= container_of(wait, struct wait_page_queue, wait);
1134 
1135 	if (!wake_page_match(wait_page, key))
1136 		return 0;
1137 
1138 	/*
1139 	 * If it's a lock handoff wait, we get the bit for it, and
1140 	 * stop walking (and do not wake it up) if we can't.
1141 	 */
1142 	flags = wait->flags;
1143 	if (flags & WQ_FLAG_EXCLUSIVE) {
1144 		if (test_bit(key->bit_nr, &key->page->flags))
1145 			return -1;
1146 		if (flags & WQ_FLAG_CUSTOM) {
1147 			if (test_and_set_bit(key->bit_nr, &key->page->flags))
1148 				return -1;
1149 			flags |= WQ_FLAG_DONE;
1150 		}
1151 	}
1152 
1153 	/*
1154 	 * We are holding the wait-queue lock, but the waiter that
1155 	 * is waiting for this will be checking the flags without
1156 	 * any locking.
1157 	 *
1158 	 * So update the flags atomically, and wake up the waiter
1159 	 * afterwards to avoid any races. This store-release pairs
1160 	 * with the load-acquire in wait_on_page_bit_common().
1161 	 */
1162 	smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
1163 	wake_up_state(wait->private, mode);
1164 
1165 	/*
1166 	 * Ok, we have successfully done what we're waiting for,
1167 	 * and we can unconditionally remove the wait entry.
1168 	 *
1169 	 * Note that this pairs with the "finish_wait()" in the
1170 	 * waiter, and has to be the absolute last thing we do.
1171 	 * After this list_del_init(&wait->entry) the wait entry
1172 	 * might be de-allocated and the process might even have
1173 	 * exited.
1174 	 */
1175 	list_del_init_careful(&wait->entry);
1176 	return (flags & WQ_FLAG_EXCLUSIVE) != 0;
1177 }
1178 
wake_up_page_bit(struct page * page,int bit_nr)1179 static void wake_up_page_bit(struct page *page, int bit_nr)
1180 {
1181 	wait_queue_head_t *q = page_waitqueue(page);
1182 	struct wait_page_key key;
1183 	unsigned long flags;
1184 	wait_queue_entry_t bookmark;
1185 
1186 	key.page = page;
1187 	key.bit_nr = bit_nr;
1188 	key.page_match = 0;
1189 
1190 	bookmark.flags = 0;
1191 	bookmark.private = NULL;
1192 	bookmark.func = NULL;
1193 	INIT_LIST_HEAD(&bookmark.entry);
1194 
1195 	spin_lock_irqsave(&q->lock, flags);
1196 	__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1197 
1198 	while (bookmark.flags & WQ_FLAG_BOOKMARK) {
1199 		/*
1200 		 * Take a breather from holding the lock,
1201 		 * allow pages that finish wake up asynchronously
1202 		 * to acquire the lock and remove themselves
1203 		 * from wait queue
1204 		 */
1205 		spin_unlock_irqrestore(&q->lock, flags);
1206 		cpu_relax();
1207 		spin_lock_irqsave(&q->lock, flags);
1208 		__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1209 	}
1210 
1211 	/*
1212 	 * It is possible for other pages to have collided on the waitqueue
1213 	 * hash, so in that case check for a page match. That prevents a long-
1214 	 * term waiter
1215 	 *
1216 	 * It is still possible to miss a case here, when we woke page waiters
1217 	 * and removed them from the waitqueue, but there are still other
1218 	 * page waiters.
1219 	 */
1220 	if (!waitqueue_active(q) || !key.page_match) {
1221 		ClearPageWaiters(page);
1222 		/*
1223 		 * It's possible to miss clearing Waiters here, when we woke
1224 		 * our page waiters, but the hashed waitqueue has waiters for
1225 		 * other pages on it.
1226 		 *
1227 		 * That's okay, it's a rare case. The next waker will clear it.
1228 		 */
1229 	}
1230 	spin_unlock_irqrestore(&q->lock, flags);
1231 }
1232 
wake_up_page(struct page * page,int bit)1233 static void wake_up_page(struct page *page, int bit)
1234 {
1235 	if (!PageWaiters(page))
1236 		return;
1237 	wake_up_page_bit(page, bit);
1238 }
1239 
1240 /*
1241  * A choice of three behaviors for wait_on_page_bit_common():
1242  */
1243 enum behavior {
1244 	EXCLUSIVE,	/* Hold ref to page and take the bit when woken, like
1245 			 * __lock_page() waiting on then setting PG_locked.
1246 			 */
1247 	SHARED,		/* Hold ref to page and check the bit when woken, like
1248 			 * wait_on_page_writeback() waiting on PG_writeback.
1249 			 */
1250 	DROP,		/* Drop ref to page before wait, no check when woken,
1251 			 * like put_and_wait_on_page_locked() on PG_locked.
1252 			 */
1253 };
1254 
1255 /*
1256  * Attempt to check (or get) the page bit, and mark us done
1257  * if successful.
1258  */
trylock_page_bit_common(struct page * page,int bit_nr,struct wait_queue_entry * wait)1259 static inline bool trylock_page_bit_common(struct page *page, int bit_nr,
1260 					struct wait_queue_entry *wait)
1261 {
1262 	if (wait->flags & WQ_FLAG_EXCLUSIVE) {
1263 		if (test_and_set_bit(bit_nr, &page->flags))
1264 			return false;
1265 	} else if (test_bit(bit_nr, &page->flags))
1266 		return false;
1267 
1268 	wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
1269 	return true;
1270 }
1271 
1272 /* How many times do we accept lock stealing from under a waiter? */
1273 int sysctl_page_lock_unfairness = 5;
1274 
wait_on_page_bit_common(wait_queue_head_t * q,struct page * page,int bit_nr,int state,enum behavior behavior)1275 static inline int wait_on_page_bit_common(wait_queue_head_t *q,
1276 	struct page *page, int bit_nr, int state, enum behavior behavior)
1277 {
1278 	int unfairness = sysctl_page_lock_unfairness;
1279 	struct wait_page_queue wait_page;
1280 	wait_queue_entry_t *wait = &wait_page.wait;
1281 	bool thrashing = false;
1282 	bool delayacct = false;
1283 	unsigned long pflags;
1284 
1285 	if (bit_nr == PG_locked &&
1286 	    !PageUptodate(page) && PageWorkingset(page)) {
1287 		if (!PageSwapBacked(page)) {
1288 			delayacct_thrashing_start();
1289 			delayacct = true;
1290 		}
1291 		psi_memstall_enter(&pflags);
1292 		thrashing = true;
1293 	}
1294 
1295 	init_wait(wait);
1296 	wait->func = wake_page_function;
1297 	wait_page.page = page;
1298 	wait_page.bit_nr = bit_nr;
1299 
1300 repeat:
1301 	wait->flags = 0;
1302 	if (behavior == EXCLUSIVE) {
1303 		wait->flags = WQ_FLAG_EXCLUSIVE;
1304 		if (--unfairness < 0)
1305 			wait->flags |= WQ_FLAG_CUSTOM;
1306 	}
1307 
1308 	/*
1309 	 * Do one last check whether we can get the
1310 	 * page bit synchronously.
1311 	 *
1312 	 * Do the SetPageWaiters() marking before that
1313 	 * to let any waker we _just_ missed know they
1314 	 * need to wake us up (otherwise they'll never
1315 	 * even go to the slow case that looks at the
1316 	 * page queue), and add ourselves to the wait
1317 	 * queue if we need to sleep.
1318 	 *
1319 	 * This part needs to be done under the queue
1320 	 * lock to avoid races.
1321 	 */
1322 	spin_lock_irq(&q->lock);
1323 	SetPageWaiters(page);
1324 	if (!trylock_page_bit_common(page, bit_nr, wait))
1325 		__add_wait_queue_entry_tail(q, wait);
1326 	spin_unlock_irq(&q->lock);
1327 
1328 	/*
1329 	 * From now on, all the logic will be based on
1330 	 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to
1331 	 * see whether the page bit testing has already
1332 	 * been done by the wake function.
1333 	 *
1334 	 * We can drop our reference to the page.
1335 	 */
1336 	if (behavior == DROP)
1337 		put_page(page);
1338 
1339 	/*
1340 	 * Note that until the "finish_wait()", or until
1341 	 * we see the WQ_FLAG_WOKEN flag, we need to
1342 	 * be very careful with the 'wait->flags', because
1343 	 * we may race with a waker that sets them.
1344 	 */
1345 	for (;;) {
1346 		unsigned int flags;
1347 
1348 		set_current_state(state);
1349 
1350 		/* Loop until we've been woken or interrupted */
1351 		flags = smp_load_acquire(&wait->flags);
1352 		if (!(flags & WQ_FLAG_WOKEN)) {
1353 			if (signal_pending_state(state, current))
1354 				break;
1355 
1356 			io_schedule();
1357 			continue;
1358 		}
1359 
1360 		/* If we were non-exclusive, we're done */
1361 		if (behavior != EXCLUSIVE)
1362 			break;
1363 
1364 		/* If the waker got the lock for us, we're done */
1365 		if (flags & WQ_FLAG_DONE)
1366 			break;
1367 
1368 		/*
1369 		 * Otherwise, if we're getting the lock, we need to
1370 		 * try to get it ourselves.
1371 		 *
1372 		 * And if that fails, we'll have to retry this all.
1373 		 */
1374 		if (unlikely(test_and_set_bit(bit_nr, &page->flags)))
1375 			goto repeat;
1376 
1377 		wait->flags |= WQ_FLAG_DONE;
1378 		break;
1379 	}
1380 
1381 	/*
1382 	 * If a signal happened, this 'finish_wait()' may remove the last
1383 	 * waiter from the wait-queues, but the PageWaiters bit will remain
1384 	 * set. That's ok. The next wakeup will take care of it, and trying
1385 	 * to do it here would be difficult and prone to races.
1386 	 */
1387 	finish_wait(q, wait);
1388 
1389 	if (thrashing) {
1390 		if (delayacct)
1391 			delayacct_thrashing_end();
1392 		psi_memstall_leave(&pflags);
1393 	}
1394 
1395 	/*
1396 	 * NOTE! The wait->flags weren't stable until we've done the
1397 	 * 'finish_wait()', and we could have exited the loop above due
1398 	 * to a signal, and had a wakeup event happen after the signal
1399 	 * test but before the 'finish_wait()'.
1400 	 *
1401 	 * So only after the finish_wait() can we reliably determine
1402 	 * if we got woken up or not, so we can now figure out the final
1403 	 * return value based on that state without races.
1404 	 *
1405 	 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
1406 	 * waiter, but an exclusive one requires WQ_FLAG_DONE.
1407 	 */
1408 	if (behavior == EXCLUSIVE)
1409 		return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
1410 
1411 	return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
1412 }
1413 
wait_on_page_bit(struct page * page,int bit_nr)1414 void wait_on_page_bit(struct page *page, int bit_nr)
1415 {
1416 	wait_queue_head_t *q = page_waitqueue(page);
1417 	wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
1418 }
1419 EXPORT_SYMBOL(wait_on_page_bit);
1420 
wait_on_page_bit_killable(struct page * page,int bit_nr)1421 int wait_on_page_bit_killable(struct page *page, int bit_nr)
1422 {
1423 	wait_queue_head_t *q = page_waitqueue(page);
1424 	return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, SHARED);
1425 }
1426 EXPORT_SYMBOL(wait_on_page_bit_killable);
1427 
1428 /**
1429  * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1430  * @page: The page to wait for.
1431  * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc).
1432  *
1433  * The caller should hold a reference on @page.  They expect the page to
1434  * become unlocked relatively soon, but do not wish to hold up migration
1435  * (for example) by holding the reference while waiting for the page to
1436  * come unlocked.  After this function returns, the caller should not
1437  * dereference @page.
1438  *
1439  * Return: 0 if the page was unlocked or -EINTR if interrupted by a signal.
1440  */
put_and_wait_on_page_locked(struct page * page,int state)1441 int put_and_wait_on_page_locked(struct page *page, int state)
1442 {
1443 	wait_queue_head_t *q;
1444 
1445 	page = compound_head(page);
1446 	q = page_waitqueue(page);
1447 	return wait_on_page_bit_common(q, page, PG_locked, state, DROP);
1448 }
1449 
1450 /**
1451  * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1452  * @page: Page defining the wait queue of interest
1453  * @waiter: Waiter to add to the queue
1454  *
1455  * Add an arbitrary @waiter to the wait queue for the nominated @page.
1456  */
add_page_wait_queue(struct page * page,wait_queue_entry_t * waiter)1457 void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
1458 {
1459 	wait_queue_head_t *q = page_waitqueue(page);
1460 	unsigned long flags;
1461 
1462 	spin_lock_irqsave(&q->lock, flags);
1463 	__add_wait_queue_entry_tail(q, waiter);
1464 	SetPageWaiters(page);
1465 	spin_unlock_irqrestore(&q->lock, flags);
1466 }
1467 EXPORT_SYMBOL_GPL(add_page_wait_queue);
1468 
1469 #ifndef clear_bit_unlock_is_negative_byte
1470 
1471 /*
1472  * PG_waiters is the high bit in the same byte as PG_lock.
1473  *
1474  * On x86 (and on many other architectures), we can clear PG_lock and
1475  * test the sign bit at the same time. But if the architecture does
1476  * not support that special operation, we just do this all by hand
1477  * instead.
1478  *
1479  * The read of PG_waiters has to be after (or concurrently with) PG_locked
1480  * being cleared, but a memory barrier should be unnecessary since it is
1481  * in the same byte as PG_locked.
1482  */
clear_bit_unlock_is_negative_byte(long nr,volatile void * mem)1483 static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
1484 {
1485 	clear_bit_unlock(nr, mem);
1486 	/* smp_mb__after_atomic(); */
1487 	return test_bit(PG_waiters, mem);
1488 }
1489 
1490 #endif
1491 
1492 /**
1493  * unlock_page - unlock a locked page
1494  * @page: the page
1495  *
1496  * Unlocks the page and wakes up sleepers in wait_on_page_locked().
1497  * Also wakes sleepers in wait_on_page_writeback() because the wakeup
1498  * mechanism between PageLocked pages and PageWriteback pages is shared.
1499  * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1500  *
1501  * Note that this depends on PG_waiters being the sign bit in the byte
1502  * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1503  * clear the PG_locked bit and test PG_waiters at the same time fairly
1504  * portably (architectures that do LL/SC can test any bit, while x86 can
1505  * test the sign bit).
1506  */
unlock_page(struct page * page)1507 void unlock_page(struct page *page)
1508 {
1509 	BUILD_BUG_ON(PG_waiters != 7);
1510 	page = compound_head(page);
1511 	VM_BUG_ON_PAGE(!PageLocked(page), page);
1512 	if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags))
1513 		wake_up_page_bit(page, PG_locked);
1514 }
1515 EXPORT_SYMBOL(unlock_page);
1516 
1517 /**
1518  * end_page_private_2 - Clear PG_private_2 and release any waiters
1519  * @page: The page
1520  *
1521  * Clear the PG_private_2 bit on a page and wake up any sleepers waiting for
1522  * this.  The page ref held for PG_private_2 being set is released.
1523  *
1524  * This is, for example, used when a netfs page is being written to a local
1525  * disk cache, thereby allowing writes to the cache for the same page to be
1526  * serialised.
1527  */
end_page_private_2(struct page * page)1528 void end_page_private_2(struct page *page)
1529 {
1530 	page = compound_head(page);
1531 	VM_BUG_ON_PAGE(!PagePrivate2(page), page);
1532 	clear_bit_unlock(PG_private_2, &page->flags);
1533 	wake_up_page_bit(page, PG_private_2);
1534 	put_page(page);
1535 }
1536 EXPORT_SYMBOL(end_page_private_2);
1537 
1538 /**
1539  * wait_on_page_private_2 - Wait for PG_private_2 to be cleared on a page
1540  * @page: The page to wait on
1541  *
1542  * Wait for PG_private_2 (aka PG_fscache) to be cleared on a page.
1543  */
wait_on_page_private_2(struct page * page)1544 void wait_on_page_private_2(struct page *page)
1545 {
1546 	page = compound_head(page);
1547 	while (PagePrivate2(page))
1548 		wait_on_page_bit(page, PG_private_2);
1549 }
1550 EXPORT_SYMBOL(wait_on_page_private_2);
1551 
1552 /**
1553  * wait_on_page_private_2_killable - Wait for PG_private_2 to be cleared on a page
1554  * @page: The page to wait on
1555  *
1556  * Wait for PG_private_2 (aka PG_fscache) to be cleared on a page or until a
1557  * fatal signal is received by the calling task.
1558  *
1559  * Return:
1560  * - 0 if successful.
1561  * - -EINTR if a fatal signal was encountered.
1562  */
wait_on_page_private_2_killable(struct page * page)1563 int wait_on_page_private_2_killable(struct page *page)
1564 {
1565 	int ret = 0;
1566 
1567 	page = compound_head(page);
1568 	while (PagePrivate2(page)) {
1569 		ret = wait_on_page_bit_killable(page, PG_private_2);
1570 		if (ret < 0)
1571 			break;
1572 	}
1573 
1574 	return ret;
1575 }
1576 EXPORT_SYMBOL(wait_on_page_private_2_killable);
1577 
1578 /**
1579  * end_page_writeback - end writeback against a page
1580  * @page: the page
1581  */
end_page_writeback(struct page * page)1582 void end_page_writeback(struct page *page)
1583 {
1584 	/*
1585 	 * TestClearPageReclaim could be used here but it is an atomic
1586 	 * operation and overkill in this particular case. Failing to
1587 	 * shuffle a page marked for immediate reclaim is too mild to
1588 	 * justify taking an atomic operation penalty at the end of
1589 	 * ever page writeback.
1590 	 */
1591 	if (PageReclaim(page)) {
1592 		ClearPageReclaim(page);
1593 		rotate_reclaimable_page(page);
1594 	}
1595 
1596 	/*
1597 	 * Writeback does not hold a page reference of its own, relying
1598 	 * on truncation to wait for the clearing of PG_writeback.
1599 	 * But here we must make sure that the page is not freed and
1600 	 * reused before the wake_up_page().
1601 	 */
1602 	get_page(page);
1603 	if (!test_clear_page_writeback(page))
1604 		BUG();
1605 
1606 	smp_mb__after_atomic();
1607 	wake_up_page(page, PG_writeback);
1608 	put_page(page);
1609 }
1610 EXPORT_SYMBOL(end_page_writeback);
1611 
1612 /*
1613  * After completing I/O on a page, call this routine to update the page
1614  * flags appropriately
1615  */
page_endio(struct page * page,bool is_write,int err)1616 void page_endio(struct page *page, bool is_write, int err)
1617 {
1618 	if (!is_write) {
1619 		if (!err) {
1620 			SetPageUptodate(page);
1621 		} else {
1622 			ClearPageUptodate(page);
1623 			SetPageError(page);
1624 		}
1625 		unlock_page(page);
1626 	} else {
1627 		if (err) {
1628 			struct address_space *mapping;
1629 
1630 			SetPageError(page);
1631 			mapping = page_mapping(page);
1632 			if (mapping)
1633 				mapping_set_error(mapping, err);
1634 		}
1635 		end_page_writeback(page);
1636 	}
1637 }
1638 EXPORT_SYMBOL_GPL(page_endio);
1639 
1640 /**
1641  * __lock_page - get a lock on the page, assuming we need to sleep to get it
1642  * @__page: the page to lock
1643  */
__lock_page(struct page * __page)1644 void __lock_page(struct page *__page)
1645 {
1646 	struct page *page = compound_head(__page);
1647 	wait_queue_head_t *q = page_waitqueue(page);
1648 	wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE,
1649 				EXCLUSIVE);
1650 }
1651 EXPORT_SYMBOL(__lock_page);
1652 
__lock_page_killable(struct page * __page)1653 int __lock_page_killable(struct page *__page)
1654 {
1655 	struct page *page = compound_head(__page);
1656 	wait_queue_head_t *q = page_waitqueue(page);
1657 	return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE,
1658 					EXCLUSIVE);
1659 }
1660 EXPORT_SYMBOL_GPL(__lock_page_killable);
1661 
__lock_page_async(struct page * page,struct wait_page_queue * wait)1662 int __lock_page_async(struct page *page, struct wait_page_queue *wait)
1663 {
1664 	struct wait_queue_head *q = page_waitqueue(page);
1665 	int ret = 0;
1666 
1667 	wait->page = page;
1668 	wait->bit_nr = PG_locked;
1669 
1670 	spin_lock_irq(&q->lock);
1671 	__add_wait_queue_entry_tail(q, &wait->wait);
1672 	SetPageWaiters(page);
1673 	ret = !trylock_page(page);
1674 	/*
1675 	 * If we were successful now, we know we're still on the
1676 	 * waitqueue as we're still under the lock. This means it's
1677 	 * safe to remove and return success, we know the callback
1678 	 * isn't going to trigger.
1679 	 */
1680 	if (!ret)
1681 		__remove_wait_queue(q, &wait->wait);
1682 	else
1683 		ret = -EIOCBQUEUED;
1684 	spin_unlock_irq(&q->lock);
1685 	return ret;
1686 }
1687 
1688 /*
1689  * Return values:
1690  * 1 - page is locked; mmap_lock is still held.
1691  * 0 - page is not locked.
1692  *     mmap_lock has been released (mmap_read_unlock(), unless flags had both
1693  *     FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
1694  *     which case mmap_lock is still held.
1695  *
1696  * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
1697  * with the page locked and the mmap_lock unperturbed.
1698  */
__lock_page_or_retry(struct page * page,struct mm_struct * mm,unsigned int flags)1699 int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
1700 			 unsigned int flags)
1701 {
1702 	if (fault_flag_allow_retry_first(flags)) {
1703 		/*
1704 		 * CAUTION! In this case, mmap_lock is not released
1705 		 * even though return 0.
1706 		 */
1707 		if (flags & FAULT_FLAG_RETRY_NOWAIT)
1708 			return 0;
1709 
1710 		mmap_read_unlock(mm);
1711 		if (flags & FAULT_FLAG_KILLABLE)
1712 			wait_on_page_locked_killable(page);
1713 		else
1714 			wait_on_page_locked(page);
1715 		return 0;
1716 	}
1717 	if (flags & FAULT_FLAG_KILLABLE) {
1718 		int ret;
1719 
1720 		ret = __lock_page_killable(page);
1721 		if (ret) {
1722 			mmap_read_unlock(mm);
1723 			return 0;
1724 		}
1725 	} else {
1726 		__lock_page(page);
1727 	}
1728 	return 1;
1729 
1730 }
1731 
1732 /**
1733  * page_cache_next_miss() - Find the next gap in the page cache.
1734  * @mapping: Mapping.
1735  * @index: Index.
1736  * @max_scan: Maximum range to search.
1737  *
1738  * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1739  * gap with the lowest index.
1740  *
1741  * This function may be called under the rcu_read_lock.  However, this will
1742  * not atomically search a snapshot of the cache at a single point in time.
1743  * For example, if a gap is created at index 5, then subsequently a gap is
1744  * created at index 10, page_cache_next_miss covering both indices may
1745  * return 10 if called under the rcu_read_lock.
1746  *
1747  * Return: The index of the gap if found, otherwise an index outside the
1748  * range specified (in which case 'return - index >= max_scan' will be true).
1749  * In the rare case of index wrap-around, 0 will be returned.
1750  */
page_cache_next_miss(struct address_space * mapping,pgoff_t index,unsigned long max_scan)1751 pgoff_t page_cache_next_miss(struct address_space *mapping,
1752 			     pgoff_t index, unsigned long max_scan)
1753 {
1754 	XA_STATE(xas, &mapping->i_pages, index);
1755 
1756 	while (max_scan--) {
1757 		void *entry = xas_next(&xas);
1758 		if (!entry || xa_is_value(entry))
1759 			break;
1760 		if (xas.xa_index == 0)
1761 			break;
1762 	}
1763 
1764 	return xas.xa_index;
1765 }
1766 EXPORT_SYMBOL(page_cache_next_miss);
1767 
1768 /**
1769  * page_cache_prev_miss() - Find the previous gap in the page cache.
1770  * @mapping: Mapping.
1771  * @index: Index.
1772  * @max_scan: Maximum range to search.
1773  *
1774  * Search the range [max(index - max_scan + 1, 0), index] for the
1775  * gap with the highest index.
1776  *
1777  * This function may be called under the rcu_read_lock.  However, this will
1778  * not atomically search a snapshot of the cache at a single point in time.
1779  * For example, if a gap is created at index 10, then subsequently a gap is
1780  * created at index 5, page_cache_prev_miss() covering both indices may
1781  * return 5 if called under the rcu_read_lock.
1782  *
1783  * Return: The index of the gap if found, otherwise an index outside the
1784  * range specified (in which case 'index - return >= max_scan' will be true).
1785  * In the rare case of wrap-around, ULONG_MAX will be returned.
1786  */
page_cache_prev_miss(struct address_space * mapping,pgoff_t index,unsigned long max_scan)1787 pgoff_t page_cache_prev_miss(struct address_space *mapping,
1788 			     pgoff_t index, unsigned long max_scan)
1789 {
1790 	XA_STATE(xas, &mapping->i_pages, index);
1791 
1792 	while (max_scan--) {
1793 		void *entry = xas_prev(&xas);
1794 		if (!entry || xa_is_value(entry))
1795 			break;
1796 		if (xas.xa_index == ULONG_MAX)
1797 			break;
1798 	}
1799 
1800 	return xas.xa_index;
1801 }
1802 EXPORT_SYMBOL(page_cache_prev_miss);
1803 
1804 /*
1805  * mapping_get_entry - Get a page cache entry.
1806  * @mapping: the address_space to search
1807  * @index: The page cache index.
1808  *
1809  * Looks up the page cache slot at @mapping & @index.  If there is a
1810  * page cache page, the head page is returned with an increased refcount.
1811  *
1812  * If the slot holds a shadow entry of a previously evicted page, or a
1813  * swap entry from shmem/tmpfs, it is returned.
1814  *
1815  * Return: The head page or shadow entry, %NULL if nothing is found.
1816  */
mapping_get_entry(struct address_space * mapping,pgoff_t index)1817 static struct page *mapping_get_entry(struct address_space *mapping,
1818 		pgoff_t index)
1819 {
1820 	XA_STATE(xas, &mapping->i_pages, index);
1821 	struct page *page;
1822 
1823 	rcu_read_lock();
1824 repeat:
1825 	xas_reset(&xas);
1826 	page = xas_load(&xas);
1827 	if (xas_retry(&xas, page))
1828 		goto repeat;
1829 	/*
1830 	 * A shadow entry of a recently evicted page, or a swap entry from
1831 	 * shmem/tmpfs.  Return it without attempting to raise page count.
1832 	 */
1833 	if (!page || xa_is_value(page))
1834 		goto out;
1835 
1836 	if (!page_cache_get_speculative(page))
1837 		goto repeat;
1838 
1839 	/*
1840 	 * Has the page moved or been split?
1841 	 * This is part of the lockless pagecache protocol. See
1842 	 * include/linux/pagemap.h for details.
1843 	 */
1844 	if (unlikely(page != xas_reload(&xas))) {
1845 		put_page(page);
1846 		goto repeat;
1847 	}
1848 out:
1849 	rcu_read_unlock();
1850 
1851 	return page;
1852 }
1853 
1854 /**
1855  * pagecache_get_page - Find and get a reference to a page.
1856  * @mapping: The address_space to search.
1857  * @index: The page index.
1858  * @fgp_flags: %FGP flags modify how the page is returned.
1859  * @gfp_mask: Memory allocation flags to use if %FGP_CREAT is specified.
1860  *
1861  * Looks up the page cache entry at @mapping & @index.
1862  *
1863  * @fgp_flags can be zero or more of these flags:
1864  *
1865  * * %FGP_ACCESSED - The page will be marked accessed.
1866  * * %FGP_LOCK - The page is returned locked.
1867  * * %FGP_HEAD - If the page is present and a THP, return the head page
1868  *   rather than the exact page specified by the index.
1869  * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it
1870  *   instead of allocating a new page to replace it.
1871  * * %FGP_CREAT - If no page is present then a new page is allocated using
1872  *   @gfp_mask and added to the page cache and the VM's LRU list.
1873  *   The page is returned locked and with an increased refcount.
1874  * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1875  *   page is already in cache.  If the page was allocated, unlock it before
1876  *   returning so the caller can do the same dance.
1877  * * %FGP_WRITE - The page will be written
1878  * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
1879  * * %FGP_NOWAIT - Don't get blocked by page lock
1880  *
1881  * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even
1882  * if the %GFP flags specified for %FGP_CREAT are atomic.
1883  *
1884  * If there is a page cache page, it is returned with an increased refcount.
1885  *
1886  * Return: The found page or %NULL otherwise.
1887  */
pagecache_get_page(struct address_space * mapping,pgoff_t index,int fgp_flags,gfp_t gfp_mask)1888 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
1889 		int fgp_flags, gfp_t gfp_mask)
1890 {
1891 	struct page *page;
1892 
1893 repeat:
1894 	page = mapping_get_entry(mapping, index);
1895 	if (xa_is_value(page)) {
1896 		if (fgp_flags & FGP_ENTRY)
1897 			return page;
1898 		page = NULL;
1899 	}
1900 	if (!page)
1901 		goto no_page;
1902 
1903 	if (fgp_flags & FGP_LOCK) {
1904 		if (fgp_flags & FGP_NOWAIT) {
1905 			if (!trylock_page(page)) {
1906 				put_page(page);
1907 				return NULL;
1908 			}
1909 		} else {
1910 			lock_page(page);
1911 		}
1912 
1913 		/* Has the page been truncated? */
1914 		if (unlikely(page->mapping != mapping)) {
1915 			unlock_page(page);
1916 			put_page(page);
1917 			goto repeat;
1918 		}
1919 		VM_BUG_ON_PAGE(!thp_contains(page, index), page);
1920 	}
1921 
1922 	if (fgp_flags & FGP_ACCESSED)
1923 		mark_page_accessed(page);
1924 	else if (fgp_flags & FGP_WRITE) {
1925 		/* Clear idle flag for buffer write */
1926 		if (page_is_idle(page))
1927 			clear_page_idle(page);
1928 	}
1929 	if (!(fgp_flags & FGP_HEAD))
1930 		page = find_subpage(page, index);
1931 
1932 no_page:
1933 	if (!page && (fgp_flags & FGP_CREAT)) {
1934 		int err;
1935 		if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
1936 			gfp_mask |= __GFP_WRITE;
1937 		if (fgp_flags & FGP_NOFS)
1938 			gfp_mask &= ~__GFP_FS;
1939 
1940 		page = __page_cache_alloc(gfp_mask);
1941 		if (!page)
1942 			return NULL;
1943 
1944 		if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
1945 			fgp_flags |= FGP_LOCK;
1946 
1947 		/* Init accessed so avoid atomic mark_page_accessed later */
1948 		if (fgp_flags & FGP_ACCESSED)
1949 			__SetPageReferenced(page);
1950 
1951 		err = add_to_page_cache_lru(page, mapping, index, gfp_mask);
1952 		if (unlikely(err)) {
1953 			put_page(page);
1954 			page = NULL;
1955 			if (err == -EEXIST)
1956 				goto repeat;
1957 		}
1958 
1959 		/*
1960 		 * add_to_page_cache_lru locks the page, and for mmap we expect
1961 		 * an unlocked page.
1962 		 */
1963 		if (page && (fgp_flags & FGP_FOR_MMAP))
1964 			unlock_page(page);
1965 	}
1966 
1967 	return page;
1968 }
1969 EXPORT_SYMBOL(pagecache_get_page);
1970 
find_get_entry(struct xa_state * xas,pgoff_t max,xa_mark_t mark)1971 static inline struct page *find_get_entry(struct xa_state *xas, pgoff_t max,
1972 		xa_mark_t mark)
1973 {
1974 	struct page *page;
1975 
1976 retry:
1977 	if (mark == XA_PRESENT)
1978 		page = xas_find(xas, max);
1979 	else
1980 		page = xas_find_marked(xas, max, mark);
1981 
1982 	if (xas_retry(xas, page))
1983 		goto retry;
1984 	/*
1985 	 * A shadow entry of a recently evicted page, a swap
1986 	 * entry from shmem/tmpfs or a DAX entry.  Return it
1987 	 * without attempting to raise page count.
1988 	 */
1989 	if (!page || xa_is_value(page))
1990 		return page;
1991 
1992 	if (!page_cache_get_speculative(page))
1993 		goto reset;
1994 
1995 	/* Has the page moved or been split? */
1996 	if (unlikely(page != xas_reload(xas))) {
1997 		put_page(page);
1998 		goto reset;
1999 	}
2000 
2001 	return page;
2002 reset:
2003 	xas_reset(xas);
2004 	goto retry;
2005 }
2006 
2007 /**
2008  * find_get_entries - gang pagecache lookup
2009  * @mapping:	The address_space to search
2010  * @start:	The starting page cache index
2011  * @end:	The final page index (inclusive).
2012  * @pvec:	Where the resulting entries are placed.
2013  * @indices:	The cache indices corresponding to the entries in @entries
2014  *
2015  * find_get_entries() will search for and return a batch of entries in
2016  * the mapping.  The entries are placed in @pvec.  find_get_entries()
2017  * takes a reference on any actual pages it returns.
2018  *
2019  * The search returns a group of mapping-contiguous page cache entries
2020  * with ascending indexes.  There may be holes in the indices due to
2021  * not-present pages.
2022  *
2023  * Any shadow entries of evicted pages, or swap entries from
2024  * shmem/tmpfs, are included in the returned array.
2025  *
2026  * If it finds a Transparent Huge Page, head or tail, find_get_entries()
2027  * stops at that page: the caller is likely to have a better way to handle
2028  * the compound page as a whole, and then skip its extent, than repeatedly
2029  * calling find_get_entries() to return all its tails.
2030  *
2031  * Return: the number of pages and shadow entries which were found.
2032  */
find_get_entries(struct address_space * mapping,pgoff_t start,pgoff_t end,struct pagevec * pvec,pgoff_t * indices)2033 unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
2034 		pgoff_t end, struct pagevec *pvec, pgoff_t *indices)
2035 {
2036 	XA_STATE(xas, &mapping->i_pages, start);
2037 	struct page *page;
2038 	unsigned int ret = 0;
2039 	unsigned nr_entries = PAGEVEC_SIZE;
2040 
2041 	rcu_read_lock();
2042 	while ((page = find_get_entry(&xas, end, XA_PRESENT))) {
2043 		/*
2044 		 * Terminate early on finding a THP, to allow the caller to
2045 		 * handle it all at once; but continue if this is hugetlbfs.
2046 		 */
2047 		if (!xa_is_value(page) && PageTransHuge(page) &&
2048 				!PageHuge(page)) {
2049 			page = find_subpage(page, xas.xa_index);
2050 			nr_entries = ret + 1;
2051 		}
2052 
2053 		indices[ret] = xas.xa_index;
2054 		pvec->pages[ret] = page;
2055 		if (++ret == nr_entries)
2056 			break;
2057 	}
2058 	rcu_read_unlock();
2059 
2060 	pvec->nr = ret;
2061 	return ret;
2062 }
2063 
2064 /**
2065  * find_lock_entries - Find a batch of pagecache entries.
2066  * @mapping:	The address_space to search.
2067  * @start:	The starting page cache index.
2068  * @end:	The final page index (inclusive).
2069  * @pvec:	Where the resulting entries are placed.
2070  * @indices:	The cache indices of the entries in @pvec.
2071  *
2072  * find_lock_entries() will return a batch of entries from @mapping.
2073  * Swap, shadow and DAX entries are included.  Pages are returned
2074  * locked and with an incremented refcount.  Pages which are locked by
2075  * somebody else or under writeback are skipped.  Only the head page of
2076  * a THP is returned.  Pages which are partially outside the range are
2077  * not returned.
2078  *
2079  * The entries have ascending indexes.  The indices may not be consecutive
2080  * due to not-present entries, THP pages, pages which could not be locked
2081  * or pages under writeback.
2082  *
2083  * Return: The number of entries which were found.
2084  */
find_lock_entries(struct address_space * mapping,pgoff_t start,pgoff_t end,struct pagevec * pvec,pgoff_t * indices)2085 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
2086 		pgoff_t end, struct pagevec *pvec, pgoff_t *indices)
2087 {
2088 	XA_STATE(xas, &mapping->i_pages, start);
2089 	struct page *page;
2090 
2091 	rcu_read_lock();
2092 	while ((page = find_get_entry(&xas, end, XA_PRESENT))) {
2093 		if (!xa_is_value(page)) {
2094 			if (page->index < start)
2095 				goto put;
2096 			VM_BUG_ON_PAGE(page->index != xas.xa_index, page);
2097 			if (page->index + thp_nr_pages(page) - 1 > end)
2098 				goto put;
2099 			if (!trylock_page(page))
2100 				goto put;
2101 			if (page->mapping != mapping || PageWriteback(page))
2102 				goto unlock;
2103 			VM_BUG_ON_PAGE(!thp_contains(page, xas.xa_index),
2104 					page);
2105 		}
2106 		indices[pvec->nr] = xas.xa_index;
2107 		if (!pagevec_add(pvec, page))
2108 			break;
2109 		goto next;
2110 unlock:
2111 		unlock_page(page);
2112 put:
2113 		put_page(page);
2114 next:
2115 		if (!xa_is_value(page) && PageTransHuge(page)) {
2116 			unsigned int nr_pages = thp_nr_pages(page);
2117 
2118 			/* Final THP may cross MAX_LFS_FILESIZE on 32-bit */
2119 			xas_set(&xas, page->index + nr_pages);
2120 			if (xas.xa_index < nr_pages)
2121 				break;
2122 		}
2123 	}
2124 	rcu_read_unlock();
2125 
2126 	return pagevec_count(pvec);
2127 }
2128 
2129 /**
2130  * find_get_pages_range - gang pagecache lookup
2131  * @mapping:	The address_space to search
2132  * @start:	The starting page index
2133  * @end:	The final page index (inclusive)
2134  * @nr_pages:	The maximum number of pages
2135  * @pages:	Where the resulting pages are placed
2136  *
2137  * find_get_pages_range() will search for and return a group of up to @nr_pages
2138  * pages in the mapping starting at index @start and up to index @end
2139  * (inclusive).  The pages are placed at @pages.  find_get_pages_range() takes
2140  * a reference against the returned pages.
2141  *
2142  * The search returns a group of mapping-contiguous pages with ascending
2143  * indexes.  There may be holes in the indices due to not-present pages.
2144  * We also update @start to index the next page for the traversal.
2145  *
2146  * Return: the number of pages which were found. If this number is
2147  * smaller than @nr_pages, the end of specified range has been
2148  * reached.
2149  */
find_get_pages_range(struct address_space * mapping,pgoff_t * start,pgoff_t end,unsigned int nr_pages,struct page ** pages)2150 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
2151 			      pgoff_t end, unsigned int nr_pages,
2152 			      struct page **pages)
2153 {
2154 	XA_STATE(xas, &mapping->i_pages, *start);
2155 	struct page *page;
2156 	unsigned ret = 0;
2157 
2158 	if (unlikely(!nr_pages))
2159 		return 0;
2160 
2161 	rcu_read_lock();
2162 	while ((page = find_get_entry(&xas, end, XA_PRESENT))) {
2163 		/* Skip over shadow, swap and DAX entries */
2164 		if (xa_is_value(page))
2165 			continue;
2166 
2167 		pages[ret] = find_subpage(page, xas.xa_index);
2168 		if (++ret == nr_pages) {
2169 			*start = xas.xa_index + 1;
2170 			goto out;
2171 		}
2172 	}
2173 
2174 	/*
2175 	 * We come here when there is no page beyond @end. We take care to not
2176 	 * overflow the index @start as it confuses some of the callers. This
2177 	 * breaks the iteration when there is a page at index -1 but that is
2178 	 * already broken anyway.
2179 	 */
2180 	if (end == (pgoff_t)-1)
2181 		*start = (pgoff_t)-1;
2182 	else
2183 		*start = end + 1;
2184 out:
2185 	rcu_read_unlock();
2186 
2187 	return ret;
2188 }
2189 
2190 /**
2191  * find_get_pages_contig - gang contiguous pagecache lookup
2192  * @mapping:	The address_space to search
2193  * @index:	The starting page index
2194  * @nr_pages:	The maximum number of pages
2195  * @pages:	Where the resulting pages are placed
2196  *
2197  * find_get_pages_contig() works exactly like find_get_pages(), except
2198  * that the returned number of pages are guaranteed to be contiguous.
2199  *
2200  * Return: the number of pages which were found.
2201  */
find_get_pages_contig(struct address_space * mapping,pgoff_t index,unsigned int nr_pages,struct page ** pages)2202 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
2203 			       unsigned int nr_pages, struct page **pages)
2204 {
2205 	XA_STATE(xas, &mapping->i_pages, index);
2206 	struct page *page;
2207 	unsigned int ret = 0;
2208 
2209 	if (unlikely(!nr_pages))
2210 		return 0;
2211 
2212 	rcu_read_lock();
2213 	for (page = xas_load(&xas); page; page = xas_next(&xas)) {
2214 		if (xas_retry(&xas, page))
2215 			continue;
2216 		/*
2217 		 * If the entry has been swapped out, we can stop looking.
2218 		 * No current caller is looking for DAX entries.
2219 		 */
2220 		if (xa_is_value(page))
2221 			break;
2222 
2223 		if (!page_cache_get_speculative(page))
2224 			goto retry;
2225 
2226 		/* Has the page moved or been split? */
2227 		if (unlikely(page != xas_reload(&xas)))
2228 			goto put_page;
2229 
2230 		pages[ret] = find_subpage(page, xas.xa_index);
2231 		if (++ret == nr_pages)
2232 			break;
2233 		continue;
2234 put_page:
2235 		put_page(page);
2236 retry:
2237 		xas_reset(&xas);
2238 	}
2239 	rcu_read_unlock();
2240 	return ret;
2241 }
2242 EXPORT_SYMBOL(find_get_pages_contig);
2243 
2244 /**
2245  * find_get_pages_range_tag - Find and return head pages matching @tag.
2246  * @mapping:	the address_space to search
2247  * @index:	the starting page index
2248  * @end:	The final page index (inclusive)
2249  * @tag:	the tag index
2250  * @nr_pages:	the maximum number of pages
2251  * @pages:	where the resulting pages are placed
2252  *
2253  * Like find_get_pages(), except we only return head pages which are tagged
2254  * with @tag.  @index is updated to the index immediately after the last
2255  * page we return, ready for the next iteration.
2256  *
2257  * Return: the number of pages which were found.
2258  */
find_get_pages_range_tag(struct address_space * mapping,pgoff_t * index,pgoff_t end,xa_mark_t tag,unsigned int nr_pages,struct page ** pages)2259 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
2260 			pgoff_t end, xa_mark_t tag, unsigned int nr_pages,
2261 			struct page **pages)
2262 {
2263 	XA_STATE(xas, &mapping->i_pages, *index);
2264 	struct page *page;
2265 	unsigned ret = 0;
2266 
2267 	if (unlikely(!nr_pages))
2268 		return 0;
2269 
2270 	rcu_read_lock();
2271 	while ((page = find_get_entry(&xas, end, tag))) {
2272 		/*
2273 		 * Shadow entries should never be tagged, but this iteration
2274 		 * is lockless so there is a window for page reclaim to evict
2275 		 * a page we saw tagged.  Skip over it.
2276 		 */
2277 		if (xa_is_value(page))
2278 			continue;
2279 
2280 		pages[ret] = page;
2281 		if (++ret == nr_pages) {
2282 			*index = page->index + thp_nr_pages(page);
2283 			goto out;
2284 		}
2285 	}
2286 
2287 	/*
2288 	 * We come here when we got to @end. We take care to not overflow the
2289 	 * index @index as it confuses some of the callers. This breaks the
2290 	 * iteration when there is a page at index -1 but that is already
2291 	 * broken anyway.
2292 	 */
2293 	if (end == (pgoff_t)-1)
2294 		*index = (pgoff_t)-1;
2295 	else
2296 		*index = end + 1;
2297 out:
2298 	rcu_read_unlock();
2299 
2300 	return ret;
2301 }
2302 EXPORT_SYMBOL(find_get_pages_range_tag);
2303 
2304 /*
2305  * CD/DVDs are error prone. When a medium error occurs, the driver may fail
2306  * a _large_ part of the i/o request. Imagine the worst scenario:
2307  *
2308  *      ---R__________________________________________B__________
2309  *         ^ reading here                             ^ bad block(assume 4k)
2310  *
2311  * read(R) => miss => readahead(R...B) => media error => frustrating retries
2312  * => failing the whole request => read(R) => read(R+1) =>
2313  * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
2314  * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
2315  * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
2316  *
2317  * It is going insane. Fix it by quickly scaling down the readahead size.
2318  */
shrink_readahead_size_eio(struct file_ra_state * ra)2319 static void shrink_readahead_size_eio(struct file_ra_state *ra)
2320 {
2321 	ra->ra_pages /= 4;
2322 }
2323 
2324 /*
2325  * filemap_get_read_batch - Get a batch of pages for read
2326  *
2327  * Get a batch of pages which represent a contiguous range of bytes
2328  * in the file.  No tail pages will be returned.  If @index is in the
2329  * middle of a THP, the entire THP will be returned.  The last page in
2330  * the batch may have Readahead set or be not Uptodate so that the
2331  * caller can take the appropriate action.
2332  */
filemap_get_read_batch(struct address_space * mapping,pgoff_t index,pgoff_t max,struct pagevec * pvec)2333 static void filemap_get_read_batch(struct address_space *mapping,
2334 		pgoff_t index, pgoff_t max, struct pagevec *pvec)
2335 {
2336 	XA_STATE(xas, &mapping->i_pages, index);
2337 	struct page *head;
2338 
2339 	rcu_read_lock();
2340 	for (head = xas_load(&xas); head; head = xas_next(&xas)) {
2341 		if (xas_retry(&xas, head))
2342 			continue;
2343 		if (xas.xa_index > max || xa_is_value(head))
2344 			break;
2345 		if (!page_cache_get_speculative(head))
2346 			goto retry;
2347 
2348 		/* Has the page moved or been split? */
2349 		if (unlikely(head != xas_reload(&xas)))
2350 			goto put_page;
2351 
2352 		if (!pagevec_add(pvec, head))
2353 			break;
2354 		if (!PageUptodate(head))
2355 			break;
2356 		if (PageReadahead(head))
2357 			break;
2358 		xas.xa_index = head->index + thp_nr_pages(head) - 1;
2359 		xas.xa_offset = (xas.xa_index >> xas.xa_shift) & XA_CHUNK_MASK;
2360 		continue;
2361 put_page:
2362 		put_page(head);
2363 retry:
2364 		xas_reset(&xas);
2365 	}
2366 	rcu_read_unlock();
2367 }
2368 
filemap_read_page(struct file * file,struct address_space * mapping,struct page * page)2369 static int filemap_read_page(struct file *file, struct address_space *mapping,
2370 		struct page *page)
2371 {
2372 	int error;
2373 
2374 	/*
2375 	 * A previous I/O error may have been due to temporary failures,
2376 	 * eg. multipath errors.  PG_error will be set again if readpage
2377 	 * fails.
2378 	 */
2379 	ClearPageError(page);
2380 	/* Start the actual read. The read will unlock the page. */
2381 	error = mapping->a_ops->readpage(file, page);
2382 	if (error)
2383 		return error;
2384 
2385 	error = wait_on_page_locked_killable(page);
2386 	if (error)
2387 		return error;
2388 	if (PageUptodate(page))
2389 		return 0;
2390 	shrink_readahead_size_eio(&file->f_ra);
2391 	return -EIO;
2392 }
2393 
filemap_range_uptodate(struct address_space * mapping,loff_t pos,struct iov_iter * iter,struct page * page)2394 static bool filemap_range_uptodate(struct address_space *mapping,
2395 		loff_t pos, struct iov_iter *iter, struct page *page)
2396 {
2397 	int count;
2398 
2399 	if (PageUptodate(page))
2400 		return true;
2401 	/* pipes can't handle partially uptodate pages */
2402 	if (iov_iter_is_pipe(iter))
2403 		return false;
2404 	if (!mapping->a_ops->is_partially_uptodate)
2405 		return false;
2406 	if (mapping->host->i_blkbits >= (PAGE_SHIFT + thp_order(page)))
2407 		return false;
2408 
2409 	count = iter->count;
2410 	if (page_offset(page) > pos) {
2411 		count -= page_offset(page) - pos;
2412 		pos = 0;
2413 	} else {
2414 		pos -= page_offset(page);
2415 	}
2416 
2417 	return mapping->a_ops->is_partially_uptodate(page, pos, count);
2418 }
2419 
filemap_update_page(struct kiocb * iocb,struct address_space * mapping,struct iov_iter * iter,struct page * page)2420 static int filemap_update_page(struct kiocb *iocb,
2421 		struct address_space *mapping, struct iov_iter *iter,
2422 		struct page *page)
2423 {
2424 	int error;
2425 
2426 	if (iocb->ki_flags & IOCB_NOWAIT) {
2427 		if (!filemap_invalidate_trylock_shared(mapping))
2428 			return -EAGAIN;
2429 	} else {
2430 		filemap_invalidate_lock_shared(mapping);
2431 	}
2432 
2433 	if (!trylock_page(page)) {
2434 		error = -EAGAIN;
2435 		if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))
2436 			goto unlock_mapping;
2437 		if (!(iocb->ki_flags & IOCB_WAITQ)) {
2438 			filemap_invalidate_unlock_shared(mapping);
2439 			put_and_wait_on_page_locked(page, TASK_KILLABLE);
2440 			return AOP_TRUNCATED_PAGE;
2441 		}
2442 		error = __lock_page_async(page, iocb->ki_waitq);
2443 		if (error)
2444 			goto unlock_mapping;
2445 	}
2446 
2447 	error = AOP_TRUNCATED_PAGE;
2448 	if (!page->mapping)
2449 		goto unlock;
2450 
2451 	error = 0;
2452 	if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, page))
2453 		goto unlock;
2454 
2455 	error = -EAGAIN;
2456 	if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ))
2457 		goto unlock;
2458 
2459 	error = filemap_read_page(iocb->ki_filp, mapping, page);
2460 	goto unlock_mapping;
2461 unlock:
2462 	unlock_page(page);
2463 unlock_mapping:
2464 	filemap_invalidate_unlock_shared(mapping);
2465 	if (error == AOP_TRUNCATED_PAGE)
2466 		put_page(page);
2467 	return error;
2468 }
2469 
filemap_create_page(struct file * file,struct address_space * mapping,pgoff_t index,struct pagevec * pvec)2470 static int filemap_create_page(struct file *file,
2471 		struct address_space *mapping, pgoff_t index,
2472 		struct pagevec *pvec)
2473 {
2474 	struct page *page;
2475 	int error;
2476 
2477 	page = page_cache_alloc(mapping);
2478 	if (!page)
2479 		return -ENOMEM;
2480 
2481 	/*
2482 	 * Protect against truncate / hole punch. Grabbing invalidate_lock here
2483 	 * assures we cannot instantiate and bring uptodate new pagecache pages
2484 	 * after evicting page cache during truncate and before actually
2485 	 * freeing blocks.  Note that we could release invalidate_lock after
2486 	 * inserting the page into page cache as the locked page would then be
2487 	 * enough to synchronize with hole punching. But there are code paths
2488 	 * such as filemap_update_page() filling in partially uptodate pages or
2489 	 * ->readpages() that need to hold invalidate_lock while mapping blocks
2490 	 * for IO so let's hold the lock here as well to keep locking rules
2491 	 * simple.
2492 	 */
2493 	filemap_invalidate_lock_shared(mapping);
2494 	error = add_to_page_cache_lru(page, mapping, index,
2495 			mapping_gfp_constraint(mapping, GFP_KERNEL));
2496 	if (error == -EEXIST)
2497 		error = AOP_TRUNCATED_PAGE;
2498 	if (error)
2499 		goto error;
2500 
2501 	error = filemap_read_page(file, mapping, page);
2502 	if (error)
2503 		goto error;
2504 
2505 	filemap_invalidate_unlock_shared(mapping);
2506 	pagevec_add(pvec, page);
2507 	return 0;
2508 error:
2509 	filemap_invalidate_unlock_shared(mapping);
2510 	put_page(page);
2511 	return error;
2512 }
2513 
filemap_readahead(struct kiocb * iocb,struct file * file,struct address_space * mapping,struct page * page,pgoff_t last_index)2514 static int filemap_readahead(struct kiocb *iocb, struct file *file,
2515 		struct address_space *mapping, struct page *page,
2516 		pgoff_t last_index)
2517 {
2518 	if (iocb->ki_flags & IOCB_NOIO)
2519 		return -EAGAIN;
2520 	page_cache_async_readahead(mapping, &file->f_ra, file, page,
2521 			page->index, last_index - page->index);
2522 	return 0;
2523 }
2524 
filemap_get_pages(struct kiocb * iocb,struct iov_iter * iter,struct pagevec * pvec)2525 static int filemap_get_pages(struct kiocb *iocb, struct iov_iter *iter,
2526 		struct pagevec *pvec)
2527 {
2528 	struct file *filp = iocb->ki_filp;
2529 	struct address_space *mapping = filp->f_mapping;
2530 	struct file_ra_state *ra = &filp->f_ra;
2531 	pgoff_t index = iocb->ki_pos >> PAGE_SHIFT;
2532 	pgoff_t last_index;
2533 	struct page *page;
2534 	int err = 0;
2535 
2536 	last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE);
2537 retry:
2538 	if (fatal_signal_pending(current))
2539 		return -EINTR;
2540 
2541 	filemap_get_read_batch(mapping, index, last_index, pvec);
2542 	if (!pagevec_count(pvec)) {
2543 		if (iocb->ki_flags & IOCB_NOIO)
2544 			return -EAGAIN;
2545 		page_cache_sync_readahead(mapping, ra, filp, index,
2546 				last_index - index);
2547 		filemap_get_read_batch(mapping, index, last_index, pvec);
2548 	}
2549 	if (!pagevec_count(pvec)) {
2550 		if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
2551 			return -EAGAIN;
2552 		err = filemap_create_page(filp, mapping,
2553 				iocb->ki_pos >> PAGE_SHIFT, pvec);
2554 		if (err == AOP_TRUNCATED_PAGE)
2555 			goto retry;
2556 		return err;
2557 	}
2558 
2559 	page = pvec->pages[pagevec_count(pvec) - 1];
2560 	if (PageReadahead(page)) {
2561 		err = filemap_readahead(iocb, filp, mapping, page, last_index);
2562 		if (err)
2563 			goto err;
2564 	}
2565 	if (!PageUptodate(page)) {
2566 		if ((iocb->ki_flags & IOCB_WAITQ) && pagevec_count(pvec) > 1)
2567 			iocb->ki_flags |= IOCB_NOWAIT;
2568 		err = filemap_update_page(iocb, mapping, iter, page);
2569 		if (err)
2570 			goto err;
2571 	}
2572 
2573 	return 0;
2574 err:
2575 	if (err < 0)
2576 		put_page(page);
2577 	if (likely(--pvec->nr))
2578 		return 0;
2579 	if (err == AOP_TRUNCATED_PAGE)
2580 		goto retry;
2581 	return err;
2582 }
2583 
2584 /**
2585  * filemap_read - Read data from the page cache.
2586  * @iocb: The iocb to read.
2587  * @iter: Destination for the data.
2588  * @already_read: Number of bytes already read by the caller.
2589  *
2590  * Copies data from the page cache.  If the data is not currently present,
2591  * uses the readahead and readpage address_space operations to fetch it.
2592  *
2593  * Return: Total number of bytes copied, including those already read by
2594  * the caller.  If an error happens before any bytes are copied, returns
2595  * a negative error number.
2596  */
filemap_read(struct kiocb * iocb,struct iov_iter * iter,ssize_t already_read)2597 ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,
2598 		ssize_t already_read)
2599 {
2600 	struct file *filp = iocb->ki_filp;
2601 	struct file_ra_state *ra = &filp->f_ra;
2602 	struct address_space *mapping = filp->f_mapping;
2603 	struct inode *inode = mapping->host;
2604 	struct pagevec pvec;
2605 	int i, error = 0;
2606 	bool writably_mapped;
2607 	loff_t isize, end_offset;
2608 
2609 	if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes))
2610 		return 0;
2611 	if (unlikely(!iov_iter_count(iter)))
2612 		return 0;
2613 
2614 	iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
2615 	pagevec_init(&pvec);
2616 
2617 	do {
2618 		cond_resched();
2619 
2620 		/*
2621 		 * If we've already successfully copied some data, then we
2622 		 * can no longer safely return -EIOCBQUEUED. Hence mark
2623 		 * an async read NOWAIT at that point.
2624 		 */
2625 		if ((iocb->ki_flags & IOCB_WAITQ) && already_read)
2626 			iocb->ki_flags |= IOCB_NOWAIT;
2627 
2628 		error = filemap_get_pages(iocb, iter, &pvec);
2629 		if (error < 0)
2630 			break;
2631 
2632 		/*
2633 		 * i_size must be checked after we know the pages are Uptodate.
2634 		 *
2635 		 * Checking i_size after the check allows us to calculate
2636 		 * the correct value for "nr", which means the zero-filled
2637 		 * part of the page is not copied back to userspace (unless
2638 		 * another truncate extends the file - this is desired though).
2639 		 */
2640 		isize = i_size_read(inode);
2641 		if (unlikely(iocb->ki_pos >= isize))
2642 			goto put_pages;
2643 		end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count);
2644 
2645 		/*
2646 		 * Once we start copying data, we don't want to be touching any
2647 		 * cachelines that might be contended:
2648 		 */
2649 		writably_mapped = mapping_writably_mapped(mapping);
2650 
2651 		/*
2652 		 * When a sequential read accesses a page several times, only
2653 		 * mark it as accessed the first time.
2654 		 */
2655 		if (iocb->ki_pos >> PAGE_SHIFT !=
2656 		    ra->prev_pos >> PAGE_SHIFT)
2657 			mark_page_accessed(pvec.pages[0]);
2658 
2659 		for (i = 0; i < pagevec_count(&pvec); i++) {
2660 			struct page *page = pvec.pages[i];
2661 			size_t page_size = thp_size(page);
2662 			size_t offset = iocb->ki_pos & (page_size - 1);
2663 			size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos,
2664 					     page_size - offset);
2665 			size_t copied;
2666 
2667 			if (end_offset < page_offset(page))
2668 				break;
2669 			if (i > 0)
2670 				mark_page_accessed(page);
2671 			/*
2672 			 * If users can be writing to this page using arbitrary
2673 			 * virtual addresses, take care about potential aliasing
2674 			 * before reading the page on the kernel side.
2675 			 */
2676 			if (writably_mapped) {
2677 				int j;
2678 
2679 				for (j = 0; j < thp_nr_pages(page); j++)
2680 					flush_dcache_page(page + j);
2681 			}
2682 
2683 			copied = copy_page_to_iter(page, offset, bytes, iter);
2684 
2685 			already_read += copied;
2686 			iocb->ki_pos += copied;
2687 			ra->prev_pos = iocb->ki_pos;
2688 
2689 			if (copied < bytes) {
2690 				error = -EFAULT;
2691 				break;
2692 			}
2693 		}
2694 put_pages:
2695 		for (i = 0; i < pagevec_count(&pvec); i++)
2696 			put_page(pvec.pages[i]);
2697 		pagevec_reinit(&pvec);
2698 	} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
2699 
2700 	file_accessed(filp);
2701 
2702 	return already_read ? already_read : error;
2703 }
2704 EXPORT_SYMBOL_GPL(filemap_read);
2705 
2706 /**
2707  * generic_file_read_iter - generic filesystem read routine
2708  * @iocb:	kernel I/O control block
2709  * @iter:	destination for the data read
2710  *
2711  * This is the "read_iter()" routine for all filesystems
2712  * that can use the page cache directly.
2713  *
2714  * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2715  * be returned when no data can be read without waiting for I/O requests
2716  * to complete; it doesn't prevent readahead.
2717  *
2718  * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2719  * requests shall be made for the read or for readahead.  When no data
2720  * can be read, -EAGAIN shall be returned.  When readahead would be
2721  * triggered, a partial, possibly empty read shall be returned.
2722  *
2723  * Return:
2724  * * number of bytes copied, even for partial reads
2725  * * negative error code (or 0 if IOCB_NOIO) if nothing was read
2726  */
2727 ssize_t
generic_file_read_iter(struct kiocb * iocb,struct iov_iter * iter)2728 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2729 {
2730 	size_t count = iov_iter_count(iter);
2731 	ssize_t retval = 0;
2732 
2733 	if (!count)
2734 		return 0; /* skip atime */
2735 
2736 	if (iocb->ki_flags & IOCB_DIRECT) {
2737 		struct file *file = iocb->ki_filp;
2738 		struct address_space *mapping = file->f_mapping;
2739 		struct inode *inode = mapping->host;
2740 		loff_t size;
2741 
2742 		size = i_size_read(inode);
2743 		if (iocb->ki_flags & IOCB_NOWAIT) {
2744 			if (filemap_range_needs_writeback(mapping, iocb->ki_pos,
2745 						iocb->ki_pos + count - 1))
2746 				return -EAGAIN;
2747 		} else {
2748 			retval = filemap_write_and_wait_range(mapping,
2749 						iocb->ki_pos,
2750 					        iocb->ki_pos + count - 1);
2751 			if (retval < 0)
2752 				return retval;
2753 		}
2754 
2755 		file_accessed(file);
2756 
2757 		retval = mapping->a_ops->direct_IO(iocb, iter);
2758 		if (retval >= 0) {
2759 			iocb->ki_pos += retval;
2760 			count -= retval;
2761 		}
2762 		if (retval != -EIOCBQUEUED)
2763 			iov_iter_revert(iter, count - iov_iter_count(iter));
2764 
2765 		/*
2766 		 * Btrfs can have a short DIO read if we encounter
2767 		 * compressed extents, so if there was an error, or if
2768 		 * we've already read everything we wanted to, or if
2769 		 * there was a short read because we hit EOF, go ahead
2770 		 * and return.  Otherwise fallthrough to buffered io for
2771 		 * the rest of the read.  Buffered reads will not work for
2772 		 * DAX files, so don't bother trying.
2773 		 */
2774 		if (retval < 0 || !count || iocb->ki_pos >= size ||
2775 		    IS_DAX(inode))
2776 			return retval;
2777 	}
2778 
2779 	return filemap_read(iocb, iter, retval);
2780 }
2781 EXPORT_SYMBOL(generic_file_read_iter);
2782 
page_seek_hole_data(struct xa_state * xas,struct address_space * mapping,struct page * page,loff_t start,loff_t end,bool seek_data)2783 static inline loff_t page_seek_hole_data(struct xa_state *xas,
2784 		struct address_space *mapping, struct page *page,
2785 		loff_t start, loff_t end, bool seek_data)
2786 {
2787 	const struct address_space_operations *ops = mapping->a_ops;
2788 	size_t offset, bsz = i_blocksize(mapping->host);
2789 
2790 	if (xa_is_value(page) || PageUptodate(page))
2791 		return seek_data ? start : end;
2792 	if (!ops->is_partially_uptodate)
2793 		return seek_data ? end : start;
2794 
2795 	xas_pause(xas);
2796 	rcu_read_unlock();
2797 	lock_page(page);
2798 	if (unlikely(page->mapping != mapping))
2799 		goto unlock;
2800 
2801 	offset = offset_in_thp(page, start) & ~(bsz - 1);
2802 
2803 	do {
2804 		if (ops->is_partially_uptodate(page, offset, bsz) == seek_data)
2805 			break;
2806 		start = (start + bsz) & ~(bsz - 1);
2807 		offset += bsz;
2808 	} while (offset < thp_size(page));
2809 unlock:
2810 	unlock_page(page);
2811 	rcu_read_lock();
2812 	return start;
2813 }
2814 
2815 static inline
seek_page_size(struct xa_state * xas,struct page * page)2816 unsigned int seek_page_size(struct xa_state *xas, struct page *page)
2817 {
2818 	if (xa_is_value(page))
2819 		return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index);
2820 	return thp_size(page);
2821 }
2822 
2823 /**
2824  * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
2825  * @mapping: Address space to search.
2826  * @start: First byte to consider.
2827  * @end: Limit of search (exclusive).
2828  * @whence: Either SEEK_HOLE or SEEK_DATA.
2829  *
2830  * If the page cache knows which blocks contain holes and which blocks
2831  * contain data, your filesystem can use this function to implement
2832  * SEEK_HOLE and SEEK_DATA.  This is useful for filesystems which are
2833  * entirely memory-based such as tmpfs, and filesystems which support
2834  * unwritten extents.
2835  *
2836  * Return: The requested offset on success, or -ENXIO if @whence specifies
2837  * SEEK_DATA and there is no data after @start.  There is an implicit hole
2838  * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
2839  * and @end contain data.
2840  */
mapping_seek_hole_data(struct address_space * mapping,loff_t start,loff_t end,int whence)2841 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
2842 		loff_t end, int whence)
2843 {
2844 	XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);
2845 	pgoff_t max = (end - 1) >> PAGE_SHIFT;
2846 	bool seek_data = (whence == SEEK_DATA);
2847 	struct page *page;
2848 
2849 	if (end <= start)
2850 		return -ENXIO;
2851 
2852 	rcu_read_lock();
2853 	while ((page = find_get_entry(&xas, max, XA_PRESENT))) {
2854 		loff_t pos = (u64)xas.xa_index << PAGE_SHIFT;
2855 		unsigned int seek_size;
2856 
2857 		if (start < pos) {
2858 			if (!seek_data)
2859 				goto unlock;
2860 			start = pos;
2861 		}
2862 
2863 		seek_size = seek_page_size(&xas, page);
2864 		pos = round_up(pos + 1, seek_size);
2865 		start = page_seek_hole_data(&xas, mapping, page, start, pos,
2866 				seek_data);
2867 		if (start < pos)
2868 			goto unlock;
2869 		if (start >= end)
2870 			break;
2871 		if (seek_size > PAGE_SIZE)
2872 			xas_set(&xas, pos >> PAGE_SHIFT);
2873 		if (!xa_is_value(page))
2874 			put_page(page);
2875 	}
2876 	if (seek_data)
2877 		start = -ENXIO;
2878 unlock:
2879 	rcu_read_unlock();
2880 	if (page && !xa_is_value(page))
2881 		put_page(page);
2882 	if (start > end)
2883 		return end;
2884 	return start;
2885 }
2886 
2887 #ifdef CONFIG_MMU
2888 #define MMAP_LOTSAMISS  (100)
2889 /*
2890  * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2891  * @vmf - the vm_fault for this fault.
2892  * @page - the page to lock.
2893  * @fpin - the pointer to the file we may pin (or is already pinned).
2894  *
2895  * This works similar to lock_page_or_retry in that it can drop the mmap_lock.
2896  * It differs in that it actually returns the page locked if it returns 1 and 0
2897  * if it couldn't lock the page.  If we did have to drop the mmap_lock then fpin
2898  * will point to the pinned file and needs to be fput()'ed at a later point.
2899  */
lock_page_maybe_drop_mmap(struct vm_fault * vmf,struct page * page,struct file ** fpin)2900 static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page,
2901 				     struct file **fpin)
2902 {
2903 	if (trylock_page(page))
2904 		return 1;
2905 
2906 	/*
2907 	 * NOTE! This will make us return with VM_FAULT_RETRY, but with
2908 	 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT
2909 	 * is supposed to work. We have way too many special cases..
2910 	 */
2911 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
2912 		return 0;
2913 
2914 	*fpin = maybe_unlock_mmap_for_io(vmf, *fpin);
2915 	if (vmf->flags & FAULT_FLAG_KILLABLE) {
2916 		if (__lock_page_killable(page)) {
2917 			/*
2918 			 * We didn't have the right flags to drop the mmap_lock,
2919 			 * but all fault_handlers only check for fatal signals
2920 			 * if we return VM_FAULT_RETRY, so we need to drop the
2921 			 * mmap_lock here and return 0 if we don't have a fpin.
2922 			 */
2923 			if (*fpin == NULL)
2924 				mmap_read_unlock(vmf->vma->vm_mm);
2925 			return 0;
2926 		}
2927 	} else
2928 		__lock_page(page);
2929 	return 1;
2930 }
2931 
2932 
2933 /*
2934  * Synchronous readahead happens when we don't even find a page in the page
2935  * cache at all.  We don't want to perform IO under the mmap sem, so if we have
2936  * to drop the mmap sem we return the file that was pinned in order for us to do
2937  * that.  If we didn't pin a file then we return NULL.  The file that is
2938  * returned needs to be fput()'ed when we're done with it.
2939  */
do_sync_mmap_readahead(struct vm_fault * vmf)2940 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
2941 {
2942 	struct file *file = vmf->vma->vm_file;
2943 	struct file_ra_state *ra = &file->f_ra;
2944 	struct address_space *mapping = file->f_mapping;
2945 	DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff);
2946 	struct file *fpin = NULL;
2947 	unsigned int mmap_miss;
2948 
2949 	/* If we don't want any read-ahead, don't bother */
2950 	if (vmf->vma->vm_flags & VM_RAND_READ)
2951 		return fpin;
2952 	if (!ra->ra_pages)
2953 		return fpin;
2954 
2955 	if (vmf->vma->vm_flags & VM_SEQ_READ) {
2956 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
2957 		page_cache_sync_ra(&ractl, ra->ra_pages);
2958 		return fpin;
2959 	}
2960 
2961 	/* Avoid banging the cache line if not needed */
2962 	mmap_miss = READ_ONCE(ra->mmap_miss);
2963 	if (mmap_miss < MMAP_LOTSAMISS * 10)
2964 		WRITE_ONCE(ra->mmap_miss, ++mmap_miss);
2965 
2966 	/*
2967 	 * Do we miss much more than hit in this file? If so,
2968 	 * stop bothering with read-ahead. It will only hurt.
2969 	 */
2970 	if (mmap_miss > MMAP_LOTSAMISS)
2971 		return fpin;
2972 
2973 	/*
2974 	 * mmap read-around
2975 	 */
2976 	fpin = maybe_unlock_mmap_for_io(vmf, fpin);
2977 	ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
2978 	ra->size = ra->ra_pages;
2979 	ra->async_size = ra->ra_pages / 4;
2980 	ractl._index = ra->start;
2981 	do_page_cache_ra(&ractl, ra->size, ra->async_size);
2982 	return fpin;
2983 }
2984 
2985 /*
2986  * Asynchronous readahead happens when we find the page and PG_readahead,
2987  * so we want to possibly extend the readahead further.  We return the file that
2988  * was pinned if we have to drop the mmap_lock in order to do IO.
2989  */
do_async_mmap_readahead(struct vm_fault * vmf,struct page * page)2990 static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
2991 					    struct page *page)
2992 {
2993 	struct file *file = vmf->vma->vm_file;
2994 	struct file_ra_state *ra = &file->f_ra;
2995 	struct address_space *mapping = file->f_mapping;
2996 	struct file *fpin = NULL;
2997 	unsigned int mmap_miss;
2998 	pgoff_t offset = vmf->pgoff;
2999 
3000 	/* If we don't want any read-ahead, don't bother */
3001 	if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
3002 		return fpin;
3003 	mmap_miss = READ_ONCE(ra->mmap_miss);
3004 	if (mmap_miss)
3005 		WRITE_ONCE(ra->mmap_miss, --mmap_miss);
3006 	if (PageReadahead(page)) {
3007 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3008 		page_cache_async_readahead(mapping, ra, file,
3009 					   page, offset, ra->ra_pages);
3010 	}
3011 	return fpin;
3012 }
3013 
3014 /**
3015  * filemap_fault - read in file data for page fault handling
3016  * @vmf:	struct vm_fault containing details of the fault
3017  *
3018  * filemap_fault() is invoked via the vma operations vector for a
3019  * mapped memory region to read in file data during a page fault.
3020  *
3021  * The goto's are kind of ugly, but this streamlines the normal case of having
3022  * it in the page cache, and handles the special cases reasonably without
3023  * having a lot of duplicated code.
3024  *
3025  * vma->vm_mm->mmap_lock must be held on entry.
3026  *
3027  * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
3028  * may be dropped before doing I/O or by lock_page_maybe_drop_mmap().
3029  *
3030  * If our return value does not have VM_FAULT_RETRY set, the mmap_lock
3031  * has not been released.
3032  *
3033  * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
3034  *
3035  * Return: bitwise-OR of %VM_FAULT_ codes.
3036  */
filemap_fault(struct vm_fault * vmf)3037 vm_fault_t filemap_fault(struct vm_fault *vmf)
3038 {
3039 	int error;
3040 	struct file *file = vmf->vma->vm_file;
3041 	struct file *fpin = NULL;
3042 	struct address_space *mapping = file->f_mapping;
3043 	struct inode *inode = mapping->host;
3044 	pgoff_t offset = vmf->pgoff;
3045 	pgoff_t max_off;
3046 	struct page *page;
3047 	vm_fault_t ret = 0;
3048 	bool mapping_locked = false;
3049 
3050 	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3051 	if (unlikely(offset >= max_off))
3052 		return VM_FAULT_SIGBUS;
3053 
3054 	/*
3055 	 * Do we have something in the page cache already?
3056 	 */
3057 	page = find_get_page(mapping, offset);
3058 	if (likely(page)) {
3059 		/*
3060 		 * We found the page, so try async readahead before waiting for
3061 		 * the lock.
3062 		 */
3063 		if (!(vmf->flags & FAULT_FLAG_TRIED))
3064 			fpin = do_async_mmap_readahead(vmf, page);
3065 		if (unlikely(!PageUptodate(page))) {
3066 			filemap_invalidate_lock_shared(mapping);
3067 			mapping_locked = true;
3068 		}
3069 	} else {
3070 		/* No page in the page cache at all */
3071 		count_vm_event(PGMAJFAULT);
3072 		count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
3073 		ret = VM_FAULT_MAJOR;
3074 		fpin = do_sync_mmap_readahead(vmf);
3075 retry_find:
3076 		/*
3077 		 * See comment in filemap_create_page() why we need
3078 		 * invalidate_lock
3079 		 */
3080 		if (!mapping_locked) {
3081 			filemap_invalidate_lock_shared(mapping);
3082 			mapping_locked = true;
3083 		}
3084 		page = pagecache_get_page(mapping, offset,
3085 					  FGP_CREAT|FGP_FOR_MMAP,
3086 					  vmf->gfp_mask);
3087 		if (!page) {
3088 			if (fpin)
3089 				goto out_retry;
3090 			filemap_invalidate_unlock_shared(mapping);
3091 			return VM_FAULT_OOM;
3092 		}
3093 	}
3094 
3095 	if (!lock_page_maybe_drop_mmap(vmf, page, &fpin))
3096 		goto out_retry;
3097 
3098 	/* Did it get truncated? */
3099 	if (unlikely(compound_head(page)->mapping != mapping)) {
3100 		unlock_page(page);
3101 		put_page(page);
3102 		goto retry_find;
3103 	}
3104 	VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
3105 
3106 	/*
3107 	 * We have a locked page in the page cache, now we need to check
3108 	 * that it's up-to-date. If not, it is going to be due to an error.
3109 	 */
3110 	if (unlikely(!PageUptodate(page))) {
3111 		/*
3112 		 * The page was in cache and uptodate and now it is not.
3113 		 * Strange but possible since we didn't hold the page lock all
3114 		 * the time. Let's drop everything get the invalidate lock and
3115 		 * try again.
3116 		 */
3117 		if (!mapping_locked) {
3118 			unlock_page(page);
3119 			put_page(page);
3120 			goto retry_find;
3121 		}
3122 		goto page_not_uptodate;
3123 	}
3124 
3125 	/*
3126 	 * We've made it this far and we had to drop our mmap_lock, now is the
3127 	 * time to return to the upper layer and have it re-find the vma and
3128 	 * redo the fault.
3129 	 */
3130 	if (fpin) {
3131 		unlock_page(page);
3132 		goto out_retry;
3133 	}
3134 	if (mapping_locked)
3135 		filemap_invalidate_unlock_shared(mapping);
3136 
3137 	/*
3138 	 * Found the page and have a reference on it.
3139 	 * We must recheck i_size under page lock.
3140 	 */
3141 	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
3142 	if (unlikely(offset >= max_off)) {
3143 		unlock_page(page);
3144 		put_page(page);
3145 		return VM_FAULT_SIGBUS;
3146 	}
3147 
3148 	vmf->page = page;
3149 	return ret | VM_FAULT_LOCKED;
3150 
3151 page_not_uptodate:
3152 	/*
3153 	 * Umm, take care of errors if the page isn't up-to-date.
3154 	 * Try to re-read it _once_. We do this synchronously,
3155 	 * because there really aren't any performance issues here
3156 	 * and we need to check for errors.
3157 	 */
3158 	fpin = maybe_unlock_mmap_for_io(vmf, fpin);
3159 	error = filemap_read_page(file, mapping, page);
3160 	if (fpin)
3161 		goto out_retry;
3162 	put_page(page);
3163 
3164 	if (!error || error == AOP_TRUNCATED_PAGE)
3165 		goto retry_find;
3166 	filemap_invalidate_unlock_shared(mapping);
3167 
3168 	return VM_FAULT_SIGBUS;
3169 
3170 out_retry:
3171 	/*
3172 	 * We dropped the mmap_lock, we need to return to the fault handler to
3173 	 * re-find the vma and come back and find our hopefully still populated
3174 	 * page.
3175 	 */
3176 	if (page)
3177 		put_page(page);
3178 	if (mapping_locked)
3179 		filemap_invalidate_unlock_shared(mapping);
3180 	if (fpin)
3181 		fput(fpin);
3182 	return ret | VM_FAULT_RETRY;
3183 }
3184 EXPORT_SYMBOL(filemap_fault);
3185 
filemap_map_pmd(struct vm_fault * vmf,struct page * page)3186 static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page)
3187 {
3188 	struct mm_struct *mm = vmf->vma->vm_mm;
3189 
3190 	/* Huge page is mapped? No need to proceed. */
3191 	if (pmd_trans_huge(*vmf->pmd)) {
3192 		unlock_page(page);
3193 		put_page(page);
3194 		return true;
3195 	}
3196 
3197 	if (pmd_none(*vmf->pmd) && PageTransHuge(page)) {
3198 	    vm_fault_t ret = do_set_pmd(vmf, page);
3199 	    if (!ret) {
3200 		    /* The page is mapped successfully, reference consumed. */
3201 		    unlock_page(page);
3202 		    return true;
3203 	    }
3204 	}
3205 
3206 	if (pmd_none(*vmf->pmd)) {
3207 		vmf->ptl = pmd_lock(mm, vmf->pmd);
3208 		if (likely(pmd_none(*vmf->pmd))) {
3209 			mm_inc_nr_ptes(mm);
3210 			pmd_populate(mm, vmf->pmd, vmf->prealloc_pte);
3211 			vmf->prealloc_pte = NULL;
3212 		}
3213 		spin_unlock(vmf->ptl);
3214 	}
3215 
3216 	/* See comment in handle_pte_fault() */
3217 	if (pmd_devmap_trans_unstable(vmf->pmd)) {
3218 		unlock_page(page);
3219 		put_page(page);
3220 		return true;
3221 	}
3222 
3223 	return false;
3224 }
3225 
next_uptodate_page(struct page * page,struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3226 static struct page *next_uptodate_page(struct page *page,
3227 				       struct address_space *mapping,
3228 				       struct xa_state *xas, pgoff_t end_pgoff)
3229 {
3230 	unsigned long max_idx;
3231 
3232 	do {
3233 		if (!page)
3234 			return NULL;
3235 		if (xas_retry(xas, page))
3236 			continue;
3237 		if (xa_is_value(page))
3238 			continue;
3239 		if (PageLocked(page))
3240 			continue;
3241 		if (!page_cache_get_speculative(page))
3242 			continue;
3243 		/* Has the page moved or been split? */
3244 		if (unlikely(page != xas_reload(xas)))
3245 			goto skip;
3246 		if (!PageUptodate(page) || PageReadahead(page))
3247 			goto skip;
3248 		if (PageHWPoison(page))
3249 			goto skip;
3250 		if (!trylock_page(page))
3251 			goto skip;
3252 		if (page->mapping != mapping)
3253 			goto unlock;
3254 		if (!PageUptodate(page))
3255 			goto unlock;
3256 		max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
3257 		if (xas->xa_index >= max_idx)
3258 			goto unlock;
3259 		return page;
3260 unlock:
3261 		unlock_page(page);
3262 skip:
3263 		put_page(page);
3264 	} while ((page = xas_next_entry(xas, end_pgoff)) != NULL);
3265 
3266 	return NULL;
3267 }
3268 
first_map_page(struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3269 static inline struct page *first_map_page(struct address_space *mapping,
3270 					  struct xa_state *xas,
3271 					  pgoff_t end_pgoff)
3272 {
3273 	return next_uptodate_page(xas_find(xas, end_pgoff),
3274 				  mapping, xas, end_pgoff);
3275 }
3276 
next_map_page(struct address_space * mapping,struct xa_state * xas,pgoff_t end_pgoff)3277 static inline struct page *next_map_page(struct address_space *mapping,
3278 					 struct xa_state *xas,
3279 					 pgoff_t end_pgoff)
3280 {
3281 	return next_uptodate_page(xas_next_entry(xas, end_pgoff),
3282 				  mapping, xas, end_pgoff);
3283 }
3284 
filemap_map_pages(struct vm_fault * vmf,pgoff_t start_pgoff,pgoff_t end_pgoff)3285 vm_fault_t filemap_map_pages(struct vm_fault *vmf,
3286 			     pgoff_t start_pgoff, pgoff_t end_pgoff)
3287 {
3288 	struct vm_area_struct *vma = vmf->vma;
3289 	struct file *file = vma->vm_file;
3290 	struct address_space *mapping = file->f_mapping;
3291 	pgoff_t last_pgoff = start_pgoff;
3292 	unsigned long addr;
3293 	XA_STATE(xas, &mapping->i_pages, start_pgoff);
3294 	struct page *head, *page;
3295 	unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss);
3296 	vm_fault_t ret = 0;
3297 
3298 	rcu_read_lock();
3299 	head = first_map_page(mapping, &xas, end_pgoff);
3300 	if (!head)
3301 		goto out;
3302 
3303 	if (filemap_map_pmd(vmf, head)) {
3304 		ret = VM_FAULT_NOPAGE;
3305 		goto out;
3306 	}
3307 
3308 	addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
3309 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
3310 	do {
3311 		page = find_subpage(head, xas.xa_index);
3312 		if (PageHWPoison(page))
3313 			goto unlock;
3314 
3315 		if (mmap_miss > 0)
3316 			mmap_miss--;
3317 
3318 		addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
3319 		vmf->pte += xas.xa_index - last_pgoff;
3320 		last_pgoff = xas.xa_index;
3321 
3322 		if (!pte_none(*vmf->pte))
3323 			goto unlock;
3324 
3325 		/* We're about to handle the fault */
3326 		if (vmf->address == addr)
3327 			ret = VM_FAULT_NOPAGE;
3328 
3329 		do_set_pte(vmf, page, addr);
3330 		/* no need to invalidate: a not-present page won't be cached */
3331 		update_mmu_cache(vma, addr, vmf->pte);
3332 		unlock_page(head);
3333 		continue;
3334 unlock:
3335 		unlock_page(head);
3336 		put_page(head);
3337 	} while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL);
3338 	pte_unmap_unlock(vmf->pte, vmf->ptl);
3339 out:
3340 	rcu_read_unlock();
3341 	WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss);
3342 	return ret;
3343 }
3344 EXPORT_SYMBOL(filemap_map_pages);
3345 
filemap_page_mkwrite(struct vm_fault * vmf)3346 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
3347 {
3348 	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
3349 	struct page *page = vmf->page;
3350 	vm_fault_t ret = VM_FAULT_LOCKED;
3351 
3352 	sb_start_pagefault(mapping->host->i_sb);
3353 	file_update_time(vmf->vma->vm_file);
3354 	lock_page(page);
3355 	if (page->mapping != mapping) {
3356 		unlock_page(page);
3357 		ret = VM_FAULT_NOPAGE;
3358 		goto out;
3359 	}
3360 	/*
3361 	 * We mark the page dirty already here so that when freeze is in
3362 	 * progress, we are guaranteed that writeback during freezing will
3363 	 * see the dirty page and writeprotect it again.
3364 	 */
3365 	set_page_dirty(page);
3366 	wait_for_stable_page(page);
3367 out:
3368 	sb_end_pagefault(mapping->host->i_sb);
3369 	return ret;
3370 }
3371 
3372 const struct vm_operations_struct generic_file_vm_ops = {
3373 	.fault		= filemap_fault,
3374 	.map_pages	= filemap_map_pages,
3375 	.page_mkwrite	= filemap_page_mkwrite,
3376 };
3377 
3378 /* This is used for a general mmap of a disk file */
3379 
generic_file_mmap(struct file * file,struct vm_area_struct * vma)3380 int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
3381 {
3382 	struct address_space *mapping = file->f_mapping;
3383 
3384 	if (!mapping->a_ops->readpage)
3385 		return -ENOEXEC;
3386 	file_accessed(file);
3387 	vma->vm_ops = &generic_file_vm_ops;
3388 	return 0;
3389 }
3390 
3391 /*
3392  * This is for filesystems which do not implement ->writepage.
3393  */
generic_file_readonly_mmap(struct file * file,struct vm_area_struct * vma)3394 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
3395 {
3396 	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
3397 		return -EINVAL;
3398 	return generic_file_mmap(file, vma);
3399 }
3400 #else
filemap_page_mkwrite(struct vm_fault * vmf)3401 vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
3402 {
3403 	return VM_FAULT_SIGBUS;
3404 }
generic_file_mmap(struct file * file,struct vm_area_struct * vma)3405 int generic_file_mmap(struct file *file, struct vm_area_struct *vma)
3406 {
3407 	return -ENOSYS;
3408 }
generic_file_readonly_mmap(struct file * file,struct vm_area_struct * vma)3409 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
3410 {
3411 	return -ENOSYS;
3412 }
3413 #endif /* CONFIG_MMU */
3414 
3415 EXPORT_SYMBOL(filemap_page_mkwrite);
3416 EXPORT_SYMBOL(generic_file_mmap);
3417 EXPORT_SYMBOL(generic_file_readonly_mmap);
3418 
wait_on_page_read(struct page * page)3419 static struct page *wait_on_page_read(struct page *page)
3420 {
3421 	if (!IS_ERR(page)) {
3422 		wait_on_page_locked(page);
3423 		if (!PageUptodate(page)) {
3424 			put_page(page);
3425 			page = ERR_PTR(-EIO);
3426 		}
3427 	}
3428 	return page;
3429 }
3430 
do_read_cache_page(struct address_space * mapping,pgoff_t index,int (* filler)(void *,struct page *),void * data,gfp_t gfp)3431 static struct page *do_read_cache_page(struct address_space *mapping,
3432 				pgoff_t index,
3433 				int (*filler)(void *, struct page *),
3434 				void *data,
3435 				gfp_t gfp)
3436 {
3437 	struct page *page;
3438 	int err;
3439 repeat:
3440 	page = find_get_page(mapping, index);
3441 	if (!page) {
3442 		page = __page_cache_alloc(gfp);
3443 		if (!page)
3444 			return ERR_PTR(-ENOMEM);
3445 		err = add_to_page_cache_lru(page, mapping, index, gfp);
3446 		if (unlikely(err)) {
3447 			put_page(page);
3448 			if (err == -EEXIST)
3449 				goto repeat;
3450 			/* Presumably ENOMEM for xarray node */
3451 			return ERR_PTR(err);
3452 		}
3453 
3454 filler:
3455 		if (filler)
3456 			err = filler(data, page);
3457 		else
3458 			err = mapping->a_ops->readpage(data, page);
3459 
3460 		if (err < 0) {
3461 			put_page(page);
3462 			return ERR_PTR(err);
3463 		}
3464 
3465 		page = wait_on_page_read(page);
3466 		if (IS_ERR(page))
3467 			return page;
3468 		goto out;
3469 	}
3470 	if (PageUptodate(page))
3471 		goto out;
3472 
3473 	/*
3474 	 * Page is not up to date and may be locked due to one of the following
3475 	 * case a: Page is being filled and the page lock is held
3476 	 * case b: Read/write error clearing the page uptodate status
3477 	 * case c: Truncation in progress (page locked)
3478 	 * case d: Reclaim in progress
3479 	 *
3480 	 * Case a, the page will be up to date when the page is unlocked.
3481 	 *    There is no need to serialise on the page lock here as the page
3482 	 *    is pinned so the lock gives no additional protection. Even if the
3483 	 *    page is truncated, the data is still valid if PageUptodate as
3484 	 *    it's a race vs truncate race.
3485 	 * Case b, the page will not be up to date
3486 	 * Case c, the page may be truncated but in itself, the data may still
3487 	 *    be valid after IO completes as it's a read vs truncate race. The
3488 	 *    operation must restart if the page is not uptodate on unlock but
3489 	 *    otherwise serialising on page lock to stabilise the mapping gives
3490 	 *    no additional guarantees to the caller as the page lock is
3491 	 *    released before return.
3492 	 * Case d, similar to truncation. If reclaim holds the page lock, it
3493 	 *    will be a race with remove_mapping that determines if the mapping
3494 	 *    is valid on unlock but otherwise the data is valid and there is
3495 	 *    no need to serialise with page lock.
3496 	 *
3497 	 * As the page lock gives no additional guarantee, we optimistically
3498 	 * wait on the page to be unlocked and check if it's up to date and
3499 	 * use the page if it is. Otherwise, the page lock is required to
3500 	 * distinguish between the different cases. The motivation is that we
3501 	 * avoid spurious serialisations and wakeups when multiple processes
3502 	 * wait on the same page for IO to complete.
3503 	 */
3504 	wait_on_page_locked(page);
3505 	if (PageUptodate(page))
3506 		goto out;
3507 
3508 	/* Distinguish between all the cases under the safety of the lock */
3509 	lock_page(page);
3510 
3511 	/* Case c or d, restart the operation */
3512 	if (!page->mapping) {
3513 		unlock_page(page);
3514 		put_page(page);
3515 		goto repeat;
3516 	}
3517 
3518 	/* Someone else locked and filled the page in a very small window */
3519 	if (PageUptodate(page)) {
3520 		unlock_page(page);
3521 		goto out;
3522 	}
3523 
3524 	/*
3525 	 * A previous I/O error may have been due to temporary
3526 	 * failures.
3527 	 * Clear page error before actual read, PG_error will be
3528 	 * set again if read page fails.
3529 	 */
3530 	ClearPageError(page);
3531 	goto filler;
3532 
3533 out:
3534 	mark_page_accessed(page);
3535 	return page;
3536 }
3537 
3538 /**
3539  * read_cache_page - read into page cache, fill it if needed
3540  * @mapping:	the page's address_space
3541  * @index:	the page index
3542  * @filler:	function to perform the read
3543  * @data:	first arg to filler(data, page) function, often left as NULL
3544  *
3545  * Read into the page cache. If a page already exists, and PageUptodate() is
3546  * not set, try to fill the page and wait for it to become unlocked.
3547  *
3548  * If the page does not get brought uptodate, return -EIO.
3549  *
3550  * The function expects mapping->invalidate_lock to be already held.
3551  *
3552  * Return: up to date page on success, ERR_PTR() on failure.
3553  */
read_cache_page(struct address_space * mapping,pgoff_t index,int (* filler)(void *,struct page *),void * data)3554 struct page *read_cache_page(struct address_space *mapping,
3555 				pgoff_t index,
3556 				int (*filler)(void *, struct page *),
3557 				void *data)
3558 {
3559 	return do_read_cache_page(mapping, index, filler, data,
3560 			mapping_gfp_mask(mapping));
3561 }
3562 EXPORT_SYMBOL(read_cache_page);
3563 
3564 /**
3565  * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3566  * @mapping:	the page's address_space
3567  * @index:	the page index
3568  * @gfp:	the page allocator flags to use if allocating
3569  *
3570  * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3571  * any new page allocations done using the specified allocation flags.
3572  *
3573  * If the page does not get brought uptodate, return -EIO.
3574  *
3575  * The function expects mapping->invalidate_lock to be already held.
3576  *
3577  * Return: up to date page on success, ERR_PTR() on failure.
3578  */
read_cache_page_gfp(struct address_space * mapping,pgoff_t index,gfp_t gfp)3579 struct page *read_cache_page_gfp(struct address_space *mapping,
3580 				pgoff_t index,
3581 				gfp_t gfp)
3582 {
3583 	return do_read_cache_page(mapping, index, NULL, NULL, gfp);
3584 }
3585 EXPORT_SYMBOL(read_cache_page_gfp);
3586 
pagecache_write_begin(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,unsigned flags,struct page ** pagep,void ** fsdata)3587 int pagecache_write_begin(struct file *file, struct address_space *mapping,
3588 				loff_t pos, unsigned len, unsigned flags,
3589 				struct page **pagep, void **fsdata)
3590 {
3591 	const struct address_space_operations *aops = mapping->a_ops;
3592 
3593 	return aops->write_begin(file, mapping, pos, len, flags,
3594 							pagep, fsdata);
3595 }
3596 EXPORT_SYMBOL(pagecache_write_begin);
3597 
pagecache_write_end(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct page * page,void * fsdata)3598 int pagecache_write_end(struct file *file, struct address_space *mapping,
3599 				loff_t pos, unsigned len, unsigned copied,
3600 				struct page *page, void *fsdata)
3601 {
3602 	const struct address_space_operations *aops = mapping->a_ops;
3603 
3604 	return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
3605 }
3606 EXPORT_SYMBOL(pagecache_write_end);
3607 
3608 /*
3609  * Warn about a page cache invalidation failure during a direct I/O write.
3610  */
dio_warn_stale_pagecache(struct file * filp)3611 void dio_warn_stale_pagecache(struct file *filp)
3612 {
3613 	static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
3614 	char pathname[128];
3615 	char *path;
3616 
3617 	errseq_set(&filp->f_mapping->wb_err, -EIO);
3618 	if (__ratelimit(&_rs)) {
3619 		path = file_path(filp, pathname, sizeof(pathname));
3620 		if (IS_ERR(path))
3621 			path = "(unknown)";
3622 		pr_crit("Page cache invalidation failure on direct I/O.  Possible data corruption due to collision with buffered I/O!\n");
3623 		pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
3624 			current->comm);
3625 	}
3626 }
3627 
3628 ssize_t
generic_file_direct_write(struct kiocb * iocb,struct iov_iter * from)3629 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
3630 {
3631 	struct file	*file = iocb->ki_filp;
3632 	struct address_space *mapping = file->f_mapping;
3633 	struct inode	*inode = mapping->host;
3634 	loff_t		pos = iocb->ki_pos;
3635 	ssize_t		written;
3636 	size_t		write_len;
3637 	pgoff_t		end;
3638 
3639 	write_len = iov_iter_count(from);
3640 	end = (pos + write_len - 1) >> PAGE_SHIFT;
3641 
3642 	if (iocb->ki_flags & IOCB_NOWAIT) {
3643 		/* If there are pages to writeback, return */
3644 		if (filemap_range_has_page(file->f_mapping, pos,
3645 					   pos + write_len - 1))
3646 			return -EAGAIN;
3647 	} else {
3648 		written = filemap_write_and_wait_range(mapping, pos,
3649 							pos + write_len - 1);
3650 		if (written)
3651 			goto out;
3652 	}
3653 
3654 	/*
3655 	 * After a write we want buffered reads to be sure to go to disk to get
3656 	 * the new data.  We invalidate clean cached page from the region we're
3657 	 * about to write.  We do this *before* the write so that we can return
3658 	 * without clobbering -EIOCBQUEUED from ->direct_IO().
3659 	 */
3660 	written = invalidate_inode_pages2_range(mapping,
3661 					pos >> PAGE_SHIFT, end);
3662 	/*
3663 	 * If a page can not be invalidated, return 0 to fall back
3664 	 * to buffered write.
3665 	 */
3666 	if (written) {
3667 		if (written == -EBUSY)
3668 			return 0;
3669 		goto out;
3670 	}
3671 
3672 	written = mapping->a_ops->direct_IO(iocb, from);
3673 
3674 	/*
3675 	 * Finally, try again to invalidate clean pages which might have been
3676 	 * cached by non-direct readahead, or faulted in by get_user_pages()
3677 	 * if the source of the write was an mmap'ed region of the file
3678 	 * we're writing.  Either one is a pretty crazy thing to do,
3679 	 * so we don't support it 100%.  If this invalidation
3680 	 * fails, tough, the write still worked...
3681 	 *
3682 	 * Most of the time we do not need this since dio_complete() will do
3683 	 * the invalidation for us. However there are some file systems that
3684 	 * do not end up with dio_complete() being called, so let's not break
3685 	 * them by removing it completely.
3686 	 *
3687 	 * Noticeable example is a blkdev_direct_IO().
3688 	 *
3689 	 * Skip invalidation for async writes or if mapping has no pages.
3690 	 */
3691 	if (written > 0 && mapping->nrpages &&
3692 	    invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))
3693 		dio_warn_stale_pagecache(file);
3694 
3695 	if (written > 0) {
3696 		pos += written;
3697 		write_len -= written;
3698 		if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
3699 			i_size_write(inode, pos);
3700 			mark_inode_dirty(inode);
3701 		}
3702 		iocb->ki_pos = pos;
3703 	}
3704 	if (written != -EIOCBQUEUED)
3705 		iov_iter_revert(from, write_len - iov_iter_count(from));
3706 out:
3707 	return written;
3708 }
3709 EXPORT_SYMBOL(generic_file_direct_write);
3710 
3711 /*
3712  * Find or create a page at the given pagecache position. Return the locked
3713  * page. This function is specifically for buffered writes.
3714  */
grab_cache_page_write_begin(struct address_space * mapping,pgoff_t index,unsigned flags)3715 struct page *grab_cache_page_write_begin(struct address_space *mapping,
3716 					pgoff_t index, unsigned flags)
3717 {
3718 	struct page *page;
3719 	int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
3720 
3721 	if (flags & AOP_FLAG_NOFS)
3722 		fgp_flags |= FGP_NOFS;
3723 
3724 	page = pagecache_get_page(mapping, index, fgp_flags,
3725 			mapping_gfp_mask(mapping));
3726 	if (page)
3727 		wait_for_stable_page(page);
3728 
3729 	return page;
3730 }
3731 EXPORT_SYMBOL(grab_cache_page_write_begin);
3732 
generic_perform_write(struct file * file,struct iov_iter * i,loff_t pos)3733 ssize_t generic_perform_write(struct file *file,
3734 				struct iov_iter *i, loff_t pos)
3735 {
3736 	struct address_space *mapping = file->f_mapping;
3737 	const struct address_space_operations *a_ops = mapping->a_ops;
3738 	long status = 0;
3739 	ssize_t written = 0;
3740 	unsigned int flags = 0;
3741 
3742 	do {
3743 		struct page *page;
3744 		unsigned long offset;	/* Offset into pagecache page */
3745 		unsigned long bytes;	/* Bytes to write to page */
3746 		size_t copied;		/* Bytes copied from user */
3747 		void *fsdata;
3748 
3749 		offset = (pos & (PAGE_SIZE - 1));
3750 		bytes = min_t(unsigned long, PAGE_SIZE - offset,
3751 						iov_iter_count(i));
3752 
3753 again:
3754 		/*
3755 		 * Bring in the user page that we will copy from _first_.
3756 		 * Otherwise there's a nasty deadlock on copying from the
3757 		 * same page as we're writing to, without it being marked
3758 		 * up-to-date.
3759 		 */
3760 		if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
3761 			status = -EFAULT;
3762 			break;
3763 		}
3764 
3765 		if (fatal_signal_pending(current)) {
3766 			status = -EINTR;
3767 			break;
3768 		}
3769 
3770 		status = a_ops->write_begin(file, mapping, pos, bytes, flags,
3771 						&page, &fsdata);
3772 		if (unlikely(status < 0))
3773 			break;
3774 
3775 		if (mapping_writably_mapped(mapping))
3776 			flush_dcache_page(page);
3777 
3778 		copied = copy_page_from_iter_atomic(page, offset, bytes, i);
3779 		flush_dcache_page(page);
3780 
3781 		status = a_ops->write_end(file, mapping, pos, bytes, copied,
3782 						page, fsdata);
3783 		if (unlikely(status != copied)) {
3784 			iov_iter_revert(i, copied - max(status, 0L));
3785 			if (unlikely(status < 0))
3786 				break;
3787 		}
3788 		cond_resched();
3789 
3790 		if (unlikely(status == 0)) {
3791 			/*
3792 			 * A short copy made ->write_end() reject the
3793 			 * thing entirely.  Might be memory poisoning
3794 			 * halfway through, might be a race with munmap,
3795 			 * might be severe memory pressure.
3796 			 */
3797 			if (copied)
3798 				bytes = copied;
3799 			goto again;
3800 		}
3801 		pos += status;
3802 		written += status;
3803 
3804 		balance_dirty_pages_ratelimited(mapping);
3805 	} while (iov_iter_count(i));
3806 
3807 	return written ? written : status;
3808 }
3809 EXPORT_SYMBOL(generic_perform_write);
3810 
3811 /**
3812  * __generic_file_write_iter - write data to a file
3813  * @iocb:	IO state structure (file, offset, etc.)
3814  * @from:	iov_iter with data to write
3815  *
3816  * This function does all the work needed for actually writing data to a
3817  * file. It does all basic checks, removes SUID from the file, updates
3818  * modification times and calls proper subroutines depending on whether we
3819  * do direct IO or a standard buffered write.
3820  *
3821  * It expects i_rwsem to be grabbed unless we work on a block device or similar
3822  * object which does not need locking at all.
3823  *
3824  * This function does *not* take care of syncing data in case of O_SYNC write.
3825  * A caller has to handle it. This is mainly due to the fact that we want to
3826  * avoid syncing under i_rwsem.
3827  *
3828  * Return:
3829  * * number of bytes written, even for truncated writes
3830  * * negative error code if no data has been written at all
3831  */
__generic_file_write_iter(struct kiocb * iocb,struct iov_iter * from)3832 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3833 {
3834 	struct file *file = iocb->ki_filp;
3835 	struct address_space *mapping = file->f_mapping;
3836 	struct inode 	*inode = mapping->host;
3837 	ssize_t		written = 0;
3838 	ssize_t		err;
3839 	ssize_t		status;
3840 
3841 	/* We can write back this queue in page reclaim */
3842 	current->backing_dev_info = inode_to_bdi(inode);
3843 	err = file_remove_privs(file);
3844 	if (err)
3845 		goto out;
3846 
3847 	err = file_update_time(file);
3848 	if (err)
3849 		goto out;
3850 
3851 	if (iocb->ki_flags & IOCB_DIRECT) {
3852 		loff_t pos, endbyte;
3853 
3854 		written = generic_file_direct_write(iocb, from);
3855 		/*
3856 		 * If the write stopped short of completing, fall back to
3857 		 * buffered writes.  Some filesystems do this for writes to
3858 		 * holes, for example.  For DAX files, a buffered write will
3859 		 * not succeed (even if it did, DAX does not handle dirty
3860 		 * page-cache pages correctly).
3861 		 */
3862 		if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
3863 			goto out;
3864 
3865 		status = generic_perform_write(file, from, pos = iocb->ki_pos);
3866 		/*
3867 		 * If generic_perform_write() returned a synchronous error
3868 		 * then we want to return the number of bytes which were
3869 		 * direct-written, or the error code if that was zero.  Note
3870 		 * that this differs from normal direct-io semantics, which
3871 		 * will return -EFOO even if some bytes were written.
3872 		 */
3873 		if (unlikely(status < 0)) {
3874 			err = status;
3875 			goto out;
3876 		}
3877 		/*
3878 		 * We need to ensure that the page cache pages are written to
3879 		 * disk and invalidated to preserve the expected O_DIRECT
3880 		 * semantics.
3881 		 */
3882 		endbyte = pos + status - 1;
3883 		err = filemap_write_and_wait_range(mapping, pos, endbyte);
3884 		if (err == 0) {
3885 			iocb->ki_pos = endbyte + 1;
3886 			written += status;
3887 			invalidate_mapping_pages(mapping,
3888 						 pos >> PAGE_SHIFT,
3889 						 endbyte >> PAGE_SHIFT);
3890 		} else {
3891 			/*
3892 			 * We don't know how much we wrote, so just return
3893 			 * the number of bytes which were direct-written
3894 			 */
3895 		}
3896 	} else {
3897 		written = generic_perform_write(file, from, iocb->ki_pos);
3898 		if (likely(written > 0))
3899 			iocb->ki_pos += written;
3900 	}
3901 out:
3902 	current->backing_dev_info = NULL;
3903 	return written ? written : err;
3904 }
3905 EXPORT_SYMBOL(__generic_file_write_iter);
3906 
3907 /**
3908  * generic_file_write_iter - write data to a file
3909  * @iocb:	IO state structure
3910  * @from:	iov_iter with data to write
3911  *
3912  * This is a wrapper around __generic_file_write_iter() to be used by most
3913  * filesystems. It takes care of syncing the file in case of O_SYNC file
3914  * and acquires i_rwsem as needed.
3915  * Return:
3916  * * negative error code if no data has been written at all of
3917  *   vfs_fsync_range() failed for a synchronous write
3918  * * number of bytes written, even for truncated writes
3919  */
generic_file_write_iter(struct kiocb * iocb,struct iov_iter * from)3920 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3921 {
3922 	struct file *file = iocb->ki_filp;
3923 	struct inode *inode = file->f_mapping->host;
3924 	ssize_t ret;
3925 
3926 	inode_lock(inode);
3927 	ret = generic_write_checks(iocb, from);
3928 	if (ret > 0)
3929 		ret = __generic_file_write_iter(iocb, from);
3930 	inode_unlock(inode);
3931 
3932 	if (ret > 0)
3933 		ret = generic_write_sync(iocb, ret);
3934 	return ret;
3935 }
3936 EXPORT_SYMBOL(generic_file_write_iter);
3937 
3938 /**
3939  * try_to_release_page() - release old fs-specific metadata on a page
3940  *
3941  * @page: the page which the kernel is trying to free
3942  * @gfp_mask: memory allocation flags (and I/O mode)
3943  *
3944  * The address_space is to try to release any data against the page
3945  * (presumably at page->private).
3946  *
3947  * This may also be called if PG_fscache is set on a page, indicating that the
3948  * page is known to the local caching routines.
3949  *
3950  * The @gfp_mask argument specifies whether I/O may be performed to release
3951  * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
3952  *
3953  * Return: %1 if the release was successful, otherwise return zero.
3954  */
try_to_release_page(struct page * page,gfp_t gfp_mask)3955 int try_to_release_page(struct page *page, gfp_t gfp_mask)
3956 {
3957 	struct address_space * const mapping = page->mapping;
3958 
3959 	BUG_ON(!PageLocked(page));
3960 	if (PageWriteback(page))
3961 		return 0;
3962 
3963 	if (mapping && mapping->a_ops->releasepage)
3964 		return mapping->a_ops->releasepage(page, gfp_mask);
3965 	return try_to_free_buffers(page);
3966 }
3967 
3968 EXPORT_SYMBOL(try_to_release_page);
3969