Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
29 #include <linux/error-injection.h>
32 #include <linux/backing-dev.h>
65 * finished 'unifying' the page and buffer cache and SMP-threaded the
66 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
68 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
74 * ->i_mmap_rwsem (truncate_pagecache)
75 * ->private_lock (__free_pte->block_dirty_folio)
76 * ->swap_lock (exclusive_swap_page, others)
77 * ->i_pages lock
79 * ->i_rwsem
80 * ->invalidate_lock (acquired by fs in truncate path)
81 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
83 * ->mmap_lock
84 * ->i_mmap_rwsem
85 * ->page_table_lock or pte_lock (various, mainly in memory.c)
86 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
88 * ->mmap_lock
89 * ->invalidate_lock (filemap_fault)
90 * ->lock_page (filemap_fault, access_process_vm)
92 * ->i_rwsem (generic_perform_write)
93 * ->mmap_lock (fault_in_readable->do_page_fault)
95 * bdi->wb.list_lock
96 * sb_lock (fs/fs-writeback.c)
97 * ->i_pages lock (__sync_single_inode)
99 * ->i_mmap_rwsem
100 * ->anon_vma.lock (vma_adjust)
102 * ->anon_vma.lock
103 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
105 * ->page_table_lock or pte_lock
106 * ->swap_lock (try_to_unmap_one)
107 * ->private_lock (try_to_unmap_one)
108 * ->i_pages lock (try_to_unmap_one)
109 * ->lruvec->lru_lock (follow_page->mark_page_accessed)
110 * ->lruvec->lru_lock (check_pte_range->isolate_lru_page)
111 * ->private_lock (page_remove_rmap->set_page_dirty)
112 * ->i_pages lock (page_remove_rmap->set_page_dirty)
113 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
114 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
115 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
116 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
117 * ->inode->i_lock (zap_pte_range->set_page_dirty)
118 * ->private_lock (zap_pte_range->block_dirty_folio)
120 * ->i_mmap_rwsem
121 * ->tasklist_lock (memory_failure, collect_procs_ao)
124 static void page_cache_delete(struct address_space *mapping, in page_cache_delete() argument
127 XA_STATE(xas, &mapping->i_pages, folio->index); in page_cache_delete()
130 mapping_set_update(&xas, mapping); in page_cache_delete()
134 xas_set_order(&xas, folio->index, folio_order(folio)); in page_cache_delete()
143 folio->mapping = NULL; in page_cache_delete()
144 /* Leave page->index set: truncation lookup relies upon it */ in page_cache_delete()
145 mapping->nrpages -= nr; in page_cache_delete()
148 static void filemap_unaccount_folio(struct address_space *mapping, in filemap_unaccount_folio() argument
156 current->comm, folio_pfn(folio)); in filemap_unaccount_folio()
157 dump_page(&folio->page, "still mapped when deleted"); in filemap_unaccount_folio()
161 if (mapping_exiting(mapping) && !folio_test_large(folio)) { in filemap_unaccount_folio()
162 int mapcount = page_mapcount(&folio->page); in filemap_unaccount_folio()
171 page_mapcount_reset(&folio->page); in filemap_unaccount_folio()
183 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); in filemap_unaccount_folio()
185 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); in filemap_unaccount_folio()
187 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); in filemap_unaccount_folio()
189 __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); in filemap_unaccount_folio()
190 filemap_nr_thps_dec(mapping); in filemap_unaccount_folio()
196 * unwritten data - on ordinary filesystems. in filemap_unaccount_folio()
198 * But it's harmless on in-memory filesystems like tmpfs; and can in filemap_unaccount_folio()
208 mapping_can_writeback(mapping))) in filemap_unaccount_folio()
209 folio_account_cleaned(folio, inode_to_wb(mapping->host)); in filemap_unaccount_folio()
214 * sure the page is locked and that nobody else uses it - or that usage
219 struct address_space *mapping = folio->mapping; in __filemap_remove_folio() local
222 filemap_unaccount_folio(mapping, folio); in __filemap_remove_folio()
223 page_cache_delete(mapping, folio, shadow); in __filemap_remove_folio()
226 void filemap_free_folio(struct address_space *mapping, struct folio *folio) in filemap_free_folio() argument
231 free_folio = mapping->a_ops->free_folio; in filemap_free_folio()
241 * filemap_remove_folio - Remove folio from page cache.
250 struct address_space *mapping = folio->mapping; in filemap_remove_folio() local
253 spin_lock(&mapping->host->i_lock); in filemap_remove_folio()
254 xa_lock_irq(&mapping->i_pages); in filemap_remove_folio()
256 xa_unlock_irq(&mapping->i_pages); in filemap_remove_folio()
257 if (mapping_shrinkable(mapping)) in filemap_remove_folio()
258 inode_add_lru(mapping->host); in filemap_remove_folio()
259 spin_unlock(&mapping->host->i_lock); in filemap_remove_folio()
261 filemap_free_folio(mapping, folio); in filemap_remove_folio()
265 * page_cache_delete_batch - delete several folios from page cache
266 * @mapping: the mapping to which folios belong
269 * The function walks over mapping->i_pages and removes folios passed in
270 * @fbatch from the mapping. The function expects @fbatch to be sorted
272 * It tolerates holes in @fbatch (mapping entries at those indices are not
277 static void page_cache_delete_batch(struct address_space *mapping, in page_cache_delete_batch() argument
280 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); in page_cache_delete_batch()
285 mapping_set_update(&xas, mapping); in page_cache_delete_batch()
300 if (folio != fbatch->folios[i]) { in page_cache_delete_batch()
301 VM_BUG_ON_FOLIO(folio->index > in page_cache_delete_batch()
302 fbatch->folios[i]->index, folio); in page_cache_delete_batch()
308 folio->mapping = NULL; in page_cache_delete_batch()
309 /* Leave folio->index set: truncation lookup relies on it */ in page_cache_delete_batch()
315 mapping->nrpages -= total_pages; in page_cache_delete_batch()
318 void delete_from_page_cache_batch(struct address_space *mapping, in delete_from_page_cache_batch() argument
326 spin_lock(&mapping->host->i_lock); in delete_from_page_cache_batch()
327 xa_lock_irq(&mapping->i_pages); in delete_from_page_cache_batch()
329 struct folio *folio = fbatch->folios[i]; in delete_from_page_cache_batch()
332 filemap_unaccount_folio(mapping, folio); in delete_from_page_cache_batch()
334 page_cache_delete_batch(mapping, fbatch); in delete_from_page_cache_batch()
335 xa_unlock_irq(&mapping->i_pages); in delete_from_page_cache_batch()
336 if (mapping_shrinkable(mapping)) in delete_from_page_cache_batch()
337 inode_add_lru(mapping->host); in delete_from_page_cache_batch()
338 spin_unlock(&mapping->host->i_lock); in delete_from_page_cache_batch()
341 filemap_free_folio(mapping, fbatch->folios[i]); in delete_from_page_cache_batch()
344 int filemap_check_errors(struct address_space *mapping) in filemap_check_errors() argument
348 if (test_bit(AS_ENOSPC, &mapping->flags) && in filemap_check_errors()
349 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_errors()
350 ret = -ENOSPC; in filemap_check_errors()
351 if (test_bit(AS_EIO, &mapping->flags) && in filemap_check_errors()
352 test_and_clear_bit(AS_EIO, &mapping->flags)) in filemap_check_errors()
353 ret = -EIO; in filemap_check_errors()
358 static int filemap_check_and_keep_errors(struct address_space *mapping) in filemap_check_and_keep_errors() argument
361 if (test_bit(AS_EIO, &mapping->flags)) in filemap_check_and_keep_errors()
362 return -EIO; in filemap_check_and_keep_errors()
363 if (test_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_and_keep_errors()
364 return -ENOSPC; in filemap_check_and_keep_errors()
369 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
370 * @mapping: address space structure to write
373 * Call writepages on the mapping using the provided wbc to control the
378 int filemap_fdatawrite_wbc(struct address_space *mapping, in filemap_fdatawrite_wbc() argument
383 if (!mapping_can_writeback(mapping) || in filemap_fdatawrite_wbc()
384 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) in filemap_fdatawrite_wbc()
387 wbc_attach_fdatawrite_inode(wbc, mapping->host); in filemap_fdatawrite_wbc()
388 ret = do_writepages(mapping, wbc); in filemap_fdatawrite_wbc()
395 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
396 * @mapping: address space structure to write
401 * Start writeback against all of a mapping's dirty pages that lie
404 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
411 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in __filemap_fdatawrite_range() argument
421 return filemap_fdatawrite_wbc(mapping, &wbc); in __filemap_fdatawrite_range()
424 static inline int __filemap_fdatawrite(struct address_space *mapping, in __filemap_fdatawrite() argument
427 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); in __filemap_fdatawrite()
430 int filemap_fdatawrite(struct address_space *mapping) in filemap_fdatawrite() argument
432 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); in filemap_fdatawrite()
436 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in filemap_fdatawrite_range() argument
439 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); in filemap_fdatawrite_range()
444 * filemap_flush - mostly a non-blocking flush
445 * @mapping: target address_space
447 * This is a mostly non-blocking flush. Not suitable for data-integrity
448 * purposes - I/O may not be started against all dirty pages.
452 int filemap_flush(struct address_space *mapping) in filemap_flush() argument
454 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); in filemap_flush()
459 * filemap_range_has_page - check if a page exists in range.
460 * @mapping: address space within which to check
470 bool filemap_range_has_page(struct address_space *mapping, in filemap_range_has_page() argument
474 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_page()
501 static void __filemap_fdatawait_range(struct address_space *mapping, in __filemap_fdatawait_range() argument
516 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, in __filemap_fdatawait_range()
533 * filemap_fdatawait_range - wait for writeback to complete
534 * @mapping: address space structure to wait for
538 * Walk the list of under-writeback pages of the given address space
548 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, in filemap_fdatawait_range() argument
551 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range()
552 return filemap_check_errors(mapping); in filemap_fdatawait_range()
557 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
558 * @mapping: address space structure to wait for
562 * Walk the list of under-writeback pages of the given address space in the
567 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
570 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, in filemap_fdatawait_range_keep_errors() argument
573 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range_keep_errors()
574 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_range_keep_errors()
579 * file_fdatawait_range - wait for writeback to complete
584 * Walk the list of under-writeback pages of the address space that file
586 * status of the address space vs. the file->f_wb_err cursor and return it.
592 * Return: error status of the address space vs. the file->f_wb_err cursor.
596 struct address_space *mapping = file->f_mapping; in file_fdatawait_range() local
598 __filemap_fdatawait_range(mapping, start_byte, end_byte); in file_fdatawait_range()
604 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
605 * @mapping: address space structure to wait for
607 * Walk the list of under-writeback pages of the given address space
612 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
617 int filemap_fdatawait_keep_errors(struct address_space *mapping) in filemap_fdatawait_keep_errors() argument
619 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); in filemap_fdatawait_keep_errors()
620 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_keep_errors()
625 static bool mapping_needs_writeback(struct address_space *mapping) in mapping_needs_writeback() argument
627 return mapping->nrpages; in mapping_needs_writeback()
630 bool filemap_range_has_writeback(struct address_space *mapping, in filemap_range_has_writeback() argument
633 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_writeback()
656 * filemap_write_and_wait_range - write out & wait on a file range
657 * @mapping: the address_space for the pages
661 * Write out and wait upon file offsets lstart->lend, inclusive.
664 * that this function can be used to write to the very end-of-file (end = -1).
668 int filemap_write_and_wait_range(struct address_space *mapping, in filemap_write_and_wait_range() argument
673 if (mapping_needs_writeback(mapping)) { in filemap_write_and_wait_range()
674 err = __filemap_fdatawrite_range(mapping, lstart, lend, in filemap_write_and_wait_range()
678 * written partially (e.g. -ENOSPC), so we wait for it. in filemap_write_and_wait_range()
679 * But the -EIO is special case, it may indicate the worst in filemap_write_and_wait_range()
682 if (err != -EIO) in filemap_write_and_wait_range()
683 __filemap_fdatawait_range(mapping, lstart, lend); in filemap_write_and_wait_range()
685 err2 = filemap_check_errors(mapping); in filemap_write_and_wait_range()
692 void __filemap_set_wb_err(struct address_space *mapping, int err) in __filemap_set_wb_err() argument
694 errseq_t eseq = errseq_set(&mapping->wb_err, err); in __filemap_set_wb_err()
696 trace_filemap_set_wb_err(mapping, eseq); in __filemap_set_wb_err()
701 * file_check_and_advance_wb_err - report wb error (if any) that was previously
709 * Grab the wb_err from the mapping. If it matches what we have in the file,
712 * If it doesn't match, then take the mapping value, set the "seen" flag in
718 * While we handle mapping->wb_err with atomic operations, the f_wb_err
727 errseq_t old = READ_ONCE(file->f_wb_err); in file_check_and_advance_wb_err()
728 struct address_space *mapping = file->f_mapping; in file_check_and_advance_wb_err() local
731 if (errseq_check(&mapping->wb_err, old)) { in file_check_and_advance_wb_err()
733 spin_lock(&file->f_lock); in file_check_and_advance_wb_err()
734 old = file->f_wb_err; in file_check_and_advance_wb_err()
735 err = errseq_check_and_advance(&mapping->wb_err, in file_check_and_advance_wb_err()
736 &file->f_wb_err); in file_check_and_advance_wb_err()
738 spin_unlock(&file->f_lock); in file_check_and_advance_wb_err()
746 clear_bit(AS_EIO, &mapping->flags); in file_check_and_advance_wb_err()
747 clear_bit(AS_ENOSPC, &mapping->flags); in file_check_and_advance_wb_err()
753 * file_write_and_wait_range - write out & wait on a file range
758 * Write out and wait upon file offsets lstart->lend, inclusive.
761 * that this function can be used to write to the very end-of-file (end = -1).
763 * After writing out and waiting on the data, we check and advance the
771 struct address_space *mapping = file->f_mapping; in file_write_and_wait_range() local
773 if (mapping_needs_writeback(mapping)) { in file_write_and_wait_range()
774 err = __filemap_fdatawrite_range(mapping, lstart, lend, in file_write_and_wait_range()
777 if (err != -EIO) in file_write_and_wait_range()
778 __filemap_fdatawait_range(mapping, lstart, lend); in file_write_and_wait_range()
788 * replace_page_cache_page - replace a pagecache page with a new one
804 struct address_space *mapping = old->mapping; in replace_page_cache_page() local
805 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; in replace_page_cache_page()
806 pgoff_t offset = old->index; in replace_page_cache_page()
807 XA_STATE(xas, &mapping->i_pages, offset); in replace_page_cache_page()
811 VM_BUG_ON_PAGE(new->mapping, new); in replace_page_cache_page()
814 new->mapping = mapping; in replace_page_cache_page()
815 new->index = offset; in replace_page_cache_page()
822 old->mapping = NULL; in replace_page_cache_page()
839 noinline int __filemap_add_folio(struct address_space *mapping, in __filemap_add_folio() argument
842 XA_STATE(xas, &mapping->i_pages, index); in __filemap_add_folio()
849 mapping_set_update(&xas, mapping); in __filemap_add_folio()
853 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); in __filemap_add_folio()
863 folio->mapping = mapping; in __filemap_add_folio()
864 folio->index = xas.xa_index; in __filemap_add_folio()
877 xas_set_err(&xas, -EEXIST); in __filemap_add_folio()
889 BUG_ON(shmem_mapping(mapping)); in __filemap_add_folio()
899 mapping->nrpages += nr; in __filemap_add_folio()
920 folio->mapping = NULL; in __filemap_add_folio()
921 /* Leave page->index set: truncation relies upon it */ in __filemap_add_folio()
927 int filemap_add_folio(struct address_space *mapping, struct folio *folio, in filemap_add_folio() argument
934 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); in filemap_add_folio()
943 * data from the working set, only to cache data that will in filemap_add_folio()
977 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
979 * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
981 * @mapping1: the first mapping to lock
982 * @mapping2: the second mapping to lock
990 down_write(&mapping1->invalidate_lock); in filemap_invalidate_lock_two()
992 down_write_nested(&mapping2->invalidate_lock, 1); in filemap_invalidate_lock_two()
997 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
999 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1001 * @mapping1: the first mapping to unlock
1002 * @mapping2: the second mapping to unlock
1008 up_write(&mapping1->invalidate_lock); in filemap_invalidate_unlock_two()
1010 up_write(&mapping2->invalidate_lock); in filemap_invalidate_unlock_two()
1044 * The page wait code treats the "wait->flags" somewhat unusually, because
1091 flags = wait->flags; in wake_page_function()
1093 if (test_bit(key->bit_nr, &key->folio->flags)) in wake_page_function()
1094 return -1; in wake_page_function()
1096 if (test_and_set_bit(key->bit_nr, &key->folio->flags)) in wake_page_function()
1097 return -1; in wake_page_function()
1103 * We are holding the wait-queue lock, but the waiter that in wake_page_function()
1108 * afterwards to avoid any races. This store-release pairs in wake_page_function()
1109 * with the load-acquire in folio_wait_bit_common(). in wake_page_function()
1111 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); in wake_page_function()
1112 wake_up_state(wait->private, mode); in wake_page_function()
1120 * After this list_del_init(&wait->entry) the wait entry in wake_page_function()
1121 * might be de-allocated and the process might even have in wake_page_function()
1124 list_del_init_careful(&wait->entry); in wake_page_function()
1144 spin_lock_irqsave(&q->lock, flags); in folio_wake_bit()
1154 spin_unlock_irqrestore(&q->lock, flags); in folio_wake_bit()
1156 spin_lock_irqsave(&q->lock, flags); in folio_wake_bit()
1172 spin_unlock_irqrestore(&q->lock, flags); in folio_wake_bit()
1204 if (wait->flags & WQ_FLAG_EXCLUSIVE) { in folio_trylock_flag()
1205 if (test_and_set_bit(bit_nr, &folio->flags)) in folio_trylock_flag()
1207 } else if (test_bit(bit_nr, &folio->flags)) in folio_trylock_flag()
1210 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; in folio_trylock_flag()
1236 wait->func = wake_page_function; in folio_wait_bit_common()
1241 wait->flags = 0; in folio_wait_bit_common()
1243 wait->flags = WQ_FLAG_EXCLUSIVE; in folio_wait_bit_common()
1244 if (--unfairness < 0) in folio_wait_bit_common()
1245 wait->flags |= WQ_FLAG_CUSTOM; in folio_wait_bit_common()
1262 spin_lock_irq(&q->lock); in folio_wait_bit_common()
1266 spin_unlock_irq(&q->lock); in folio_wait_bit_common()
1282 * be very careful with the 'wait->flags', because in folio_wait_bit_common()
1291 flags = smp_load_acquire(&wait->flags); in folio_wait_bit_common()
1300 /* If we were non-exclusive, we're done */ in folio_wait_bit_common()
1317 wait->flags |= WQ_FLAG_DONE; in folio_wait_bit_common()
1323 * waiter from the wait-queues, but the folio waiters bit will remain in folio_wait_bit_common()
1335 * NOTE! The wait->flags weren't stable until we've done the in folio_wait_bit_common()
1344 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive in folio_wait_bit_common()
1348 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; in folio_wait_bit_common()
1350 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; in folio_wait_bit_common()
1355 * migration_entry_wait_on_locked - Wait for a migration entry to be removed
1391 wait->func = wake_page_function; in migration_entry_wait_on_locked()
1394 wait->flags = 0; in migration_entry_wait_on_locked()
1396 spin_lock_irq(&q->lock); in migration_entry_wait_on_locked()
1400 spin_unlock_irq(&q->lock); in migration_entry_wait_on_locked()
1418 flags = smp_load_acquire(&wait->flags); in migration_entry_wait_on_locked()
1451 * folio_put_wait_locked - Drop a reference and wait for it to be unlocked
1461 * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal.
1469 * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue
1480 spin_lock_irqsave(&q->lock, flags); in folio_add_wait_queue()
1483 spin_unlock_irqrestore(&q->lock, flags); in folio_add_wait_queue()
1511 * folio_unlock - Unlock a locked folio.
1531 * folio_end_private_2 - Clear PG_private_2 and wake any waiters.
1551 * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio.
1564 * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio.
1571 * - 0 if successful.
1572 * - -EINTR if a fatal signal was encountered.
1589 * folio_end_writeback - End writeback against a folio.
1641 struct address_space *mapping; in page_endio() local
1644 mapping = folio_mapping(folio); in page_endio()
1645 if (mapping) in page_endio()
1646 mapping_set_error(mapping, err); in page_endio()
1654 * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it.
1676 wait->folio = folio; in __folio_lock_async()
1677 wait->bit_nr = PG_locked; in __folio_lock_async()
1679 spin_lock_irq(&q->lock); in __folio_lock_async()
1680 __add_wait_queue_entry_tail(q, &wait->wait); in __folio_lock_async()
1690 __remove_wait_queue(q, &wait->wait); in __folio_lock_async()
1692 ret = -EIOCBQUEUED; in __folio_lock_async()
1693 spin_unlock_irq(&q->lock); in __folio_lock_async()
1699 * true - folio is locked; mmap_lock is still held.
1700 * false - folio is not locked.
1742 * page_cache_next_miss() - Find the next gap in the page cache.
1743 * @mapping: Mapping.
1747 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1757 * range specified (in which case 'return - index >= max_scan' will be true).
1758 * In the rare case of index wrap-around, 0 will be returned.
1760 pgoff_t page_cache_next_miss(struct address_space *mapping, in page_cache_next_miss() argument
1763 XA_STATE(xas, &mapping->i_pages, index); in page_cache_next_miss()
1765 while (max_scan--) { in page_cache_next_miss()
1778 * page_cache_prev_miss() - Find the previous gap in the page cache.
1779 * @mapping: Mapping.
1783 * Search the range [max(index - max_scan + 1, 0), index] for the
1793 * range specified (in which case 'index - return >= max_scan' will be true).
1794 * In the rare case of wrap-around, ULONG_MAX will be returned.
1796 pgoff_t page_cache_prev_miss(struct address_space *mapping, in page_cache_prev_miss() argument
1799 XA_STATE(xas, &mapping->i_pages, index); in page_cache_prev_miss()
1801 while (max_scan--) { in page_cache_prev_miss()
1834 * mapping_get_entry - Get a page cache entry.
1835 * @mapping: the address_space to search
1838 * Looks up the page cache entry at @mapping & @index. If it is a folio,
1845 static void *mapping_get_entry(struct address_space *mapping, pgoff_t index) in mapping_get_entry() argument
1847 XA_STATE(xas, &mapping->i_pages, index); in mapping_get_entry()
1877 * __filemap_get_folio - Find and get a reference to a folio.
1878 * @mapping: The address_space to search.
1883 * Looks up the page cache entry at @mapping & @index.
1887 * * %FGP_ACCESSED - The folio will be marked accessed.
1888 * * %FGP_LOCK - The folio is returned locked.
1889 * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it
1891 * * %FGP_CREAT - If no page is present then a new page is allocated using
1894 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1897 * * %FGP_WRITE - The page will be written to by the caller.
1898 * * %FGP_NOFS - __GFP_FS will get cleared in gfp.
1899 * * %FGP_NOWAIT - Don't get blocked by page lock.
1900 * * %FGP_STABLE - Wait for the folio to be stable (finished writeback)
1909 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, in __filemap_get_folio() argument
1915 folio = mapping_get_entry(mapping, index); in __filemap_get_folio()
1935 if (unlikely(folio->mapping != mapping)) { in __filemap_get_folio()
1956 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) in __filemap_get_folio()
1976 err = filemap_add_folio(mapping, folio, index, gfp); in __filemap_get_folio()
1980 if (err == -EEXIST) in __filemap_get_folio()
2032 * find_get_entries - gang pagecache lookup
2033 * @mapping: The address_space to search
2040 * the mapping. The entries are placed in @fbatch. find_get_entries()
2044 * due to not-present entries or large folios.
2051 unsigned find_get_entries(struct address_space *mapping, pgoff_t start, in find_get_entries() argument
2054 XA_STATE(xas, &mapping->i_pages, start); in find_get_entries()
2059 indices[fbatch->nr] = xas.xa_index; in find_get_entries()
2069 * find_lock_entries - Find a batch of pagecache entries.
2070 * @mapping: The address_space to search.
2076 * find_lock_entries() will return a batch of entries from @mapping.
2083 * due to not-present entries, large folios, folios which could not be
2088 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, in find_lock_entries() argument
2091 XA_STATE(xas, &mapping->i_pages, start); in find_lock_entries()
2097 if (folio->index < start) in find_lock_entries()
2099 if (folio->index + folio_nr_pages(folio) - 1 > end) in find_lock_entries()
2103 if (folio->mapping != mapping || in find_lock_entries()
2109 indices[fbatch->nr] = xas.xa_index; in find_lock_entries()
2124 * filemap_get_folios - Get a batch of folios
2125 * @mapping: The address_space to search
2130 * Search for and return a batch of folios in the mapping starting at
2144 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, in filemap_get_folios() argument
2147 XA_STATE(xas, &mapping->i_pages, *start); in filemap_get_folios()
2160 *start = folio->index + nr; in filemap_get_folios()
2168 * breaks the iteration when there is a page at index -1 but that is in filemap_get_folios()
2171 if (end == (pgoff_t)-1) in filemap_get_folios()
2172 *start = (pgoff_t)-1; in filemap_get_folios()
2189 return index < folio->index + folio_nr_pages(folio) - 1; in folio_more_pages()
2193 * filemap_get_folios_contig - Get a batch of contiguous folios
2194 * @mapping: The address_space to search
2207 unsigned filemap_get_folios_contig(struct address_space *mapping, in filemap_get_folios_contig() argument
2210 XA_STATE(xas, &mapping->i_pages, *start); in filemap_get_folios_contig()
2238 *start = folio->index + nr; in filemap_get_folios_contig()
2253 folio = fbatch->folios[nr - 1]; in filemap_get_folios_contig()
2255 *start = folio->index + 1; in filemap_get_folios_contig()
2257 *start = folio->index + folio_nr_pages(folio); in filemap_get_folios_contig()
2266 * find_get_pages_range_tag - Find and return head pages matching @tag.
2267 * @mapping: the address_space to search
2280 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, in find_get_pages_range_tag() argument
2284 XA_STATE(xas, &mapping->i_pages, *index); in find_get_pages_range_tag()
2301 pages[ret] = &folio->page; in find_get_pages_range_tag()
2303 *index = folio->index + folio_nr_pages(folio); in find_get_pages_range_tag()
2311 * iteration when there is a page at index -1 but that is already in find_get_pages_range_tag()
2314 if (end == (pgoff_t)-1) in find_get_pages_range_tag()
2315 *index = (pgoff_t)-1; in find_get_pages_range_tag()
2329 * ---R__________________________________________B__________
2342 ra->ra_pages /= 4; in shrink_readahead_size_eio()
2346 * filemap_get_read_batch - Get a batch of folios for read
2354 static void filemap_get_read_batch(struct address_space *mapping, in filemap_get_read_batch() argument
2357 XA_STATE(xas, &mapping->i_pages, index); in filemap_get_read_batch()
2380 xas_advance(&xas, folio->index + folio_nr_pages(folio) - 1); in filemap_get_read_batch()
2419 shrink_readahead_size_eio(&file->f_ra); in filemap_read_folio()
2420 return -EIO; in filemap_read_folio()
2423 static bool filemap_range_uptodate(struct address_space *mapping, in filemap_range_uptodate() argument
2433 if (!mapping->a_ops->is_partially_uptodate) in filemap_range_uptodate()
2435 if (mapping->host->i_blkbits >= folio_shift(folio)) in filemap_range_uptodate()
2438 count = iter->count; in filemap_range_uptodate()
2440 count -= folio_pos(folio) - pos; in filemap_range_uptodate()
2443 pos -= folio_pos(folio); in filemap_range_uptodate()
2446 return mapping->a_ops->is_partially_uptodate(folio, pos, count); in filemap_range_uptodate()
2450 struct address_space *mapping, struct iov_iter *iter, in filemap_update_page() argument
2455 if (iocb->ki_flags & IOCB_NOWAIT) { in filemap_update_page()
2456 if (!filemap_invalidate_trylock_shared(mapping)) in filemap_update_page()
2457 return -EAGAIN; in filemap_update_page()
2459 filemap_invalidate_lock_shared(mapping); in filemap_update_page()
2463 error = -EAGAIN; in filemap_update_page()
2464 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) in filemap_update_page()
2466 if (!(iocb->ki_flags & IOCB_WAITQ)) { in filemap_update_page()
2467 filemap_invalidate_unlock_shared(mapping); in filemap_update_page()
2475 error = __folio_lock_async(folio, iocb->ki_waitq); in filemap_update_page()
2481 if (!folio->mapping) in filemap_update_page()
2485 if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, folio)) in filemap_update_page()
2488 error = -EAGAIN; in filemap_update_page()
2489 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) in filemap_update_page()
2492 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio, in filemap_update_page()
2498 filemap_invalidate_unlock_shared(mapping); in filemap_update_page()
2505 struct address_space *mapping, pgoff_t index, in filemap_create_folio() argument
2511 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0); in filemap_create_folio()
2513 return -ENOMEM; in filemap_create_folio()
2524 * pages or ->readahead() that need to hold invalidate_lock in filemap_create_folio()
2525 * while mapping blocks for IO so let's hold the lock here as in filemap_create_folio()
2528 filemap_invalidate_lock_shared(mapping); in filemap_create_folio()
2529 error = filemap_add_folio(mapping, folio, index, in filemap_create_folio()
2530 mapping_gfp_constraint(mapping, GFP_KERNEL)); in filemap_create_folio()
2531 if (error == -EEXIST) in filemap_create_folio()
2536 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); in filemap_create_folio()
2540 filemap_invalidate_unlock_shared(mapping); in filemap_create_folio()
2544 filemap_invalidate_unlock_shared(mapping); in filemap_create_folio()
2550 struct address_space *mapping, struct folio *folio, in filemap_readahead() argument
2553 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); in filemap_readahead()
2555 if (iocb->ki_flags & IOCB_NOIO) in filemap_readahead()
2556 return -EAGAIN; in filemap_readahead()
2557 page_cache_async_ra(&ractl, folio, last_index - folio->index); in filemap_readahead()
2564 struct file *filp = iocb->ki_filp; in filemap_get_pages()
2565 struct address_space *mapping = filp->f_mapping; in filemap_get_pages() local
2566 struct file_ra_state *ra = &filp->f_ra; in filemap_get_pages()
2567 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; in filemap_get_pages()
2572 last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE); in filemap_get_pages()
2575 return -EINTR; in filemap_get_pages()
2577 filemap_get_read_batch(mapping, index, last_index, fbatch); in filemap_get_pages()
2579 if (iocb->ki_flags & IOCB_NOIO) in filemap_get_pages()
2580 return -EAGAIN; in filemap_get_pages()
2581 page_cache_sync_readahead(mapping, ra, filp, index, in filemap_get_pages()
2582 last_index - index); in filemap_get_pages()
2583 filemap_get_read_batch(mapping, index, last_index, fbatch); in filemap_get_pages()
2586 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) in filemap_get_pages()
2587 return -EAGAIN; in filemap_get_pages()
2588 err = filemap_create_folio(filp, mapping, in filemap_get_pages()
2589 iocb->ki_pos >> PAGE_SHIFT, fbatch); in filemap_get_pages()
2595 folio = fbatch->folios[folio_batch_count(fbatch) - 1]; in filemap_get_pages()
2597 err = filemap_readahead(iocb, filp, mapping, folio, last_index); in filemap_get_pages()
2602 if ((iocb->ki_flags & IOCB_WAITQ) && in filemap_get_pages()
2604 iocb->ki_flags |= IOCB_NOWAIT; in filemap_get_pages()
2605 err = filemap_update_page(iocb, mapping, iter, folio); in filemap_get_pages()
2614 if (likely(--fbatch->nr)) in filemap_get_pages()
2629 * filemap_read - Read data from the page cache.
2631 * @iter: Destination for the data.
2634 * Copies data from the page cache. If the data is not currently present,
2644 struct file *filp = iocb->ki_filp; in filemap_read()
2645 struct file_ra_state *ra = &filp->f_ra; in filemap_read()
2646 struct address_space *mapping = filp->f_mapping; in filemap_read() local
2647 struct inode *inode = mapping->host; in filemap_read()
2653 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) in filemap_read()
2658 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); in filemap_read()
2665 * If we've already successfully copied some data, then we in filemap_read()
2666 * can no longer safely return -EIOCBQUEUED. Hence mark in filemap_read()
2669 if ((iocb->ki_flags & IOCB_WAITQ) && already_read) in filemap_read()
2670 iocb->ki_flags |= IOCB_NOWAIT; in filemap_read()
2672 if (unlikely(iocb->ki_pos >= i_size_read(inode))) in filemap_read()
2683 * the correct value for "nr", which means the zero-filled in filemap_read()
2685 * another truncate extends the file - this is desired though). in filemap_read()
2688 if (unlikely(iocb->ki_pos >= isize)) in filemap_read()
2690 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); in filemap_read()
2693 * Once we start copying data, we don't want to be touching any in filemap_read()
2696 writably_mapped = mapping_writably_mapped(mapping); in filemap_read()
2702 if (!pos_same_folio(iocb->ki_pos, ra->prev_pos - 1, in filemap_read()
2709 size_t offset = iocb->ki_pos & (fsize - 1); in filemap_read()
2710 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, in filemap_read()
2711 fsize - offset); in filemap_read()
2729 iocb->ki_pos += copied; in filemap_read()
2730 ra->prev_pos = iocb->ki_pos; in filemap_read()
2733 error = -EFAULT; in filemap_read()
2741 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); in filemap_read()
2750 * generic_file_read_iter - generic filesystem read routine
2752 * @iter: destination for the data read
2757 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2758 * be returned when no data can be read without waiting for I/O requests
2761 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2762 * requests shall be made for the read or for readahead. When no data
2763 * can be read, -EAGAIN shall be returned. When readahead would be
2779 if (iocb->ki_flags & IOCB_DIRECT) { in generic_file_read_iter()
2780 struct file *file = iocb->ki_filp; in generic_file_read_iter()
2781 struct address_space *mapping = file->f_mapping; in generic_file_read_iter() local
2782 struct inode *inode = mapping->host; in generic_file_read_iter()
2784 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_read_iter()
2785 if (filemap_range_needs_writeback(mapping, iocb->ki_pos, in generic_file_read_iter()
2786 iocb->ki_pos + count - 1)) in generic_file_read_iter()
2787 return -EAGAIN; in generic_file_read_iter()
2789 retval = filemap_write_and_wait_range(mapping, in generic_file_read_iter()
2790 iocb->ki_pos, in generic_file_read_iter()
2791 iocb->ki_pos + count - 1); in generic_file_read_iter()
2798 retval = mapping->a_ops->direct_IO(iocb, iter); in generic_file_read_iter()
2800 iocb->ki_pos += retval; in generic_file_read_iter()
2801 count -= retval; in generic_file_read_iter()
2803 if (retval != -EIOCBQUEUED) in generic_file_read_iter()
2804 iov_iter_revert(iter, count - iov_iter_count(iter)); in generic_file_read_iter()
2817 if (iocb->ki_pos >= i_size_read(inode)) in generic_file_read_iter()
2826 struct address_space *mapping, struct folio *folio, in folio_seek_hole_data() argument
2829 const struct address_space_operations *ops = mapping->a_ops; in folio_seek_hole_data()
2830 size_t offset, bsz = i_blocksize(mapping->host); in folio_seek_hole_data()
2834 if (!ops->is_partially_uptodate) in folio_seek_hole_data()
2840 if (unlikely(folio->mapping != mapping)) in folio_seek_hole_data()
2843 offset = offset_in_folio(folio, start) & ~(bsz - 1); in folio_seek_hole_data()
2846 if (ops->is_partially_uptodate(folio, offset, bsz) == in folio_seek_hole_data()
2849 start = (start + bsz) & ~(bsz - 1); in folio_seek_hole_data()
2861 return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index); in seek_folio_size()
2866 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
2867 * @mapping: Address space to search.
2873 * contain data, your filesystem can use this function to implement
2875 * entirely memory-based such as tmpfs, and filesystems which support
2878 * Return: The requested offset on success, or -ENXIO if @whence specifies
2879 * SEEK_DATA and there is no data after @start. There is an implicit hole
2880 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
2881 * and @end contain data.
2883 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, in mapping_seek_hole_data() argument
2886 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); in mapping_seek_hole_data()
2887 pgoff_t max = (end - 1) >> PAGE_SHIFT; in mapping_seek_hole_data()
2892 return -ENXIO; in mapping_seek_hole_data()
2907 start = folio_seek_hole_data(&xas, mapping, folio, start, pos, in mapping_seek_hole_data()
2919 start = -ENXIO; in mapping_seek_hole_data()
2932 * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2933 * @vmf - the vm_fault for this fault.
2934 * @folio - the folio to lock.
2935 * @fpin - the pointer to the file we may pin (or is already pinned).
2954 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_folio_maybe_drop_mmap()
2958 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_folio_maybe_drop_mmap()
2967 mmap_read_unlock(vmf->vma->vm_mm); in lock_folio_maybe_drop_mmap()
2985 struct file *file = vmf->vma->vm_file; in do_sync_mmap_readahead()
2986 struct file_ra_state *ra = &file->f_ra; in do_sync_mmap_readahead()
2987 struct address_space *mapping = file->f_mapping; in do_sync_mmap_readahead() local
2988 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); in do_sync_mmap_readahead()
2990 unsigned long vm_flags = vmf->vma->vm_flags; in do_sync_mmap_readahead()
2997 ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1); in do_sync_mmap_readahead()
2998 ra->size = HPAGE_PMD_NR; in do_sync_mmap_readahead()
3004 ra->size *= 2; in do_sync_mmap_readahead()
3005 ra->async_size = HPAGE_PMD_NR; in do_sync_mmap_readahead()
3011 /* If we don't want any read-ahead, don't bother */ in do_sync_mmap_readahead()
3014 if (!ra->ra_pages) in do_sync_mmap_readahead()
3019 page_cache_sync_ra(&ractl, ra->ra_pages); in do_sync_mmap_readahead()
3024 mmap_miss = READ_ONCE(ra->mmap_miss); in do_sync_mmap_readahead()
3026 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); in do_sync_mmap_readahead()
3030 * stop bothering with read-ahead. It will only hurt. in do_sync_mmap_readahead()
3036 * mmap read-around in do_sync_mmap_readahead()
3039 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); in do_sync_mmap_readahead()
3040 ra->size = ra->ra_pages; in do_sync_mmap_readahead()
3041 ra->async_size = ra->ra_pages / 4; in do_sync_mmap_readahead()
3042 ractl._index = ra->start; in do_sync_mmap_readahead()
3055 struct file *file = vmf->vma->vm_file; in do_async_mmap_readahead()
3056 struct file_ra_state *ra = &file->f_ra; in do_async_mmap_readahead()
3057 DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff); in do_async_mmap_readahead()
3061 /* If we don't want any read-ahead, don't bother */ in do_async_mmap_readahead()
3062 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) in do_async_mmap_readahead()
3065 mmap_miss = READ_ONCE(ra->mmap_miss); in do_async_mmap_readahead()
3067 WRITE_ONCE(ra->mmap_miss, --mmap_miss); in do_async_mmap_readahead()
3071 page_cache_async_ra(&ractl, folio, ra->ra_pages); in do_async_mmap_readahead()
3077 * filemap_fault - read in file data for page fault handling
3081 * mapped memory region to read in file data during a page fault.
3087 * vma->vm_mm->mmap_lock must be held on entry.
3097 * Return: bitwise-OR of %VM_FAULT_ codes.
3102 struct file *file = vmf->vma->vm_file; in filemap_fault()
3104 struct address_space *mapping = file->f_mapping; in filemap_fault() local
3105 struct inode *inode = mapping->host; in filemap_fault()
3106 pgoff_t max_idx, index = vmf->pgoff; in filemap_fault()
3118 folio = filemap_get_folio(mapping, index); in filemap_fault()
3124 if (!(vmf->flags & FAULT_FLAG_TRIED)) in filemap_fault()
3127 filemap_invalidate_lock_shared(mapping); in filemap_fault()
3133 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); in filemap_fault()
3142 filemap_invalidate_lock_shared(mapping); in filemap_fault()
3145 folio = __filemap_get_folio(mapping, index, in filemap_fault()
3147 vmf->gfp_mask); in filemap_fault()
3151 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3160 if (unlikely(folio->mapping != mapping)) { in filemap_fault()
3169 * that it's up-to-date. If not, it is going to be due to an error. in filemap_fault()
3188 * time to return to the upper layer and have it re-find the vma and in filemap_fault()
3196 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3209 vmf->page = folio_file_page(folio, index); in filemap_fault()
3214 * Umm, take care of errors if the page isn't up-to-date. in filemap_fault()
3215 * Try to re-read it _once_. We do this synchronously, in filemap_fault()
3220 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); in filemap_fault()
3227 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3234 * re-find the vma and come back and find our hopefully still populated in filemap_fault()
3240 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3249 struct mm_struct *mm = vmf->vma->vm_mm; in filemap_map_pmd()
3252 if (pmd_trans_huge(*vmf->pmd)) { in filemap_map_pmd()
3258 if (pmd_none(*vmf->pmd) && PageTransHuge(page)) { in filemap_map_pmd()
3267 if (pmd_none(*vmf->pmd)) in filemap_map_pmd()
3268 pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); in filemap_map_pmd()
3271 if (pmd_devmap_trans_unstable(vmf->pmd)) { in filemap_map_pmd()
3281 struct address_space *mapping, in next_uptodate_page() argument
3304 if (folio->mapping != mapping) in next_uptodate_page()
3308 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in next_uptodate_page()
3309 if (xas->xa_index >= max_idx) in next_uptodate_page()
3321 static inline struct folio *first_map_page(struct address_space *mapping, in first_map_page() argument
3326 mapping, xas, end_pgoff); in first_map_page()
3329 static inline struct folio *next_map_page(struct address_space *mapping, in next_map_page() argument
3334 mapping, xas, end_pgoff); in next_map_page()
3340 struct vm_area_struct *vma = vmf->vma; in filemap_map_pages()
3341 struct file *file = vma->vm_file; in filemap_map_pages()
3342 struct address_space *mapping = file->f_mapping; in filemap_map_pages() local
3345 XA_STATE(xas, &mapping->i_pages, start_pgoff); in filemap_map_pages()
3348 unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); in filemap_map_pages()
3352 folio = first_map_page(mapping, &xas, end_pgoff); in filemap_map_pages()
3356 if (filemap_map_pmd(vmf, &folio->page)) { in filemap_map_pages()
3361 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); in filemap_map_pages()
3362 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); in filemap_map_pages()
3370 mmap_miss--; in filemap_map_pages()
3372 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; in filemap_map_pages()
3373 vmf->pte += xas.xa_index - last_pgoff; in filemap_map_pages()
3379 * fault-around logic. in filemap_map_pages()
3381 if (!pte_none(*vmf->pte)) in filemap_map_pages()
3385 if (vmf->address == addr) in filemap_map_pages()
3389 /* no need to invalidate: a not-present page won't be cached */ in filemap_map_pages()
3390 update_mmu_cache(vma, addr, vmf->pte); in filemap_map_pages()
3405 } while ((folio = next_map_page(mapping, &xas, end_pgoff)) != NULL); in filemap_map_pages()
3406 pte_unmap_unlock(vmf->pte, vmf->ptl); in filemap_map_pages()
3409 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); in filemap_map_pages()
3416 struct address_space *mapping = vmf->vma->vm_file->f_mapping; in filemap_page_mkwrite() local
3417 struct folio *folio = page_folio(vmf->page); in filemap_page_mkwrite()
3420 sb_start_pagefault(mapping->host->i_sb); in filemap_page_mkwrite()
3421 file_update_time(vmf->vma->vm_file); in filemap_page_mkwrite()
3423 if (folio->mapping != mapping) { in filemap_page_mkwrite()
3436 sb_end_pagefault(mapping->host->i_sb); in filemap_page_mkwrite()
3450 struct address_space *mapping = file->f_mapping; in generic_file_mmap() local
3452 if (!mapping->a_ops->read_folio) in generic_file_mmap()
3453 return -ENOEXEC; in generic_file_mmap()
3455 vma->vm_ops = &generic_file_vm_ops; in generic_file_mmap()
3460 * This is for filesystems which do not implement ->writepage.
3464 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) in generic_file_readonly_mmap()
3465 return -EINVAL; in generic_file_readonly_mmap()
3475 return -ENOSYS; in generic_file_mmap()
3479 return -ENOSYS; in generic_file_readonly_mmap()
3487 static struct folio *do_read_cache_folio(struct address_space *mapping, in do_read_cache_folio() argument
3494 filler = mapping->a_ops->read_folio; in do_read_cache_folio()
3496 folio = filemap_get_folio(mapping, index); in do_read_cache_folio()
3500 return ERR_PTR(-ENOMEM); in do_read_cache_folio()
3501 err = filemap_add_folio(mapping, folio, index, gfp); in do_read_cache_folio()
3504 if (err == -EEXIST) in do_read_cache_folio()
3520 /* Folio was truncated from mapping */ in do_read_cache_folio()
3521 if (!folio->mapping) { in do_read_cache_folio()
3548 * read_cache_folio - Read into page cache, fill it if needed.
3549 * @mapping: The address_space to read from.
3551 * @filler: Function to perform the read, or NULL to use aops->read_folio().
3560 * Context: May sleep. Expects mapping->invalidate_lock to be held.
3563 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, in read_cache_folio() argument
3566 return do_read_cache_folio(mapping, index, filler, file, in read_cache_folio()
3567 mapping_gfp_mask(mapping)); in read_cache_folio()
3571 static struct page *do_read_cache_page(struct address_space *mapping, in do_read_cache_page() argument
3576 folio = do_read_cache_folio(mapping, index, filler, file, gfp); in do_read_cache_page()
3578 return &folio->page; in do_read_cache_page()
3582 struct page *read_cache_page(struct address_space *mapping, in read_cache_page() argument
3585 return do_read_cache_page(mapping, index, filler, file, in read_cache_page()
3586 mapping_gfp_mask(mapping)); in read_cache_page()
3591 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3592 * @mapping: the page's address_space
3596 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3599 * If the page does not get brought uptodate, return -EIO.
3601 * The function expects mapping->invalidate_lock to be already held.
3605 struct page *read_cache_page_gfp(struct address_space *mapping, in read_cache_page_gfp() argument
3609 return do_read_cache_page(mapping, index, NULL, NULL, gfp); in read_cache_page_gfp()
3622 errseq_set(&filp->f_mapping->wb_err, -EIO); in dio_warn_stale_pagecache()
3627 …pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision… in dio_warn_stale_pagecache()
3628 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, in dio_warn_stale_pagecache()
3629 current->comm); in dio_warn_stale_pagecache()
3636 struct file *file = iocb->ki_filp; in generic_file_direct_write()
3637 struct address_space *mapping = file->f_mapping; in generic_file_direct_write() local
3638 struct inode *inode = mapping->host; in generic_file_direct_write()
3639 loff_t pos = iocb->ki_pos; in generic_file_direct_write()
3645 end = (pos + write_len - 1) >> PAGE_SHIFT; in generic_file_direct_write()
3647 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_direct_write()
3649 if (filemap_range_has_page(file->f_mapping, pos, in generic_file_direct_write()
3650 pos + write_len - 1)) in generic_file_direct_write()
3651 return -EAGAIN; in generic_file_direct_write()
3653 written = filemap_write_and_wait_range(mapping, pos, in generic_file_direct_write()
3654 pos + write_len - 1); in generic_file_direct_write()
3661 * the new data. We invalidate clean cached page from the region we're in generic_file_direct_write()
3663 * without clobbering -EIOCBQUEUED from ->direct_IO(). in generic_file_direct_write()
3665 written = invalidate_inode_pages2_range(mapping, in generic_file_direct_write()
3672 if (written == -EBUSY) in generic_file_direct_write()
3677 written = mapping->a_ops->direct_IO(iocb, from); in generic_file_direct_write()
3681 * cached by non-direct readahead, or faulted in by get_user_pages() in generic_file_direct_write()
3694 * Skip invalidation for async writes or if mapping has no pages. in generic_file_direct_write()
3696 if (written > 0 && mapping->nrpages && in generic_file_direct_write()
3697 invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end)) in generic_file_direct_write()
3702 write_len -= written; in generic_file_direct_write()
3703 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { in generic_file_direct_write()
3707 iocb->ki_pos = pos; in generic_file_direct_write()
3709 if (written != -EIOCBQUEUED) in generic_file_direct_write()
3710 iov_iter_revert(from, write_len - iov_iter_count(from)); in generic_file_direct_write()
3718 struct file *file = iocb->ki_filp; in generic_perform_write()
3719 loff_t pos = iocb->ki_pos; in generic_perform_write()
3720 struct address_space *mapping = file->f_mapping; in generic_perform_write() local
3721 const struct address_space_operations *a_ops = mapping->a_ops; in generic_perform_write()
3732 offset = (pos & (PAGE_SIZE - 1)); in generic_perform_write()
3733 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3741 * up-to-date. in generic_perform_write()
3744 status = -EFAULT; in generic_perform_write()
3749 status = -EINTR; in generic_perform_write()
3753 status = a_ops->write_begin(file, mapping, pos, bytes, in generic_perform_write()
3758 if (mapping_writably_mapped(mapping)) in generic_perform_write()
3764 status = a_ops->write_end(file, mapping, pos, bytes, copied, in generic_perform_write()
3767 iov_iter_revert(i, copied - max(status, 0L)); in generic_perform_write()
3775 * A short copy made ->write_end() reject the in generic_perform_write()
3787 balance_dirty_pages_ratelimited(mapping); in generic_perform_write()
3795 * __generic_file_write_iter - write data to a file
3797 * @from: iov_iter with data to write
3799 * This function does all the work needed for actually writing data to a
3807 * This function does *not* take care of syncing data in case of O_SYNC write.
3813 * * negative error code if no data has been written at all
3817 struct file *file = iocb->ki_filp; in __generic_file_write_iter()
3818 struct address_space *mapping = file->f_mapping; in __generic_file_write_iter() local
3819 struct inode *inode = mapping->host; in __generic_file_write_iter()
3825 current->backing_dev_info = inode_to_bdi(inode); in __generic_file_write_iter()
3834 if (iocb->ki_flags & IOCB_DIRECT) { in __generic_file_write_iter()
3843 * page-cache pages correctly). in __generic_file_write_iter()
3848 pos = iocb->ki_pos; in __generic_file_write_iter()
3853 * direct-written, or the error code if that was zero. Note in __generic_file_write_iter()
3854 * that this differs from normal direct-io semantics, which in __generic_file_write_iter()
3855 * will return -EFOO even if some bytes were written. in __generic_file_write_iter()
3866 endbyte = pos + status - 1; in __generic_file_write_iter()
3867 err = filemap_write_and_wait_range(mapping, pos, endbyte); in __generic_file_write_iter()
3869 iocb->ki_pos = endbyte + 1; in __generic_file_write_iter()
3871 invalidate_mapping_pages(mapping, in __generic_file_write_iter()
3877 * the number of bytes which were direct-written in __generic_file_write_iter()
3883 iocb->ki_pos += written; in __generic_file_write_iter()
3886 current->backing_dev_info = NULL; in __generic_file_write_iter()
3892 * generic_file_write_iter - write data to a file
3894 * @from: iov_iter with data to write
3900 * * negative error code if no data has been written at all of
3906 struct file *file = iocb->ki_filp; in generic_file_write_iter()
3907 struct inode *inode = file->f_mapping->host; in generic_file_write_iter()
3923 * filemap_release_folio() - Release fs-specific metadata on a folio.
3927 * The address_space is trying to release any data attached to a folio
3928 * (presumably at folio->private).
3941 struct address_space * const mapping = folio->mapping; in filemap_release_folio() local
3947 if (mapping && mapping->a_ops->release_folio) in filemap_release_folio()
3948 return mapping->a_ops->release_folio(folio, gfp); in filemap_release_folio()