Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
28 #include <linux/error-injection.h>
31 #include <linux/backing-dev.h>
65 * finished 'unifying' the page and buffer cache and SMP-threaded the
66 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
68 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
74 * ->i_mmap_rwsem (truncate_pagecache)
75 * ->private_lock (__free_pte->__set_page_dirty_buffers)
76 * ->swap_lock (exclusive_swap_page, others)
77 * ->i_pages lock
79 * ->i_rwsem
80 * ->invalidate_lock (acquired by fs in truncate path)
81 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
83 * ->mmap_lock
84 * ->i_mmap_rwsem
85 * ->page_table_lock or pte_lock (various, mainly in memory.c)
86 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
88 * ->mmap_lock
89 * ->invalidate_lock (filemap_fault)
90 * ->lock_page (filemap_fault, access_process_vm)
92 * ->i_rwsem (generic_perform_write)
93 * ->mmap_lock (fault_in_pages_readable->do_page_fault)
95 * bdi->wb.list_lock
96 * sb_lock (fs/fs-writeback.c)
97 * ->i_pages lock (__sync_single_inode)
99 * ->i_mmap_rwsem
100 * ->anon_vma.lock (vma_adjust)
102 * ->anon_vma.lock
103 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
105 * ->page_table_lock or pte_lock
106 * ->swap_lock (try_to_unmap_one)
107 * ->private_lock (try_to_unmap_one)
108 * ->i_pages lock (try_to_unmap_one)
109 * ->lruvec->lru_lock (follow_page->mark_page_accessed)
110 * ->lruvec->lru_lock (check_pte_range->isolate_lru_page)
111 * ->private_lock (page_remove_rmap->set_page_dirty)
112 * ->i_pages lock (page_remove_rmap->set_page_dirty)
113 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
114 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
115 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
116 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
117 * ->inode->i_lock (zap_pte_range->set_page_dirty)
118 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
120 * ->i_mmap_rwsem
121 * ->tasklist_lock (memory_failure, collect_procs_ao)
124 static void page_cache_delete(struct address_space *mapping, in page_cache_delete() argument
127 XA_STATE(xas, &mapping->i_pages, page->index); in page_cache_delete()
130 mapping_set_update(&xas, mapping); in page_cache_delete()
134 xas_set_order(&xas, page->index, compound_order(page)); in page_cache_delete()
145 page->mapping = NULL; in page_cache_delete()
146 /* Leave page->index set: truncation lookup relies upon it */ in page_cache_delete()
147 mapping->nrpages -= nr; in page_cache_delete()
150 static void unaccount_page_cache_page(struct address_space *mapping, in unaccount_page_cache_page() argument
158 * stale data around in the cleancache once our page is gone in unaccount_page_cache_page()
163 cleancache_invalidate_page(mapping, page); in unaccount_page_cache_page()
171 current->comm, page_to_pfn(page)); in unaccount_page_cache_page()
177 if (mapping_exiting(mapping) && in unaccount_page_cache_page()
196 __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr); in unaccount_page_cache_page()
198 __mod_lruvec_page_state(page, NR_SHMEM, -nr); in unaccount_page_cache_page()
200 __mod_lruvec_page_state(page, NR_SHMEM_THPS, -nr); in unaccount_page_cache_page()
202 __mod_lruvec_page_state(page, NR_FILE_THPS, -nr); in unaccount_page_cache_page()
203 filemap_nr_thps_dec(mapping); in unaccount_page_cache_page()
209 * unwritten data. in unaccount_page_cache_page()
217 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); in unaccount_page_cache_page()
222 * sure the page is locked and that nobody else uses it - or that usage
227 struct address_space *mapping = page->mapping; in __delete_from_page_cache() local
231 unaccount_page_cache_page(mapping, page); in __delete_from_page_cache()
232 page_cache_delete(mapping, page, shadow); in __delete_from_page_cache()
235 static void page_cache_free_page(struct address_space *mapping, in page_cache_free_page() argument
240 freepage = mapping->a_ops->freepage; in page_cache_free_page()
253 * delete_from_page_cache - delete page from page cache
262 struct address_space *mapping = page_mapping(page); in delete_from_page_cache() local
265 xa_lock_irq(&mapping->i_pages); in delete_from_page_cache()
267 xa_unlock_irq(&mapping->i_pages); in delete_from_page_cache()
269 page_cache_free_page(mapping, page); in delete_from_page_cache()
274 * page_cache_delete_batch - delete several pages from page cache
275 * @mapping: the mapping to which pages belong
278 * The function walks over mapping->i_pages and removes pages passed in @pvec
279 * from the mapping. The function expects @pvec to be sorted by page index
281 * It tolerates holes in @pvec (mapping entries at those indices are not
287 static void page_cache_delete_batch(struct address_space *mapping, in page_cache_delete_batch() argument
290 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); in page_cache_delete_batch()
295 mapping_set_update(&xas, mapping); in page_cache_delete_batch()
310 if (page != pvec->pages[i]) { in page_cache_delete_batch()
311 VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index, in page_cache_delete_batch()
318 if (page->index == xas.xa_index) in page_cache_delete_batch()
319 page->mapping = NULL; in page_cache_delete_batch()
320 /* Leave page->index set: truncation lookup relies on it */ in page_cache_delete_batch()
324 * page or the index is of the last sub-page of this compound in page_cache_delete_batch()
327 if (page->index + compound_nr(page) - 1 == xas.xa_index) in page_cache_delete_batch()
332 mapping->nrpages -= total_pages; in page_cache_delete_batch()
335 void delete_from_page_cache_batch(struct address_space *mapping, in delete_from_page_cache_batch() argument
343 xa_lock_irq(&mapping->i_pages); in delete_from_page_cache_batch()
345 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); in delete_from_page_cache_batch()
347 unaccount_page_cache_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
349 page_cache_delete_batch(mapping, pvec); in delete_from_page_cache_batch()
350 xa_unlock_irq(&mapping->i_pages); in delete_from_page_cache_batch()
353 page_cache_free_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
356 int filemap_check_errors(struct address_space *mapping) in filemap_check_errors() argument
360 if (test_bit(AS_ENOSPC, &mapping->flags) && in filemap_check_errors()
361 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_errors()
362 ret = -ENOSPC; in filemap_check_errors()
363 if (test_bit(AS_EIO, &mapping->flags) && in filemap_check_errors()
364 test_and_clear_bit(AS_EIO, &mapping->flags)) in filemap_check_errors()
365 ret = -EIO; in filemap_check_errors()
370 static int filemap_check_and_keep_errors(struct address_space *mapping) in filemap_check_and_keep_errors() argument
373 if (test_bit(AS_EIO, &mapping->flags)) in filemap_check_and_keep_errors()
374 return -EIO; in filemap_check_and_keep_errors()
375 if (test_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_and_keep_errors()
376 return -ENOSPC; in filemap_check_and_keep_errors()
381 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
382 * @mapping: address space structure to write
385 * Call writepages on the mapping using the provided wbc to control the
390 int filemap_fdatawrite_wbc(struct address_space *mapping, in filemap_fdatawrite_wbc() argument
395 if (!mapping_can_writeback(mapping) || in filemap_fdatawrite_wbc()
396 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) in filemap_fdatawrite_wbc()
399 wbc_attach_fdatawrite_inode(wbc, mapping->host); in filemap_fdatawrite_wbc()
400 ret = do_writepages(mapping, wbc); in filemap_fdatawrite_wbc()
407 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
408 * @mapping: address space structure to write
413 * Start writeback against all of a mapping's dirty pages that lie
416 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
423 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in __filemap_fdatawrite_range() argument
433 return filemap_fdatawrite_wbc(mapping, &wbc); in __filemap_fdatawrite_range()
436 static inline int __filemap_fdatawrite(struct address_space *mapping, in __filemap_fdatawrite() argument
439 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); in __filemap_fdatawrite()
442 int filemap_fdatawrite(struct address_space *mapping) in filemap_fdatawrite() argument
444 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); in filemap_fdatawrite()
448 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in filemap_fdatawrite_range() argument
451 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); in filemap_fdatawrite_range()
456 * filemap_flush - mostly a non-blocking flush
457 * @mapping: target address_space
459 * This is a mostly non-blocking flush. Not suitable for data-integrity
460 * purposes - I/O may not be started against all dirty pages.
464 int filemap_flush(struct address_space *mapping) in filemap_flush() argument
466 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); in filemap_flush()
471 * filemap_range_has_page - check if a page exists in range.
472 * @mapping: address space within which to check
482 bool filemap_range_has_page(struct address_space *mapping, in filemap_range_has_page() argument
486 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_page()
513 static void __filemap_fdatawait_range(struct address_space *mapping, in __filemap_fdatawait_range() argument
528 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, in __filemap_fdatawait_range()
545 * filemap_fdatawait_range - wait for writeback to complete
546 * @mapping: address space structure to wait for
550 * Walk the list of under-writeback pages of the given address space
560 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, in filemap_fdatawait_range() argument
563 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range()
564 return filemap_check_errors(mapping); in filemap_fdatawait_range()
569 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
570 * @mapping: address space structure to wait for
574 * Walk the list of under-writeback pages of the given address space in the
579 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
582 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, in filemap_fdatawait_range_keep_errors() argument
585 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range_keep_errors()
586 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_range_keep_errors()
591 * file_fdatawait_range - wait for writeback to complete
596 * Walk the list of under-writeback pages of the address space that file
598 * status of the address space vs. the file->f_wb_err cursor and return it.
604 * Return: error status of the address space vs. the file->f_wb_err cursor.
608 struct address_space *mapping = file->f_mapping; in file_fdatawait_range() local
610 __filemap_fdatawait_range(mapping, start_byte, end_byte); in file_fdatawait_range()
616 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
617 * @mapping: address space structure to wait for
619 * Walk the list of under-writeback pages of the given address space
624 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
629 int filemap_fdatawait_keep_errors(struct address_space *mapping) in filemap_fdatawait_keep_errors() argument
631 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); in filemap_fdatawait_keep_errors()
632 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_keep_errors()
637 static bool mapping_needs_writeback(struct address_space *mapping) in mapping_needs_writeback() argument
639 return mapping->nrpages; in mapping_needs_writeback()
643 * filemap_range_needs_writeback - check if range potentially needs writeback
644 * @mapping: address space within which to check
656 bool filemap_range_needs_writeback(struct address_space *mapping, in filemap_range_needs_writeback() argument
659 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_needs_writeback()
663 if (!mapping_needs_writeback(mapping)) in filemap_range_needs_writeback()
665 if (!mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && in filemap_range_needs_writeback()
666 !mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) in filemap_range_needs_writeback()
686 * filemap_write_and_wait_range - write out & wait on a file range
687 * @mapping: the address_space for the pages
691 * Write out and wait upon file offsets lstart->lend, inclusive.
694 * that this function can be used to write to the very end-of-file (end = -1).
698 int filemap_write_and_wait_range(struct address_space *mapping, in filemap_write_and_wait_range() argument
703 if (mapping_needs_writeback(mapping)) { in filemap_write_and_wait_range()
704 err = __filemap_fdatawrite_range(mapping, lstart, lend, in filemap_write_and_wait_range()
708 * written partially (e.g. -ENOSPC), so we wait for it. in filemap_write_and_wait_range()
709 * But the -EIO is special case, it may indicate the worst in filemap_write_and_wait_range()
712 if (err != -EIO) { in filemap_write_and_wait_range()
713 int err2 = filemap_fdatawait_range(mapping, in filemap_write_and_wait_range()
719 filemap_check_errors(mapping); in filemap_write_and_wait_range()
722 err = filemap_check_errors(mapping); in filemap_write_and_wait_range()
728 void __filemap_set_wb_err(struct address_space *mapping, int err) in __filemap_set_wb_err() argument
730 errseq_t eseq = errseq_set(&mapping->wb_err, err); in __filemap_set_wb_err()
732 trace_filemap_set_wb_err(mapping, eseq); in __filemap_set_wb_err()
737 * file_check_and_advance_wb_err - report wb error (if any) that was previously
745 * Grab the wb_err from the mapping. If it matches what we have in the file,
748 * If it doesn't match, then take the mapping value, set the "seen" flag in
754 * While we handle mapping->wb_err with atomic operations, the f_wb_err
763 errseq_t old = READ_ONCE(file->f_wb_err); in file_check_and_advance_wb_err()
764 struct address_space *mapping = file->f_mapping; in file_check_and_advance_wb_err() local
767 if (errseq_check(&mapping->wb_err, old)) { in file_check_and_advance_wb_err()
769 spin_lock(&file->f_lock); in file_check_and_advance_wb_err()
770 old = file->f_wb_err; in file_check_and_advance_wb_err()
771 err = errseq_check_and_advance(&mapping->wb_err, in file_check_and_advance_wb_err()
772 &file->f_wb_err); in file_check_and_advance_wb_err()
774 spin_unlock(&file->f_lock); in file_check_and_advance_wb_err()
782 clear_bit(AS_EIO, &mapping->flags); in file_check_and_advance_wb_err()
783 clear_bit(AS_ENOSPC, &mapping->flags); in file_check_and_advance_wb_err()
789 * file_write_and_wait_range - write out & wait on a file range
794 * Write out and wait upon file offsets lstart->lend, inclusive.
797 * that this function can be used to write to the very end-of-file (end = -1).
799 * After writing out and waiting on the data, we check and advance the
807 struct address_space *mapping = file->f_mapping; in file_write_and_wait_range() local
809 if (mapping_needs_writeback(mapping)) { in file_write_and_wait_range()
810 err = __filemap_fdatawrite_range(mapping, lstart, lend, in file_write_and_wait_range()
813 if (err != -EIO) in file_write_and_wait_range()
814 __filemap_fdatawait_range(mapping, lstart, lend); in file_write_and_wait_range()
824 * replace_page_cache_page - replace a pagecache page with a new one
838 struct address_space *mapping = old->mapping; in replace_page_cache_page() local
839 void (*freepage)(struct page *) = mapping->a_ops->freepage; in replace_page_cache_page()
840 pgoff_t offset = old->index; in replace_page_cache_page()
841 XA_STATE(xas, &mapping->i_pages, offset); in replace_page_cache_page()
845 VM_BUG_ON_PAGE(new->mapping, new); in replace_page_cache_page()
848 new->mapping = mapping; in replace_page_cache_page()
849 new->index = offset; in replace_page_cache_page()
856 old->mapping = NULL; in replace_page_cache_page()
874 struct address_space *mapping, in __add_to_page_cache_locked() argument
878 XA_STATE(xas, &mapping->i_pages, offset); in __add_to_page_cache_locked()
885 mapping_set_update(&xas, mapping); in __add_to_page_cache_locked()
888 page->mapping = mapping; in __add_to_page_cache_locked()
889 page->index = offset; in __add_to_page_cache_locked()
911 xas_set_err(&xas, -EEXIST); in __add_to_page_cache_locked()
931 mapping->nrpages++; in __add_to_page_cache_locked()
950 page->mapping = NULL; in __add_to_page_cache_locked()
951 /* Leave page->index set: truncation relies upon it */ in __add_to_page_cache_locked()
958 * add_to_page_cache_locked - add a locked page to the pagecache
960 * @mapping: the page's address_space
969 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, in add_to_page_cache_locked() argument
972 return __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_locked()
977 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, in add_to_page_cache_lru() argument
984 ret = __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_lru()
994 * data from the working set, only to cache data that will in add_to_page_cache_lru()
1028 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
1030 * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
1032 * @mapping1: the first mapping to lock
1033 * @mapping2: the second mapping to lock
1041 down_write(&mapping1->invalidate_lock); in filemap_invalidate_lock_two()
1043 down_write_nested(&mapping2->invalidate_lock, 1); in filemap_invalidate_lock_two()
1048 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
1050 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1052 * @mapping1: the first mapping to unlock
1053 * @mapping2: the second mapping to unlock
1059 up_write(&mapping1->invalidate_lock); in filemap_invalidate_unlock_two()
1061 up_write(&mapping2->invalidate_lock); in filemap_invalidate_unlock_two()
1095 * The page wait code treats the "wait->flags" somewhat unusually, because
1142 flags = wait->flags; in wake_page_function()
1144 if (test_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1145 return -1; in wake_page_function()
1147 if (test_and_set_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1148 return -1; in wake_page_function()
1154 * We are holding the wait-queue lock, but the waiter that in wake_page_function()
1159 * afterwards to avoid any races. This store-release pairs in wake_page_function()
1160 * with the load-acquire in wait_on_page_bit_common(). in wake_page_function()
1162 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); in wake_page_function()
1163 wake_up_state(wait->private, mode); in wake_page_function()
1171 * After this list_del_init(&wait->entry) the wait entry in wake_page_function()
1172 * might be de-allocated and the process might even have in wake_page_function()
1175 list_del_init_careful(&wait->entry); in wake_page_function()
1195 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1205 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1207 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1213 * hash, so in that case check for a page match. That prevents a long- in wake_up_page_bit()
1230 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1262 if (wait->flags & WQ_FLAG_EXCLUSIVE) { in trylock_page_bit_common()
1263 if (test_and_set_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1265 } else if (test_bit(bit_nr, &page->flags)) in trylock_page_bit_common()
1268 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; in trylock_page_bit_common()
1296 wait->func = wake_page_function; in wait_on_page_bit_common()
1301 wait->flags = 0; in wait_on_page_bit_common()
1303 wait->flags = WQ_FLAG_EXCLUSIVE; in wait_on_page_bit_common()
1304 if (--unfairness < 0) in wait_on_page_bit_common()
1305 wait->flags |= WQ_FLAG_CUSTOM; in wait_on_page_bit_common()
1322 spin_lock_irq(&q->lock); in wait_on_page_bit_common()
1326 spin_unlock_irq(&q->lock); in wait_on_page_bit_common()
1342 * be very careful with the 'wait->flags', because in wait_on_page_bit_common()
1351 flags = smp_load_acquire(&wait->flags); in wait_on_page_bit_common()
1360 /* If we were non-exclusive, we're done */ in wait_on_page_bit_common()
1374 if (unlikely(test_and_set_bit(bit_nr, &page->flags))) in wait_on_page_bit_common()
1377 wait->flags |= WQ_FLAG_DONE; in wait_on_page_bit_common()
1383 * waiter from the wait-queues, but the PageWaiters bit will remain in wait_on_page_bit_common()
1396 * NOTE! The wait->flags weren't stable until we've done the in wait_on_page_bit_common()
1405 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive in wait_on_page_bit_common()
1409 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; in wait_on_page_bit_common()
1411 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; in wait_on_page_bit_common()
1429 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1439 * Return: 0 if the page was unlocked or -EINTR if interrupted by a signal.
1451 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1462 spin_lock_irqsave(&q->lock, flags); in add_page_wait_queue()
1465 spin_unlock_irqrestore(&q->lock, flags); in add_page_wait_queue()
1493 * unlock_page - unlock a locked page
1499 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1502 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1512 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) in unlock_page()
1518 * end_page_private_2 - Clear PG_private_2 and release any waiters
1532 clear_bit_unlock(PG_private_2, &page->flags); in end_page_private_2()
1539 * wait_on_page_private_2 - Wait for PG_private_2 to be cleared on a page
1553 * wait_on_page_private_2_killable - Wait for PG_private_2 to be cleared on a page
1560 * - 0 if successful.
1561 * - -EINTR if a fatal signal was encountered.
1579 * end_page_writeback - end writeback against a page
1628 struct address_space *mapping; in page_endio() local
1631 mapping = page_mapping(page); in page_endio()
1632 if (mapping) in page_endio()
1633 mapping_set_error(mapping, err); in page_endio()
1641 * __lock_page - get a lock on the page, assuming we need to sleep to get it
1667 wait->page = page; in __lock_page_async()
1668 wait->bit_nr = PG_locked; in __lock_page_async()
1670 spin_lock_irq(&q->lock); in __lock_page_async()
1671 __add_wait_queue_entry_tail(q, &wait->wait); in __lock_page_async()
1681 __remove_wait_queue(q, &wait->wait); in __lock_page_async()
1683 ret = -EIOCBQUEUED; in __lock_page_async()
1684 spin_unlock_irq(&q->lock); in __lock_page_async()
1690 * 1 - page is locked; mmap_lock is still held.
1691 * 0 - page is not locked.
1733 * page_cache_next_miss() - Find the next gap in the page cache.
1734 * @mapping: Mapping.
1738 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1748 * range specified (in which case 'return - index >= max_scan' will be true).
1749 * In the rare case of index wrap-around, 0 will be returned.
1751 pgoff_t page_cache_next_miss(struct address_space *mapping, in page_cache_next_miss() argument
1754 XA_STATE(xas, &mapping->i_pages, index); in page_cache_next_miss()
1756 while (max_scan--) { in page_cache_next_miss()
1769 * page_cache_prev_miss() - Find the previous gap in the page cache.
1770 * @mapping: Mapping.
1774 * Search the range [max(index - max_scan + 1, 0), index] for the
1784 * range specified (in which case 'index - return >= max_scan' will be true).
1785 * In the rare case of wrap-around, ULONG_MAX will be returned.
1787 pgoff_t page_cache_prev_miss(struct address_space *mapping, in page_cache_prev_miss() argument
1790 XA_STATE(xas, &mapping->i_pages, index); in page_cache_prev_miss()
1792 while (max_scan--) { in page_cache_prev_miss()
1805 * mapping_get_entry - Get a page cache entry.
1806 * @mapping: the address_space to search
1809 * Looks up the page cache slot at @mapping & @index. If there is a
1817 static struct page *mapping_get_entry(struct address_space *mapping, in mapping_get_entry() argument
1820 XA_STATE(xas, &mapping->i_pages, index); in mapping_get_entry()
1855 * pagecache_get_page - Find and get a reference to a page.
1856 * @mapping: The address_space to search.
1861 * Looks up the page cache entry at @mapping & @index.
1865 * * %FGP_ACCESSED - The page will be marked accessed.
1866 * * %FGP_LOCK - The page is returned locked.
1867 * * %FGP_HEAD - If the page is present and a THP, return the head page
1869 * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it
1871 * * %FGP_CREAT - If no page is present then a new page is allocated using
1874 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1877 * * %FGP_WRITE - The page will be written
1878 * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
1879 * * %FGP_NOWAIT - Don't get blocked by page lock
1888 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, in pagecache_get_page() argument
1894 page = mapping_get_entry(mapping, index); in pagecache_get_page()
1914 if (unlikely(page->mapping != mapping)) { in pagecache_get_page()
1935 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) in pagecache_get_page()
1951 err = add_to_page_cache_lru(page, mapping, index, gfp_mask); in pagecache_get_page()
1955 if (err == -EEXIST) in pagecache_get_page()
2008 * find_get_entries - gang pagecache lookup
2009 * @mapping: The address_space to search
2016 * the mapping. The entries are placed in @pvec. find_get_entries()
2019 * The search returns a group of mapping-contiguous page cache entries
2021 * not-present pages.
2033 unsigned find_get_entries(struct address_space *mapping, pgoff_t start, in find_get_entries() argument
2036 XA_STATE(xas, &mapping->i_pages, start); in find_get_entries()
2054 pvec->pages[ret] = page; in find_get_entries()
2060 pvec->nr = ret; in find_get_entries()
2065 * find_lock_entries - Find a batch of pagecache entries.
2066 * @mapping: The address_space to search.
2072 * find_lock_entries() will return a batch of entries from @mapping.
2080 * due to not-present entries, THP pages, pages which could not be locked
2085 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, in find_lock_entries() argument
2088 XA_STATE(xas, &mapping->i_pages, start); in find_lock_entries()
2094 if (page->index < start) in find_lock_entries()
2096 VM_BUG_ON_PAGE(page->index != xas.xa_index, page); in find_lock_entries()
2097 if (page->index + thp_nr_pages(page) - 1 > end) in find_lock_entries()
2101 if (page->mapping != mapping || PageWriteback(page)) in find_lock_entries()
2106 indices[pvec->nr] = xas.xa_index; in find_lock_entries()
2118 /* Final THP may cross MAX_LFS_FILESIZE on 32-bit */ in find_lock_entries()
2119 xas_set(&xas, page->index + nr_pages); in find_lock_entries()
2130 * find_get_pages_range - gang pagecache lookup
2131 * @mapping: The address_space to search
2138 * pages in the mapping starting at index @start and up to index @end
2142 * The search returns a group of mapping-contiguous pages with ascending
2143 * indexes. There may be holes in the indices due to not-present pages.
2150 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, in find_get_pages_range() argument
2154 XA_STATE(xas, &mapping->i_pages, *start); in find_get_pages_range()
2177 * breaks the iteration when there is a page at index -1 but that is in find_get_pages_range()
2180 if (end == (pgoff_t)-1) in find_get_pages_range()
2181 *start = (pgoff_t)-1; in find_get_pages_range()
2191 * find_get_pages_contig - gang contiguous pagecache lookup
2192 * @mapping: The address_space to search
2202 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, in find_get_pages_contig() argument
2205 XA_STATE(xas, &mapping->i_pages, index); in find_get_pages_contig()
2245 * find_get_pages_range_tag - Find and return head pages matching @tag.
2246 * @mapping: the address_space to search
2259 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, in find_get_pages_range_tag() argument
2263 XA_STATE(xas, &mapping->i_pages, *index); in find_get_pages_range_tag()
2282 *index = page->index + thp_nr_pages(page); in find_get_pages_range_tag()
2290 * iteration when there is a page at index -1 but that is already in find_get_pages_range_tag()
2293 if (end == (pgoff_t)-1) in find_get_pages_range_tag()
2294 *index = (pgoff_t)-1; in find_get_pages_range_tag()
2308 * ---R__________________________________________B__________
2321 ra->ra_pages /= 4; in shrink_readahead_size_eio()
2325 * filemap_get_read_batch - Get a batch of pages for read
2333 static void filemap_get_read_batch(struct address_space *mapping, in filemap_get_read_batch() argument
2336 XA_STATE(xas, &mapping->i_pages, index); in filemap_get_read_batch()
2358 xas.xa_index = head->index + thp_nr_pages(head) - 1; in filemap_get_read_batch()
2369 static int filemap_read_page(struct file *file, struct address_space *mapping, in filemap_read_page() argument
2381 error = mapping->a_ops->readpage(file, page); in filemap_read_page()
2390 shrink_readahead_size_eio(&file->f_ra); in filemap_read_page()
2391 return -EIO; in filemap_read_page()
2394 static bool filemap_range_uptodate(struct address_space *mapping, in filemap_range_uptodate() argument
2404 if (!mapping->a_ops->is_partially_uptodate) in filemap_range_uptodate()
2406 if (mapping->host->i_blkbits >= (PAGE_SHIFT + thp_order(page))) in filemap_range_uptodate()
2409 count = iter->count; in filemap_range_uptodate()
2411 count -= page_offset(page) - pos; in filemap_range_uptodate()
2414 pos -= page_offset(page); in filemap_range_uptodate()
2417 return mapping->a_ops->is_partially_uptodate(page, pos, count); in filemap_range_uptodate()
2421 struct address_space *mapping, struct iov_iter *iter, in filemap_update_page() argument
2426 if (iocb->ki_flags & IOCB_NOWAIT) { in filemap_update_page()
2427 if (!filemap_invalidate_trylock_shared(mapping)) in filemap_update_page()
2428 return -EAGAIN; in filemap_update_page()
2430 filemap_invalidate_lock_shared(mapping); in filemap_update_page()
2434 error = -EAGAIN; in filemap_update_page()
2435 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) in filemap_update_page()
2437 if (!(iocb->ki_flags & IOCB_WAITQ)) { in filemap_update_page()
2438 filemap_invalidate_unlock_shared(mapping); in filemap_update_page()
2442 error = __lock_page_async(page, iocb->ki_waitq); in filemap_update_page()
2448 if (!page->mapping) in filemap_update_page()
2452 if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, page)) in filemap_update_page()
2455 error = -EAGAIN; in filemap_update_page()
2456 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) in filemap_update_page()
2459 error = filemap_read_page(iocb->ki_filp, mapping, page); in filemap_update_page()
2464 filemap_invalidate_unlock_shared(mapping); in filemap_update_page()
2471 struct address_space *mapping, pgoff_t index, in filemap_create_page() argument
2477 page = page_cache_alloc(mapping); in filemap_create_page()
2479 return -ENOMEM; in filemap_create_page()
2489 * ->readpages() that need to hold invalidate_lock while mapping blocks in filemap_create_page()
2493 filemap_invalidate_lock_shared(mapping); in filemap_create_page()
2494 error = add_to_page_cache_lru(page, mapping, index, in filemap_create_page()
2495 mapping_gfp_constraint(mapping, GFP_KERNEL)); in filemap_create_page()
2496 if (error == -EEXIST) in filemap_create_page()
2501 error = filemap_read_page(file, mapping, page); in filemap_create_page()
2505 filemap_invalidate_unlock_shared(mapping); in filemap_create_page()
2509 filemap_invalidate_unlock_shared(mapping); in filemap_create_page()
2515 struct address_space *mapping, struct page *page, in filemap_readahead() argument
2518 if (iocb->ki_flags & IOCB_NOIO) in filemap_readahead()
2519 return -EAGAIN; in filemap_readahead()
2520 page_cache_async_readahead(mapping, &file->f_ra, file, page, in filemap_readahead()
2521 page->index, last_index - page->index); in filemap_readahead()
2528 struct file *filp = iocb->ki_filp; in filemap_get_pages()
2529 struct address_space *mapping = filp->f_mapping; in filemap_get_pages() local
2530 struct file_ra_state *ra = &filp->f_ra; in filemap_get_pages()
2531 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; in filemap_get_pages()
2536 last_index = DIV_ROUND_UP(iocb->ki_pos + iter->count, PAGE_SIZE); in filemap_get_pages()
2539 return -EINTR; in filemap_get_pages()
2541 filemap_get_read_batch(mapping, index, last_index, pvec); in filemap_get_pages()
2543 if (iocb->ki_flags & IOCB_NOIO) in filemap_get_pages()
2544 return -EAGAIN; in filemap_get_pages()
2545 page_cache_sync_readahead(mapping, ra, filp, index, in filemap_get_pages()
2546 last_index - index); in filemap_get_pages()
2547 filemap_get_read_batch(mapping, index, last_index, pvec); in filemap_get_pages()
2550 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) in filemap_get_pages()
2551 return -EAGAIN; in filemap_get_pages()
2552 err = filemap_create_page(filp, mapping, in filemap_get_pages()
2553 iocb->ki_pos >> PAGE_SHIFT, pvec); in filemap_get_pages()
2559 page = pvec->pages[pagevec_count(pvec) - 1]; in filemap_get_pages()
2561 err = filemap_readahead(iocb, filp, mapping, page, last_index); in filemap_get_pages()
2566 if ((iocb->ki_flags & IOCB_WAITQ) && pagevec_count(pvec) > 1) in filemap_get_pages()
2567 iocb->ki_flags |= IOCB_NOWAIT; in filemap_get_pages()
2568 err = filemap_update_page(iocb, mapping, iter, page); in filemap_get_pages()
2577 if (likely(--pvec->nr)) in filemap_get_pages()
2585 * filemap_read - Read data from the page cache.
2587 * @iter: Destination for the data.
2590 * Copies data from the page cache. If the data is not currently present,
2600 struct file *filp = iocb->ki_filp; in filemap_read()
2601 struct file_ra_state *ra = &filp->f_ra; in filemap_read()
2602 struct address_space *mapping = filp->f_mapping; in filemap_read() local
2603 struct inode *inode = mapping->host; in filemap_read()
2609 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) in filemap_read()
2614 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); in filemap_read()
2621 * If we've already successfully copied some data, then we in filemap_read()
2622 * can no longer safely return -EIOCBQUEUED. Hence mark in filemap_read()
2625 if ((iocb->ki_flags & IOCB_WAITQ) && already_read) in filemap_read()
2626 iocb->ki_flags |= IOCB_NOWAIT; in filemap_read()
2636 * the correct value for "nr", which means the zero-filled in filemap_read()
2638 * another truncate extends the file - this is desired though). in filemap_read()
2641 if (unlikely(iocb->ki_pos >= isize)) in filemap_read()
2643 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); in filemap_read()
2646 * Once we start copying data, we don't want to be touching any in filemap_read()
2649 writably_mapped = mapping_writably_mapped(mapping); in filemap_read()
2655 if (iocb->ki_pos >> PAGE_SHIFT != in filemap_read()
2656 ra->prev_pos >> PAGE_SHIFT) in filemap_read()
2662 size_t offset = iocb->ki_pos & (page_size - 1); in filemap_read()
2663 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, in filemap_read()
2664 page_size - offset); in filemap_read()
2686 iocb->ki_pos += copied; in filemap_read()
2687 ra->prev_pos = iocb->ki_pos; in filemap_read()
2690 error = -EFAULT; in filemap_read()
2698 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error); in filemap_read()
2707 * generic_file_read_iter - generic filesystem read routine
2709 * @iter: destination for the data read
2714 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2715 * be returned when no data can be read without waiting for I/O requests
2718 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2719 * requests shall be made for the read or for readahead. When no data
2720 * can be read, -EAGAIN shall be returned. When readahead would be
2736 if (iocb->ki_flags & IOCB_DIRECT) { in generic_file_read_iter()
2737 struct file *file = iocb->ki_filp; in generic_file_read_iter()
2738 struct address_space *mapping = file->f_mapping; in generic_file_read_iter() local
2739 struct inode *inode = mapping->host; in generic_file_read_iter()
2743 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_read_iter()
2744 if (filemap_range_needs_writeback(mapping, iocb->ki_pos, in generic_file_read_iter()
2745 iocb->ki_pos + count - 1)) in generic_file_read_iter()
2746 return -EAGAIN; in generic_file_read_iter()
2748 retval = filemap_write_and_wait_range(mapping, in generic_file_read_iter()
2749 iocb->ki_pos, in generic_file_read_iter()
2750 iocb->ki_pos + count - 1); in generic_file_read_iter()
2757 retval = mapping->a_ops->direct_IO(iocb, iter); in generic_file_read_iter()
2759 iocb->ki_pos += retval; in generic_file_read_iter()
2760 count -= retval; in generic_file_read_iter()
2762 if (retval != -EIOCBQUEUED) in generic_file_read_iter()
2763 iov_iter_revert(iter, count - iov_iter_count(iter)); in generic_file_read_iter()
2774 if (retval < 0 || !count || iocb->ki_pos >= size || in generic_file_read_iter()
2784 struct address_space *mapping, struct page *page, in page_seek_hole_data() argument
2787 const struct address_space_operations *ops = mapping->a_ops; in page_seek_hole_data()
2788 size_t offset, bsz = i_blocksize(mapping->host); in page_seek_hole_data()
2792 if (!ops->is_partially_uptodate) in page_seek_hole_data()
2798 if (unlikely(page->mapping != mapping)) in page_seek_hole_data()
2801 offset = offset_in_thp(page, start) & ~(bsz - 1); in page_seek_hole_data()
2804 if (ops->is_partially_uptodate(page, offset, bsz) == seek_data) in page_seek_hole_data()
2806 start = (start + bsz) & ~(bsz - 1); in page_seek_hole_data()
2819 return PAGE_SIZE << xa_get_order(xas->xa, xas->xa_index); in seek_page_size()
2824 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
2825 * @mapping: Address space to search.
2831 * contain data, your filesystem can use this function to implement
2833 * entirely memory-based such as tmpfs, and filesystems which support
2836 * Return: The requested offset on success, or -ENXIO if @whence specifies
2837 * SEEK_DATA and there is no data after @start. There is an implicit hole
2838 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
2839 * and @end contain data.
2841 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, in mapping_seek_hole_data() argument
2844 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); in mapping_seek_hole_data()
2845 pgoff_t max = (end - 1) >> PAGE_SHIFT; in mapping_seek_hole_data()
2850 return -ENXIO; in mapping_seek_hole_data()
2865 start = page_seek_hole_data(&xas, mapping, page, start, pos, in mapping_seek_hole_data()
2877 start = -ENXIO; in mapping_seek_hole_data()
2890 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2891 * @vmf - the vm_fault for this fault.
2892 * @page - the page to lock.
2893 * @fpin - the pointer to the file we may pin (or is already pinned).
2911 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_page_maybe_drop_mmap()
2915 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_page_maybe_drop_mmap()
2924 mmap_read_unlock(vmf->vma->vm_mm); in lock_page_maybe_drop_mmap()
2942 struct file *file = vmf->vma->vm_file; in do_sync_mmap_readahead()
2943 struct file_ra_state *ra = &file->f_ra; in do_sync_mmap_readahead()
2944 struct address_space *mapping = file->f_mapping; in do_sync_mmap_readahead() local
2945 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); in do_sync_mmap_readahead()
2949 /* If we don't want any read-ahead, don't bother */ in do_sync_mmap_readahead()
2950 if (vmf->vma->vm_flags & VM_RAND_READ) in do_sync_mmap_readahead()
2952 if (!ra->ra_pages) in do_sync_mmap_readahead()
2955 if (vmf->vma->vm_flags & VM_SEQ_READ) { in do_sync_mmap_readahead()
2957 page_cache_sync_ra(&ractl, ra->ra_pages); in do_sync_mmap_readahead()
2962 mmap_miss = READ_ONCE(ra->mmap_miss); in do_sync_mmap_readahead()
2964 WRITE_ONCE(ra->mmap_miss, ++mmap_miss); in do_sync_mmap_readahead()
2968 * stop bothering with read-ahead. It will only hurt. in do_sync_mmap_readahead()
2974 * mmap read-around in do_sync_mmap_readahead()
2977 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); in do_sync_mmap_readahead()
2978 ra->size = ra->ra_pages; in do_sync_mmap_readahead()
2979 ra->async_size = ra->ra_pages / 4; in do_sync_mmap_readahead()
2980 ractl._index = ra->start; in do_sync_mmap_readahead()
2981 do_page_cache_ra(&ractl, ra->size, ra->async_size); in do_sync_mmap_readahead()
2993 struct file *file = vmf->vma->vm_file; in do_async_mmap_readahead()
2994 struct file_ra_state *ra = &file->f_ra; in do_async_mmap_readahead()
2995 struct address_space *mapping = file->f_mapping; in do_async_mmap_readahead() local
2998 pgoff_t offset = vmf->pgoff; in do_async_mmap_readahead()
3000 /* If we don't want any read-ahead, don't bother */ in do_async_mmap_readahead()
3001 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) in do_async_mmap_readahead()
3003 mmap_miss = READ_ONCE(ra->mmap_miss); in do_async_mmap_readahead()
3005 WRITE_ONCE(ra->mmap_miss, --mmap_miss); in do_async_mmap_readahead()
3008 page_cache_async_readahead(mapping, ra, file, in do_async_mmap_readahead()
3009 page, offset, ra->ra_pages); in do_async_mmap_readahead()
3015 * filemap_fault - read in file data for page fault handling
3019 * mapped memory region to read in file data during a page fault.
3025 * vma->vm_mm->mmap_lock must be held on entry.
3035 * Return: bitwise-OR of %VM_FAULT_ codes.
3040 struct file *file = vmf->vma->vm_file; in filemap_fault()
3042 struct address_space *mapping = file->f_mapping; in filemap_fault() local
3043 struct inode *inode = mapping->host; in filemap_fault()
3044 pgoff_t offset = vmf->pgoff; in filemap_fault()
3057 page = find_get_page(mapping, offset); in filemap_fault()
3063 if (!(vmf->flags & FAULT_FLAG_TRIED)) in filemap_fault()
3066 filemap_invalidate_lock_shared(mapping); in filemap_fault()
3072 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); in filemap_fault()
3081 filemap_invalidate_lock_shared(mapping); in filemap_fault()
3084 page = pagecache_get_page(mapping, offset, in filemap_fault()
3086 vmf->gfp_mask); in filemap_fault()
3090 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3099 if (unlikely(compound_head(page)->mapping != mapping)) { in filemap_fault()
3108 * that it's up-to-date. If not, it is going to be due to an error. in filemap_fault()
3127 * time to return to the upper layer and have it re-find the vma and in filemap_fault()
3135 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3148 vmf->page = page; in filemap_fault()
3153 * Umm, take care of errors if the page isn't up-to-date. in filemap_fault()
3154 * Try to re-read it _once_. We do this synchronously, in filemap_fault()
3159 error = filemap_read_page(file, mapping, page); in filemap_fault()
3166 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3173 * re-find the vma and come back and find our hopefully still populated in filemap_fault()
3179 filemap_invalidate_unlock_shared(mapping); in filemap_fault()
3188 struct mm_struct *mm = vmf->vma->vm_mm; in filemap_map_pmd()
3191 if (pmd_trans_huge(*vmf->pmd)) { in filemap_map_pmd()
3197 if (pmd_none(*vmf->pmd) && PageTransHuge(page)) { in filemap_map_pmd()
3206 if (pmd_none(*vmf->pmd)) { in filemap_map_pmd()
3207 vmf->ptl = pmd_lock(mm, vmf->pmd); in filemap_map_pmd()
3208 if (likely(pmd_none(*vmf->pmd))) { in filemap_map_pmd()
3210 pmd_populate(mm, vmf->pmd, vmf->prealloc_pte); in filemap_map_pmd()
3211 vmf->prealloc_pte = NULL; in filemap_map_pmd()
3213 spin_unlock(vmf->ptl); in filemap_map_pmd()
3217 if (pmd_devmap_trans_unstable(vmf->pmd)) { in filemap_map_pmd()
3227 struct address_space *mapping, in next_uptodate_page() argument
3252 if (page->mapping != mapping) in next_uptodate_page()
3256 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in next_uptodate_page()
3257 if (xas->xa_index >= max_idx) in next_uptodate_page()
3269 static inline struct page *first_map_page(struct address_space *mapping, in first_map_page() argument
3274 mapping, xas, end_pgoff); in first_map_page()
3277 static inline struct page *next_map_page(struct address_space *mapping, in next_map_page() argument
3282 mapping, xas, end_pgoff); in next_map_page()
3288 struct vm_area_struct *vma = vmf->vma; in filemap_map_pages()
3289 struct file *file = vma->vm_file; in filemap_map_pages()
3290 struct address_space *mapping = file->f_mapping; in filemap_map_pages() local
3293 XA_STATE(xas, &mapping->i_pages, start_pgoff); in filemap_map_pages()
3295 unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); in filemap_map_pages()
3299 head = first_map_page(mapping, &xas, end_pgoff); in filemap_map_pages()
3308 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); in filemap_map_pages()
3309 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); in filemap_map_pages()
3316 mmap_miss--; in filemap_map_pages()
3318 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; in filemap_map_pages()
3319 vmf->pte += xas.xa_index - last_pgoff; in filemap_map_pages()
3322 if (!pte_none(*vmf->pte)) in filemap_map_pages()
3326 if (vmf->address == addr) in filemap_map_pages()
3330 /* no need to invalidate: a not-present page won't be cached */ in filemap_map_pages()
3331 update_mmu_cache(vma, addr, vmf->pte); in filemap_map_pages()
3337 } while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL); in filemap_map_pages()
3338 pte_unmap_unlock(vmf->pte, vmf->ptl); in filemap_map_pages()
3341 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); in filemap_map_pages()
3348 struct address_space *mapping = vmf->vma->vm_file->f_mapping; in filemap_page_mkwrite() local
3349 struct page *page = vmf->page; in filemap_page_mkwrite()
3352 sb_start_pagefault(mapping->host->i_sb); in filemap_page_mkwrite()
3353 file_update_time(vmf->vma->vm_file); in filemap_page_mkwrite()
3355 if (page->mapping != mapping) { in filemap_page_mkwrite()
3368 sb_end_pagefault(mapping->host->i_sb); in filemap_page_mkwrite()
3382 struct address_space *mapping = file->f_mapping; in generic_file_mmap() local
3384 if (!mapping->a_ops->readpage) in generic_file_mmap()
3385 return -ENOEXEC; in generic_file_mmap()
3387 vma->vm_ops = &generic_file_vm_ops; in generic_file_mmap()
3392 * This is for filesystems which do not implement ->writepage.
3396 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) in generic_file_readonly_mmap()
3397 return -EINVAL; in generic_file_readonly_mmap()
3407 return -ENOSYS; in generic_file_mmap()
3411 return -ENOSYS; in generic_file_readonly_mmap()
3425 page = ERR_PTR(-EIO); in wait_on_page_read()
3431 static struct page *do_read_cache_page(struct address_space *mapping, in do_read_cache_page() argument
3434 void *data, in do_read_cache_page() argument
3440 page = find_get_page(mapping, index); in do_read_cache_page()
3444 return ERR_PTR(-ENOMEM); in do_read_cache_page()
3445 err = add_to_page_cache_lru(page, mapping, index, gfp); in do_read_cache_page()
3448 if (err == -EEXIST) in do_read_cache_page()
3456 err = filler(data, page); in do_read_cache_page()
3458 err = mapping->a_ops->readpage(data, page); in do_read_cache_page()
3483 * page is truncated, the data is still valid if PageUptodate as in do_read_cache_page()
3486 * Case c, the page may be truncated but in itself, the data may still in do_read_cache_page()
3489 * otherwise serialising on page lock to stabilise the mapping gives in do_read_cache_page()
3493 * will be a race with remove_mapping that determines if the mapping in do_read_cache_page()
3494 * is valid on unlock but otherwise the data is valid and there is in do_read_cache_page()
3512 if (!page->mapping) { in do_read_cache_page()
3539 * read_cache_page - read into page cache, fill it if needed
3540 * @mapping: the page's address_space
3543 * @data: first arg to filler(data, page) function, often left as NULL
3548 * If the page does not get brought uptodate, return -EIO.
3550 * The function expects mapping->invalidate_lock to be already held.
3554 struct page *read_cache_page(struct address_space *mapping, in read_cache_page() argument
3557 void *data) in read_cache_page() argument
3559 return do_read_cache_page(mapping, index, filler, data, in read_cache_page()
3560 mapping_gfp_mask(mapping)); in read_cache_page()
3565 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3566 * @mapping: the page's address_space
3570 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3573 * If the page does not get brought uptodate, return -EIO.
3575 * The function expects mapping->invalidate_lock to be already held.
3579 struct page *read_cache_page_gfp(struct address_space *mapping, in read_cache_page_gfp() argument
3583 return do_read_cache_page(mapping, index, NULL, NULL, gfp); in read_cache_page_gfp()
3587 int pagecache_write_begin(struct file *file, struct address_space *mapping, in pagecache_write_begin() argument
3591 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_begin()
3593 return aops->write_begin(file, mapping, pos, len, flags, in pagecache_write_begin()
3598 int pagecache_write_end(struct file *file, struct address_space *mapping, in pagecache_write_end() argument
3602 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_end()
3604 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); in pagecache_write_end()
3617 errseq_set(&filp->f_mapping->wb_err, -EIO); in dio_warn_stale_pagecache()
3622 …pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision… in dio_warn_stale_pagecache()
3623 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid, in dio_warn_stale_pagecache()
3624 current->comm); in dio_warn_stale_pagecache()
3631 struct file *file = iocb->ki_filp; in generic_file_direct_write()
3632 struct address_space *mapping = file->f_mapping; in generic_file_direct_write() local
3633 struct inode *inode = mapping->host; in generic_file_direct_write()
3634 loff_t pos = iocb->ki_pos; in generic_file_direct_write()
3640 end = (pos + write_len - 1) >> PAGE_SHIFT; in generic_file_direct_write()
3642 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_direct_write()
3644 if (filemap_range_has_page(file->f_mapping, pos, in generic_file_direct_write()
3645 pos + write_len - 1)) in generic_file_direct_write()
3646 return -EAGAIN; in generic_file_direct_write()
3648 written = filemap_write_and_wait_range(mapping, pos, in generic_file_direct_write()
3649 pos + write_len - 1); in generic_file_direct_write()
3656 * the new data. We invalidate clean cached page from the region we're in generic_file_direct_write()
3658 * without clobbering -EIOCBQUEUED from ->direct_IO(). in generic_file_direct_write()
3660 written = invalidate_inode_pages2_range(mapping, in generic_file_direct_write()
3667 if (written == -EBUSY) in generic_file_direct_write()
3672 written = mapping->a_ops->direct_IO(iocb, from); in generic_file_direct_write()
3676 * cached by non-direct readahead, or faulted in by get_user_pages() in generic_file_direct_write()
3689 * Skip invalidation for async writes or if mapping has no pages. in generic_file_direct_write()
3691 if (written > 0 && mapping->nrpages && in generic_file_direct_write()
3692 invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end)) in generic_file_direct_write()
3697 write_len -= written; in generic_file_direct_write()
3698 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { in generic_file_direct_write()
3702 iocb->ki_pos = pos; in generic_file_direct_write()
3704 if (written != -EIOCBQUEUED) in generic_file_direct_write()
3705 iov_iter_revert(from, write_len - iov_iter_count(from)); in generic_file_direct_write()
3715 struct page *grab_cache_page_write_begin(struct address_space *mapping, in grab_cache_page_write_begin() argument
3724 page = pagecache_get_page(mapping, index, fgp_flags, in grab_cache_page_write_begin()
3725 mapping_gfp_mask(mapping)); in grab_cache_page_write_begin()
3736 struct address_space *mapping = file->f_mapping; in generic_perform_write() local
3737 const struct address_space_operations *a_ops = mapping->a_ops; in generic_perform_write()
3749 offset = (pos & (PAGE_SIZE - 1)); in generic_perform_write()
3750 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3758 * up-to-date. in generic_perform_write()
3761 status = -EFAULT; in generic_perform_write()
3766 status = -EINTR; in generic_perform_write()
3770 status = a_ops->write_begin(file, mapping, pos, bytes, flags, in generic_perform_write()
3775 if (mapping_writably_mapped(mapping)) in generic_perform_write()
3781 status = a_ops->write_end(file, mapping, pos, bytes, copied, in generic_perform_write()
3784 iov_iter_revert(i, copied - max(status, 0L)); in generic_perform_write()
3792 * A short copy made ->write_end() reject the in generic_perform_write()
3804 balance_dirty_pages_ratelimited(mapping); in generic_perform_write()
3812 * __generic_file_write_iter - write data to a file
3814 * @from: iov_iter with data to write
3816 * This function does all the work needed for actually writing data to a
3824 * This function does *not* take care of syncing data in case of O_SYNC write.
3830 * * negative error code if no data has been written at all
3834 struct file *file = iocb->ki_filp; in __generic_file_write_iter()
3835 struct address_space *mapping = file->f_mapping; in __generic_file_write_iter() local
3836 struct inode *inode = mapping->host; in __generic_file_write_iter()
3842 current->backing_dev_info = inode_to_bdi(inode); in __generic_file_write_iter()
3851 if (iocb->ki_flags & IOCB_DIRECT) { in __generic_file_write_iter()
3860 * page-cache pages correctly). in __generic_file_write_iter()
3865 status = generic_perform_write(file, from, pos = iocb->ki_pos); in __generic_file_write_iter()
3869 * direct-written, or the error code if that was zero. Note in __generic_file_write_iter()
3870 * that this differs from normal direct-io semantics, which in __generic_file_write_iter()
3871 * will return -EFOO even if some bytes were written. in __generic_file_write_iter()
3882 endbyte = pos + status - 1; in __generic_file_write_iter()
3883 err = filemap_write_and_wait_range(mapping, pos, endbyte); in __generic_file_write_iter()
3885 iocb->ki_pos = endbyte + 1; in __generic_file_write_iter()
3887 invalidate_mapping_pages(mapping, in __generic_file_write_iter()
3893 * the number of bytes which were direct-written in __generic_file_write_iter()
3897 written = generic_perform_write(file, from, iocb->ki_pos); in __generic_file_write_iter()
3899 iocb->ki_pos += written; in __generic_file_write_iter()
3902 current->backing_dev_info = NULL; in __generic_file_write_iter()
3908 * generic_file_write_iter - write data to a file
3910 * @from: iov_iter with data to write
3916 * * negative error code if no data has been written at all of
3922 struct file *file = iocb->ki_filp; in generic_file_write_iter()
3923 struct inode *inode = file->f_mapping->host; in generic_file_write_iter()
3939 * try_to_release_page() - release old fs-specific metadata on a page
3944 * The address_space is to try to release any data against the page
3945 * (presumably at page->private).
3957 struct address_space * const mapping = page->mapping; in try_to_release_page() local
3963 if (mapping && mapping->a_ops->releasepage) in try_to_release_page()
3964 return mapping->a_ops->releasepage(page, gfp_mask); in try_to_release_page()