Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
28 #include <linux/error-injection.h>
31 #include <linux/backing-dev.h>
62 * finished 'unifying' the page and buffer cache and SMP-threaded the
63 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
65 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
71 * ->i_mmap_rwsem (truncate_pagecache)
72 * ->private_lock (__free_pte->__set_page_dirty_buffers)
73 * ->swap_lock (exclusive_swap_page, others)
74 * ->i_pages lock
76 * ->i_mutex
77 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
79 * ->mmap_sem
80 * ->i_mmap_rwsem
81 * ->page_table_lock or pte_lock (various, mainly in memory.c)
82 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
84 * ->mmap_sem
85 * ->lock_page (access_process_vm)
87 * ->i_mutex (generic_perform_write)
88 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
90 * bdi->wb.list_lock
91 * sb_lock (fs/fs-writeback.c)
92 * ->i_pages lock (__sync_single_inode)
94 * ->i_mmap_rwsem
95 * ->anon_vma.lock (vma_adjust)
97 * ->anon_vma.lock
98 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
100 * ->page_table_lock or pte_lock
101 * ->swap_lock (try_to_unmap_one)
102 * ->private_lock (try_to_unmap_one)
103 * ->i_pages lock (try_to_unmap_one)
104 * ->pgdat->lru_lock (follow_page->mark_page_accessed)
105 * ->pgdat->lru_lock (check_pte_range->isolate_lru_page)
106 * ->private_lock (page_remove_rmap->set_page_dirty)
107 * ->i_pages lock (page_remove_rmap->set_page_dirty)
108 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
109 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
110 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
111 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
112 * ->inode->i_lock (zap_pte_range->set_page_dirty)
113 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
115 * ->i_mmap_rwsem
116 * ->tasklist_lock (memory_failure, collect_procs_ao)
119 static void page_cache_delete(struct address_space *mapping, in page_cache_delete() argument
122 XA_STATE(xas, &mapping->i_pages, page->index); in page_cache_delete()
125 mapping_set_update(&xas, mapping); in page_cache_delete()
129 xas_set_order(&xas, page->index, compound_order(page)); in page_cache_delete()
140 page->mapping = NULL; in page_cache_delete()
141 /* Leave page->index set: truncation lookup relies upon it */ in page_cache_delete()
144 mapping->nrexceptional += nr; in page_cache_delete()
153 mapping->nrpages -= nr; in page_cache_delete()
156 static void unaccount_page_cache_page(struct address_space *mapping, in unaccount_page_cache_page() argument
164 * stale data around in the cleancache once our page is gone in unaccount_page_cache_page()
169 cleancache_invalidate_page(mapping, page); in unaccount_page_cache_page()
177 current->comm, page_to_pfn(page)); in unaccount_page_cache_page()
183 if (mapping_exiting(mapping) && in unaccount_page_cache_page()
202 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); in unaccount_page_cache_page()
204 __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr); in unaccount_page_cache_page()
209 filemap_nr_thps_dec(mapping); in unaccount_page_cache_page()
215 * unwritten data. in unaccount_page_cache_page()
223 account_page_cleaned(page, mapping, inode_to_wb(mapping->host)); in unaccount_page_cache_page()
228 * sure the page is locked and that nobody else uses it - or that usage
233 struct address_space *mapping = page->mapping; in __delete_from_page_cache() local
237 unaccount_page_cache_page(mapping, page); in __delete_from_page_cache()
238 page_cache_delete(mapping, page, shadow); in __delete_from_page_cache()
241 static void page_cache_free_page(struct address_space *mapping, in page_cache_free_page() argument
246 freepage = mapping->a_ops->freepage; in page_cache_free_page()
259 * delete_from_page_cache - delete page from page cache
268 struct address_space *mapping = page_mapping(page); in delete_from_page_cache() local
272 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache()
274 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache()
276 page_cache_free_page(mapping, page); in delete_from_page_cache()
281 * page_cache_delete_batch - delete several pages from page cache
282 * @mapping: the mapping to which pages belong
285 * The function walks over mapping->i_pages and removes pages passed in @pvec
286 * from the mapping. The function expects @pvec to be sorted by page index
288 * It tolerates holes in @pvec (mapping entries at those indices are not
294 static void page_cache_delete_batch(struct address_space *mapping, in page_cache_delete_batch() argument
297 XA_STATE(xas, &mapping->i_pages, pvec->pages[0]->index); in page_cache_delete_batch()
302 mapping_set_update(&xas, mapping); in page_cache_delete_batch()
317 if (page != pvec->pages[i]) { in page_cache_delete_batch()
318 VM_BUG_ON_PAGE(page->index > pvec->pages[i]->index, in page_cache_delete_batch()
325 if (page->index == xas.xa_index) in page_cache_delete_batch()
326 page->mapping = NULL; in page_cache_delete_batch()
327 /* Leave page->index set: truncation lookup relies on it */ in page_cache_delete_batch()
331 * page or the index is of the last sub-page of this compound in page_cache_delete_batch()
334 if (page->index + compound_nr(page) - 1 == xas.xa_index) in page_cache_delete_batch()
339 mapping->nrpages -= total_pages; in page_cache_delete_batch()
342 void delete_from_page_cache_batch(struct address_space *mapping, in delete_from_page_cache_batch() argument
351 xa_lock_irqsave(&mapping->i_pages, flags); in delete_from_page_cache_batch()
353 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); in delete_from_page_cache_batch()
355 unaccount_page_cache_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
357 page_cache_delete_batch(mapping, pvec); in delete_from_page_cache_batch()
358 xa_unlock_irqrestore(&mapping->i_pages, flags); in delete_from_page_cache_batch()
361 page_cache_free_page(mapping, pvec->pages[i]); in delete_from_page_cache_batch()
364 int filemap_check_errors(struct address_space *mapping) in filemap_check_errors() argument
368 if (test_bit(AS_ENOSPC, &mapping->flags) && in filemap_check_errors()
369 test_and_clear_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_errors()
370 ret = -ENOSPC; in filemap_check_errors()
371 if (test_bit(AS_EIO, &mapping->flags) && in filemap_check_errors()
372 test_and_clear_bit(AS_EIO, &mapping->flags)) in filemap_check_errors()
373 ret = -EIO; in filemap_check_errors()
378 static int filemap_check_and_keep_errors(struct address_space *mapping) in filemap_check_and_keep_errors() argument
381 if (test_bit(AS_EIO, &mapping->flags)) in filemap_check_and_keep_errors()
382 return -EIO; in filemap_check_and_keep_errors()
383 if (test_bit(AS_ENOSPC, &mapping->flags)) in filemap_check_and_keep_errors()
384 return -ENOSPC; in filemap_check_and_keep_errors()
389 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
390 * @mapping: address space structure to write
395 * Start writeback against all of a mapping's dirty pages that lie
398 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
405 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in __filemap_fdatawrite_range() argument
416 if (!mapping_cap_writeback_dirty(mapping) || in __filemap_fdatawrite_range()
417 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) in __filemap_fdatawrite_range()
420 wbc_attach_fdatawrite_inode(&wbc, mapping->host); in __filemap_fdatawrite_range()
421 ret = do_writepages(mapping, &wbc); in __filemap_fdatawrite_range()
426 static inline int __filemap_fdatawrite(struct address_space *mapping, in __filemap_fdatawrite() argument
429 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode); in __filemap_fdatawrite()
432 int filemap_fdatawrite(struct address_space *mapping) in filemap_fdatawrite() argument
434 return __filemap_fdatawrite(mapping, WB_SYNC_ALL); in filemap_fdatawrite()
438 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, in filemap_fdatawrite_range() argument
441 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL); in filemap_fdatawrite_range()
446 * filemap_flush - mostly a non-blocking flush
447 * @mapping: target address_space
449 * This is a mostly non-blocking flush. Not suitable for data-integrity
450 * purposes - I/O may not be started against all dirty pages.
454 int filemap_flush(struct address_space *mapping) in filemap_flush() argument
456 return __filemap_fdatawrite(mapping, WB_SYNC_NONE); in filemap_flush()
461 * filemap_range_has_page - check if a page exists in range.
462 * @mapping: address space within which to check
472 bool filemap_range_has_page(struct address_space *mapping, in filemap_range_has_page() argument
476 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); in filemap_range_has_page()
503 static void __filemap_fdatawait_range(struct address_space *mapping, in __filemap_fdatawait_range() argument
518 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, in __filemap_fdatawait_range()
535 * filemap_fdatawait_range - wait for writeback to complete
536 * @mapping: address space structure to wait for
540 * Walk the list of under-writeback pages of the given address space
550 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, in filemap_fdatawait_range() argument
553 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range()
554 return filemap_check_errors(mapping); in filemap_fdatawait_range()
559 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
560 * @mapping: address space structure to wait for
564 * Walk the list of under-writeback pages of the given address space in the
569 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
572 int filemap_fdatawait_range_keep_errors(struct address_space *mapping, in filemap_fdatawait_range_keep_errors() argument
575 __filemap_fdatawait_range(mapping, start_byte, end_byte); in filemap_fdatawait_range_keep_errors()
576 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_range_keep_errors()
581 * file_fdatawait_range - wait for writeback to complete
586 * Walk the list of under-writeback pages of the address space that file
588 * status of the address space vs. the file->f_wb_err cursor and return it.
594 * Return: error status of the address space vs. the file->f_wb_err cursor.
598 struct address_space *mapping = file->f_mapping; in file_fdatawait_range() local
600 __filemap_fdatawait_range(mapping, start_byte, end_byte); in file_fdatawait_range()
606 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
607 * @mapping: address space structure to wait for
609 * Walk the list of under-writeback pages of the given address space
614 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
619 int filemap_fdatawait_keep_errors(struct address_space *mapping) in filemap_fdatawait_keep_errors() argument
621 __filemap_fdatawait_range(mapping, 0, LLONG_MAX); in filemap_fdatawait_keep_errors()
622 return filemap_check_and_keep_errors(mapping); in filemap_fdatawait_keep_errors()
627 static bool mapping_needs_writeback(struct address_space *mapping) in mapping_needs_writeback() argument
629 if (dax_mapping(mapping)) in mapping_needs_writeback()
630 return mapping->nrexceptional; in mapping_needs_writeback()
632 return mapping->nrpages; in mapping_needs_writeback()
635 int filemap_write_and_wait(struct address_space *mapping) in filemap_write_and_wait() argument
639 if (mapping_needs_writeback(mapping)) { in filemap_write_and_wait()
640 err = filemap_fdatawrite(mapping); in filemap_write_and_wait()
643 * written partially (e.g. -ENOSPC), so we wait for it. in filemap_write_and_wait()
644 * But the -EIO is special case, it may indicate the worst in filemap_write_and_wait()
647 if (err != -EIO) { in filemap_write_and_wait()
648 int err2 = filemap_fdatawait(mapping); in filemap_write_and_wait()
653 filemap_check_errors(mapping); in filemap_write_and_wait()
656 err = filemap_check_errors(mapping); in filemap_write_and_wait()
663 * filemap_write_and_wait_range - write out & wait on a file range
664 * @mapping: the address_space for the pages
668 * Write out and wait upon file offsets lstart->lend, inclusive.
671 * that this function can be used to write to the very end-of-file (end = -1).
675 int filemap_write_and_wait_range(struct address_space *mapping, in filemap_write_and_wait_range() argument
680 if (mapping_needs_writeback(mapping)) { in filemap_write_and_wait_range()
681 err = __filemap_fdatawrite_range(mapping, lstart, lend, in filemap_write_and_wait_range()
684 if (err != -EIO) { in filemap_write_and_wait_range()
685 int err2 = filemap_fdatawait_range(mapping, in filemap_write_and_wait_range()
691 filemap_check_errors(mapping); in filemap_write_and_wait_range()
694 err = filemap_check_errors(mapping); in filemap_write_and_wait_range()
700 void __filemap_set_wb_err(struct address_space *mapping, int err) in __filemap_set_wb_err() argument
702 errseq_t eseq = errseq_set(&mapping->wb_err, err); in __filemap_set_wb_err()
704 trace_filemap_set_wb_err(mapping, eseq); in __filemap_set_wb_err()
709 * file_check_and_advance_wb_err - report wb error (if any) that was previously
717 * Grab the wb_err from the mapping. If it matches what we have in the file,
720 * If it doesn't match, then take the mapping value, set the "seen" flag in
726 * While we handle mapping->wb_err with atomic operations, the f_wb_err
735 errseq_t old = READ_ONCE(file->f_wb_err); in file_check_and_advance_wb_err()
736 struct address_space *mapping = file->f_mapping; in file_check_and_advance_wb_err() local
739 if (errseq_check(&mapping->wb_err, old)) { in file_check_and_advance_wb_err()
741 spin_lock(&file->f_lock); in file_check_and_advance_wb_err()
742 old = file->f_wb_err; in file_check_and_advance_wb_err()
743 err = errseq_check_and_advance(&mapping->wb_err, in file_check_and_advance_wb_err()
744 &file->f_wb_err); in file_check_and_advance_wb_err()
746 spin_unlock(&file->f_lock); in file_check_and_advance_wb_err()
754 clear_bit(AS_EIO, &mapping->flags); in file_check_and_advance_wb_err()
755 clear_bit(AS_ENOSPC, &mapping->flags); in file_check_and_advance_wb_err()
761 * file_write_and_wait_range - write out & wait on a file range
766 * Write out and wait upon file offsets lstart->lend, inclusive.
769 * that this function can be used to write to the very end-of-file (end = -1).
771 * After writing out and waiting on the data, we check and advance the
779 struct address_space *mapping = file->f_mapping; in file_write_and_wait_range() local
781 if (mapping_needs_writeback(mapping)) { in file_write_and_wait_range()
782 err = __filemap_fdatawrite_range(mapping, lstart, lend, in file_write_and_wait_range()
785 if (err != -EIO) in file_write_and_wait_range()
786 __filemap_fdatawait_range(mapping, lstart, lend); in file_write_and_wait_range()
796 * replace_page_cache_page - replace a pagecache page with a new one
813 struct address_space *mapping = old->mapping; in replace_page_cache_page() local
814 void (*freepage)(struct page *) = mapping->a_ops->freepage; in replace_page_cache_page()
815 pgoff_t offset = old->index; in replace_page_cache_page()
816 XA_STATE(xas, &mapping->i_pages, offset); in replace_page_cache_page()
821 VM_BUG_ON_PAGE(new->mapping, new); in replace_page_cache_page()
824 new->mapping = mapping; in replace_page_cache_page()
825 new->index = offset; in replace_page_cache_page()
830 old->mapping = NULL; in replace_page_cache_page()
851 struct address_space *mapping, in __add_to_page_cache_locked() argument
855 XA_STATE(xas, &mapping->i_pages, offset); in __add_to_page_cache_locked()
863 mapping_set_update(&xas, mapping); in __add_to_page_cache_locked()
866 error = mem_cgroup_try_charge(page, current->mm, in __add_to_page_cache_locked()
873 page->mapping = mapping; in __add_to_page_cache_locked()
874 page->index = offset; in __add_to_page_cache_locked()
880 xas_set_err(&xas, -EEXIST); in __add_to_page_cache_locked()
886 mapping->nrexceptional--; in __add_to_page_cache_locked()
890 mapping->nrpages++; in __add_to_page_cache_locked()
907 page->mapping = NULL; in __add_to_page_cache_locked()
908 /* Leave page->index set: truncation relies upon it */ in __add_to_page_cache_locked()
917 * add_to_page_cache_locked - add a locked page to the pagecache
919 * @mapping: the page's address_space
928 int add_to_page_cache_locked(struct page *page, struct address_space *mapping, in add_to_page_cache_locked() argument
931 return __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_locked()
936 int add_to_page_cache_lru(struct page *page, struct address_space *mapping, in add_to_page_cache_lru() argument
943 ret = __add_to_page_cache_locked(page, mapping, offset, in add_to_page_cache_lru()
953 * data from the working set, only to cache data that will in add_to_page_cache_lru()
1015 /* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
1034 if (wait_page->page != key->page) in wake_page_function()
1036 key->page_match = 1; in wake_page_function()
1038 if (wait_page->bit_nr != key->bit_nr) in wake_page_function()
1049 if (test_bit(key->bit_nr, &key->page->flags)) in wake_page_function()
1050 return -1; in wake_page_function()
1071 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1081 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1083 spin_lock_irqsave(&q->lock, flags); in wake_up_page_bit()
1089 * hash, so in that case check for a page match. That prevents a long- in wake_up_page_bit()
1106 spin_unlock_irqrestore(&q->lock, flags); in wake_up_page_bit()
1153 wait->flags = behavior == EXCLUSIVE ? WQ_FLAG_EXCLUSIVE : 0; in wait_on_page_bit_common()
1154 wait->func = wake_page_function; in wait_on_page_bit_common()
1159 spin_lock_irq(&q->lock); in wait_on_page_bit_common()
1161 if (likely(list_empty(&wait->entry))) { in wait_on_page_bit_common()
1168 spin_unlock_irq(&q->lock); in wait_on_page_bit_common()
1170 bit_is_set = test_bit(bit_nr, &page->flags); in wait_on_page_bit_common()
1178 if (!test_and_set_bit_lock(bit_nr, &page->flags)) in wait_on_page_bit_common()
1181 if (!test_bit(bit_nr, &page->flags)) in wait_on_page_bit_common()
1186 ret = -EINTR; in wait_on_page_bit_common()
1192 * We can no longer safely access page->flags: in wait_on_page_bit_common()
1196 * But best check for -EINTR above before breaking. in wait_on_page_bit_common()
1212 * !waitqueue_active would be possible (by open-coding finish_wait), in wait_on_page_bit_common()
1236 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1255 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1266 spin_lock_irqsave(&q->lock, flags); in add_page_wait_queue()
1269 spin_unlock_irqrestore(&q->lock, flags); in add_page_wait_queue()
1297 * unlock_page - unlock a locked page
1303 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1306 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1316 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags)) in unlock_page()
1322 * end_page_writeback - end writeback against a page
1363 struct address_space *mapping; in page_endio() local
1366 mapping = page_mapping(page); in page_endio()
1367 if (mapping) in page_endio()
1368 mapping_set_error(mapping, err); in page_endio()
1376 * __lock_page - get a lock on the page, assuming we need to sleep to get it
1399 * 1 - page is locked; mmap_sem is still held.
1400 * 0 - page is not locked.
1419 up_read(&mm->mmap_sem); in __lock_page_or_retry()
1431 up_read(&mm->mmap_sem); in __lock_page_or_retry()
1441 * page_cache_next_miss() - Find the next gap in the page cache.
1442 * @mapping: Mapping.
1446 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1456 * range specified (in which case 'return - index >= max_scan' will be true).
1457 * In the rare case of index wrap-around, 0 will be returned.
1459 pgoff_t page_cache_next_miss(struct address_space *mapping, in page_cache_next_miss() argument
1462 XA_STATE(xas, &mapping->i_pages, index); in page_cache_next_miss()
1464 while (max_scan--) { in page_cache_next_miss()
1477 * page_cache_prev_miss() - Find the previous gap in the page cache.
1478 * @mapping: Mapping.
1482 * Search the range [max(index - max_scan + 1, 0), index] for the
1492 * range specified (in which case 'index - return >= max_scan' will be true).
1493 * In the rare case of wrap-around, ULONG_MAX will be returned.
1495 pgoff_t page_cache_prev_miss(struct address_space *mapping, in page_cache_prev_miss() argument
1498 XA_STATE(xas, &mapping->i_pages, index); in page_cache_prev_miss()
1500 while (max_scan--) { in page_cache_prev_miss()
1513 * find_get_entry - find and get a page cache entry
1514 * @mapping: the address_space to search
1517 * Looks up the page cache slot at @mapping & @offset. If there is a
1525 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) in find_get_entry() argument
1527 XA_STATE(xas, &mapping->i_pages, offset); in find_get_entry()
1564 * find_lock_entry - locate, pin and lock a page cache entry
1565 * @mapping: the address_space to search
1568 * Looks up the page cache slot at @mapping & @offset. If there is a
1579 struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset) in find_lock_entry() argument
1584 page = find_get_entry(mapping, offset); in find_lock_entry()
1588 if (unlikely(page_mapping(page) != mapping)) { in find_lock_entry()
1600 * pagecache_get_page - find and get a page reference
1601 * @mapping: the address_space to search
1604 * @gfp_mask: gfp mask to use for the page cache data page allocation
1606 * Looks up the page cache slot at @mapping & @offset.
1612 * - FGP_ACCESSED: the page will be marked accessed
1613 * - FGP_LOCK: Page is return locked
1614 * - FGP_CREAT: If page is not present then a new page is allocated using
1618 * - FGP_FOR_MMAP: Similar to FGP_CREAT, only we want to allow the caller to do
1629 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, in pagecache_get_page() argument
1635 page = find_get_entry(mapping, offset); in pagecache_get_page()
1652 if (unlikely(compound_head(page)->mapping != mapping)) { in pagecache_get_page()
1657 VM_BUG_ON_PAGE(page->index != offset, page); in pagecache_get_page()
1666 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping)) in pagecache_get_page()
1682 err = add_to_page_cache_lru(page, mapping, offset, gfp_mask); in pagecache_get_page()
1686 if (err == -EEXIST) in pagecache_get_page()
1703 * find_get_entries - gang pagecache lookup
1704 * @mapping: The address_space to search
1711 * @nr_entries entries in the mapping. The entries are placed at
1715 * The search returns a group of mapping-contiguous page cache entries
1717 * not-present pages.
1724 unsigned find_get_entries(struct address_space *mapping, in find_get_entries() argument
1728 XA_STATE(xas, &mapping->i_pages, start); in find_get_entries()
1771 * find_get_pages_range - gang pagecache lookup
1772 * @mapping: The address_space to search
1779 * pages in the mapping starting at index @start and up to index @end
1783 * The search returns a group of mapping-contiguous pages with ascending
1784 * indexes. There may be holes in the indices due to not-present pages.
1791 unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, in find_get_pages_range() argument
1795 XA_STATE(xas, &mapping->i_pages, *start); in find_get_pages_range()
1832 * breaks the iteration when there is a page at index -1 but that is in find_get_pages_range()
1835 if (end == (pgoff_t)-1) in find_get_pages_range()
1836 *start = (pgoff_t)-1; in find_get_pages_range()
1846 * find_get_pages_contig - gang contiguous pagecache lookup
1847 * @mapping: The address_space to search
1857 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, in find_get_pages_contig() argument
1860 XA_STATE(xas, &mapping->i_pages, index); in find_get_pages_contig()
1900 * find_get_pages_range_tag - find and return pages in given range matching @tag
1901 * @mapping: the address_space to search
1913 unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, in find_get_pages_range_tag() argument
1917 XA_STATE(xas, &mapping->i_pages, *index); in find_get_pages_range_tag()
1958 * iteration when there is a page at index -1 but that is already in find_get_pages_range_tag()
1961 if (end == (pgoff_t)-1) in find_get_pages_range_tag()
1962 *index = (pgoff_t)-1; in find_get_pages_range_tag()
1976 * ---R__________________________________________B__________
1990 ra->ra_pages /= 4; in shrink_readahead_size_eio()
1994 * generic_file_buffered_read - generic file read routine
1996 * @iter: data destination
2000 * mapping->a_ops->readpage() function for the actual low-level stuff.
2012 struct file *filp = iocb->ki_filp; in generic_file_buffered_read()
2013 struct address_space *mapping = filp->f_mapping; in generic_file_buffered_read() local
2014 struct inode *inode = mapping->host; in generic_file_buffered_read()
2015 struct file_ra_state *ra = &filp->f_ra; in generic_file_buffered_read()
2016 loff_t *ppos = &iocb->ki_pos; in generic_file_buffered_read()
2024 if (unlikely(*ppos >= inode->i_sb->s_maxbytes)) in generic_file_buffered_read()
2026 iov_iter_truncate(iter, inode->i_sb->s_maxbytes); in generic_file_buffered_read()
2029 prev_index = ra->prev_pos >> PAGE_SHIFT; in generic_file_buffered_read()
2030 prev_offset = ra->prev_pos & (PAGE_SIZE-1); in generic_file_buffered_read()
2031 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT; in generic_file_buffered_read()
2043 error = -EINTR; in generic_file_buffered_read()
2047 page = find_get_page(mapping, index); in generic_file_buffered_read()
2049 if (iocb->ki_flags & IOCB_NOWAIT) in generic_file_buffered_read()
2051 page_cache_sync_readahead(mapping, in generic_file_buffered_read()
2053 index, last_index - index); in generic_file_buffered_read()
2054 page = find_get_page(mapping, index); in generic_file_buffered_read()
2059 page_cache_async_readahead(mapping, in generic_file_buffered_read()
2061 index, last_index - index); in generic_file_buffered_read()
2064 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_buffered_read()
2080 if (inode->i_blkbits == PAGE_SHIFT || in generic_file_buffered_read()
2081 !mapping->a_ops->is_partially_uptodate) in generic_file_buffered_read()
2089 if (!page->mapping) in generic_file_buffered_read()
2091 if (!mapping->a_ops->is_partially_uptodate(page, in generic_file_buffered_read()
2092 offset, iter->count)) in generic_file_buffered_read()
2101 * the correct value for "nr", which means the zero-filled in generic_file_buffered_read()
2103 * another truncate extends the file - this is desired though). in generic_file_buffered_read()
2107 end_index = (isize - 1) >> PAGE_SHIFT; in generic_file_buffered_read()
2116 nr = ((isize - 1) & ~PAGE_MASK) + 1; in generic_file_buffered_read()
2122 nr = nr - offset; in generic_file_buffered_read()
2128 if (mapping_writably_mapped(mapping)) in generic_file_buffered_read()
2140 * Ok, we have the page, and it's up-to-date, so in generic_file_buffered_read()
2155 error = -EFAULT; in generic_file_buffered_read()
2168 if (!page->mapping) { in generic_file_buffered_read()
2188 error = mapping->a_ops->readpage(filp, page); in generic_file_buffered_read()
2204 if (page->mapping == NULL) { in generic_file_buffered_read()
2214 error = -EIO; in generic_file_buffered_read()
2232 page = page_cache_alloc(mapping); in generic_file_buffered_read()
2234 error = -ENOMEM; in generic_file_buffered_read()
2237 error = add_to_page_cache_lru(page, mapping, index, in generic_file_buffered_read()
2238 mapping_gfp_constraint(mapping, GFP_KERNEL)); in generic_file_buffered_read()
2241 if (error == -EEXIST) { in generic_file_buffered_read()
2251 error = -EAGAIN; in generic_file_buffered_read()
2253 ra->prev_pos = prev_index; in generic_file_buffered_read()
2254 ra->prev_pos <<= PAGE_SHIFT; in generic_file_buffered_read()
2255 ra->prev_pos |= prev_offset; in generic_file_buffered_read()
2263 * generic_file_read_iter - generic filesystem read routine
2265 * @iter: destination for the data read
2282 if (iocb->ki_flags & IOCB_DIRECT) { in generic_file_read_iter()
2283 struct file *file = iocb->ki_filp; in generic_file_read_iter()
2284 struct address_space *mapping = file->f_mapping; in generic_file_read_iter() local
2285 struct inode *inode = mapping->host; in generic_file_read_iter()
2289 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_read_iter()
2290 if (filemap_range_has_page(mapping, iocb->ki_pos, in generic_file_read_iter()
2291 iocb->ki_pos + count - 1)) in generic_file_read_iter()
2292 return -EAGAIN; in generic_file_read_iter()
2294 retval = filemap_write_and_wait_range(mapping, in generic_file_read_iter()
2295 iocb->ki_pos, in generic_file_read_iter()
2296 iocb->ki_pos + count - 1); in generic_file_read_iter()
2303 retval = mapping->a_ops->direct_IO(iocb, iter); in generic_file_read_iter()
2305 iocb->ki_pos += retval; in generic_file_read_iter()
2306 count -= retval; in generic_file_read_iter()
2308 iov_iter_revert(iter, count - iov_iter_count(iter)); in generic_file_read_iter()
2319 if (retval < 0 || !count || iocb->ki_pos >= size || in generic_file_read_iter()
2335 int flags = vmf->flags; in maybe_unlock_mmap_for_io()
2347 fpin = get_file(vmf->vma->vm_file); in maybe_unlock_mmap_for_io()
2348 up_read(&vmf->vma->vm_mm->mmap_sem); in maybe_unlock_mmap_for_io()
2354 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem
2355 * @vmf - the vm_fault for this fault.
2356 * @page - the page to lock.
2357 * @fpin - the pointer to the file we may pin (or is already pinned).
2375 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_page_maybe_drop_mmap()
2379 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_page_maybe_drop_mmap()
2388 up_read(&vmf->vma->vm_mm->mmap_sem); in lock_page_maybe_drop_mmap()
2406 struct file *file = vmf->vma->vm_file; in do_sync_mmap_readahead()
2407 struct file_ra_state *ra = &file->f_ra; in do_sync_mmap_readahead()
2408 struct address_space *mapping = file->f_mapping; in do_sync_mmap_readahead() local
2410 pgoff_t offset = vmf->pgoff; in do_sync_mmap_readahead()
2412 /* If we don't want any read-ahead, don't bother */ in do_sync_mmap_readahead()
2413 if (vmf->vma->vm_flags & VM_RAND_READ) in do_sync_mmap_readahead()
2415 if (!ra->ra_pages) in do_sync_mmap_readahead()
2418 if (vmf->vma->vm_flags & VM_SEQ_READ) { in do_sync_mmap_readahead()
2420 page_cache_sync_readahead(mapping, ra, file, offset, in do_sync_mmap_readahead()
2421 ra->ra_pages); in do_sync_mmap_readahead()
2426 if (ra->mmap_miss < MMAP_LOTSAMISS * 10) in do_sync_mmap_readahead()
2427 ra->mmap_miss++; in do_sync_mmap_readahead()
2431 * stop bothering with read-ahead. It will only hurt. in do_sync_mmap_readahead()
2433 if (ra->mmap_miss > MMAP_LOTSAMISS) in do_sync_mmap_readahead()
2437 * mmap read-around in do_sync_mmap_readahead()
2440 ra->start = max_t(long, 0, offset - ra->ra_pages / 2); in do_sync_mmap_readahead()
2441 ra->size = ra->ra_pages; in do_sync_mmap_readahead()
2442 ra->async_size = ra->ra_pages / 4; in do_sync_mmap_readahead()
2443 ra_submit(ra, mapping, file); in do_sync_mmap_readahead()
2455 struct file *file = vmf->vma->vm_file; in do_async_mmap_readahead()
2456 struct file_ra_state *ra = &file->f_ra; in do_async_mmap_readahead()
2457 struct address_space *mapping = file->f_mapping; in do_async_mmap_readahead() local
2459 pgoff_t offset = vmf->pgoff; in do_async_mmap_readahead()
2461 /* If we don't want any read-ahead, don't bother */ in do_async_mmap_readahead()
2462 if (vmf->vma->vm_flags & VM_RAND_READ) in do_async_mmap_readahead()
2464 if (ra->mmap_miss > 0) in do_async_mmap_readahead()
2465 ra->mmap_miss--; in do_async_mmap_readahead()
2468 page_cache_async_readahead(mapping, ra, file, in do_async_mmap_readahead()
2469 page, offset, ra->ra_pages); in do_async_mmap_readahead()
2475 * filemap_fault - read in file data for page fault handling
2479 * mapped memory region to read in file data during a page fault.
2485 * vma->vm_mm->mmap_sem must be held on entry.
2495 * Return: bitwise-OR of %VM_FAULT_ codes.
2500 struct file *file = vmf->vma->vm_file; in filemap_fault()
2502 struct address_space *mapping = file->f_mapping; in filemap_fault() local
2503 struct file_ra_state *ra = &file->f_ra; in filemap_fault()
2504 struct inode *inode = mapping->host; in filemap_fault()
2505 pgoff_t offset = vmf->pgoff; in filemap_fault()
2517 page = find_get_page(mapping, offset); in filemap_fault()
2518 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) { in filemap_fault()
2527 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); in filemap_fault()
2531 page = pagecache_get_page(mapping, offset, in filemap_fault()
2533 vmf->gfp_mask); in filemap_fault()
2537 return vmf_error(-ENOMEM); in filemap_fault()
2545 if (unlikely(compound_head(page)->mapping != mapping)) { in filemap_fault()
2554 * that it's up-to-date. If not, it is going to be due to an error. in filemap_fault()
2561 * time to return to the upper layer and have it re-find the vma and in filemap_fault()
2580 vmf->page = page; in filemap_fault()
2585 * Umm, take care of errors if the page isn't up-to-date. in filemap_fault()
2586 * Try to re-read it _once_. We do this synchronously, in filemap_fault()
2592 error = mapping->a_ops->readpage(file, page); in filemap_fault()
2596 error = -EIO; in filemap_fault()
2612 * re-find the vma and come back and find our hopefully still populated in filemap_fault()
2626 struct file *file = vmf->vma->vm_file; in filemap_map_pages()
2627 struct address_space *mapping = file->f_mapping; in filemap_map_pages() local
2630 XA_STATE(xas, &mapping->i_pages, start_pgoff); in filemap_map_pages()
2661 if (page->mapping != mapping || !PageUptodate(page)) in filemap_map_pages()
2664 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); in filemap_map_pages()
2665 if (page->index >= max_idx) in filemap_map_pages()
2668 if (file->f_ra.mmap_miss > 0) in filemap_map_pages()
2669 file->f_ra.mmap_miss--; in filemap_map_pages()
2671 vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; in filemap_map_pages()
2672 if (vmf->pte) in filemap_map_pages()
2673 vmf->pte += xas.xa_index - last_pgoff; in filemap_map_pages()
2685 if (pmd_trans_huge(*vmf->pmd)) in filemap_map_pages()
2694 struct page *page = vmf->page; in filemap_page_mkwrite()
2695 struct inode *inode = file_inode(vmf->vma->vm_file); in filemap_page_mkwrite()
2698 sb_start_pagefault(inode->i_sb); in filemap_page_mkwrite()
2699 file_update_time(vmf->vma->vm_file); in filemap_page_mkwrite()
2701 if (page->mapping != inode->i_mapping) { in filemap_page_mkwrite()
2714 sb_end_pagefault(inode->i_sb); in filemap_page_mkwrite()
2728 struct address_space *mapping = file->f_mapping; in generic_file_mmap() local
2730 if (!mapping->a_ops->readpage) in generic_file_mmap()
2731 return -ENOEXEC; in generic_file_mmap()
2733 vma->vm_ops = &generic_file_vm_ops; in generic_file_mmap()
2738 * This is for filesystems which do not implement ->writepage.
2742 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) in generic_file_readonly_mmap()
2743 return -EINVAL; in generic_file_readonly_mmap()
2753 return -ENOSYS; in generic_file_mmap()
2757 return -ENOSYS; in generic_file_readonly_mmap()
2771 page = ERR_PTR(-EIO); in wait_on_page_read()
2777 static struct page *do_read_cache_page(struct address_space *mapping, in do_read_cache_page() argument
2780 void *data, in do_read_cache_page() argument
2786 page = find_get_page(mapping, index); in do_read_cache_page()
2790 return ERR_PTR(-ENOMEM); in do_read_cache_page()
2791 err = add_to_page_cache_lru(page, mapping, index, gfp); in do_read_cache_page()
2794 if (err == -EEXIST) in do_read_cache_page()
2802 err = filler(data, page); in do_read_cache_page()
2804 err = mapping->a_ops->readpage(data, page); in do_read_cache_page()
2829 * the page is truncated, the data is still valid if PageUptodate as in do_read_cache_page()
2832 * Case c, the page may be truncated but in itself, the data may still in do_read_cache_page()
2835 * otherwise serialising on page lock to stabilise the mapping gives in do_read_cache_page()
2839 * will be a race with remove_mapping that determines if the mapping in do_read_cache_page()
2840 * is valid on unlock but otherwise the data is valid and there is in do_read_cache_page()
2858 if (!page->mapping) { in do_read_cache_page()
2877 * read_cache_page - read into page cache, fill it if needed
2878 * @mapping: the page's address_space
2881 * @data: first arg to filler(data, page) function, often left as NULL
2886 * If the page does not get brought uptodate, return -EIO.
2890 struct page *read_cache_page(struct address_space *mapping, in read_cache_page() argument
2893 void *data) in read_cache_page() argument
2895 return do_read_cache_page(mapping, index, filler, data, in read_cache_page()
2896 mapping_gfp_mask(mapping)); in read_cache_page()
2901 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
2902 * @mapping: the page's address_space
2906 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
2909 * If the page does not get brought uptodate, return -EIO.
2913 struct page *read_cache_page_gfp(struct address_space *mapping, in read_cache_page_gfp() argument
2917 return do_read_cache_page(mapping, index, NULL, NULL, gfp); in read_cache_page_gfp()
2924 * exceeds the limit we return -EFBIG.
2929 struct inode *inode = file->f_mapping->host; in generic_write_check_limits()
2930 loff_t max_size = inode->i_sb->s_maxbytes; in generic_write_check_limits()
2936 return -EFBIG; in generic_write_check_limits()
2938 *count = min(*count, limit - pos); in generic_write_check_limits()
2941 if (!(file->f_flags & O_LARGEFILE)) in generic_write_check_limits()
2945 return -EFBIG; in generic_write_check_limits()
2947 *count = min(*count, max_size - pos); in generic_write_check_limits()
2961 struct file *file = iocb->ki_filp; in generic_write_checks()
2962 struct inode *inode = file->f_mapping->host; in generic_write_checks()
2967 return -ETXTBSY; in generic_write_checks()
2973 if (iocb->ki_flags & IOCB_APPEND) in generic_write_checks()
2974 iocb->ki_pos = i_size_read(inode); in generic_write_checks()
2976 if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) in generic_write_checks()
2977 return -EINVAL; in generic_write_checks()
2980 ret = generic_write_check_limits(file, iocb->ki_pos, &count); in generic_write_checks()
3000 struct inode *inode_in = file_in->f_mapping->host; in generic_remap_checks()
3001 struct inode *inode_out = file_out->f_mapping->host; in generic_remap_checks()
3005 loff_t bs = inode_out->i_sb->s_blocksize; in generic_remap_checks()
3010 return -EINVAL; in generic_remap_checks()
3014 return -EINVAL; in generic_remap_checks()
3023 return -EINVAL; in generic_remap_checks()
3027 return -EINVAL; in generic_remap_checks()
3028 count = min(count, size_in - (uint64_t)pos_in); in generic_remap_checks()
3038 * Otherwise, make sure the count is also block-aligned, having in generic_remap_checks()
3042 bcount = ALIGN(size_in, bs) - pos_in; in generic_remap_checks()
3053 return -EINVAL; in generic_remap_checks()
3060 return -EINVAL; in generic_remap_checks()
3077 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) in generic_file_rw_checks()
3078 return -EISDIR; in generic_file_rw_checks()
3079 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) in generic_file_rw_checks()
3080 return -EINVAL; in generic_file_rw_checks()
3082 if (!(file_in->f_mode & FMODE_READ) || in generic_file_rw_checks()
3083 !(file_out->f_mode & FMODE_WRITE) || in generic_file_rw_checks()
3084 (file_out->f_flags & O_APPEND)) in generic_file_rw_checks()
3085 return -EBADF; in generic_file_rw_checks()
3113 return -EPERM; in generic_copy_file_checks()
3116 return -ETXTBSY; in generic_copy_file_checks()
3120 return -EOVERFLOW; in generic_copy_file_checks()
3127 count = min(count, size_in - (uint64_t)pos_in); in generic_copy_file_checks()
3137 return -EINVAL; in generic_copy_file_checks()
3143 int pagecache_write_begin(struct file *file, struct address_space *mapping, in pagecache_write_begin() argument
3147 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_begin()
3149 return aops->write_begin(file, mapping, pos, len, flags, in pagecache_write_begin()
3154 int pagecache_write_end(struct file *file, struct address_space *mapping, in pagecache_write_end() argument
3158 const struct address_space_operations *aops = mapping->a_ops; in pagecache_write_end()
3160 return aops->write_end(file, mapping, pos, len, copied, page, fsdata); in pagecache_write_end()
3167 struct file *file = iocb->ki_filp; in generic_file_direct_write()
3168 struct address_space *mapping = file->f_mapping; in generic_file_direct_write() local
3169 struct inode *inode = mapping->host; in generic_file_direct_write()
3170 loff_t pos = iocb->ki_pos; in generic_file_direct_write()
3176 end = (pos + write_len - 1) >> PAGE_SHIFT; in generic_file_direct_write()
3178 if (iocb->ki_flags & IOCB_NOWAIT) { in generic_file_direct_write()
3180 if (filemap_range_has_page(inode->i_mapping, pos, in generic_file_direct_write()
3181 pos + write_len - 1)) in generic_file_direct_write()
3182 return -EAGAIN; in generic_file_direct_write()
3184 written = filemap_write_and_wait_range(mapping, pos, in generic_file_direct_write()
3185 pos + write_len - 1); in generic_file_direct_write()
3192 * the new data. We invalidate clean cached page from the region we're in generic_file_direct_write()
3194 * without clobbering -EIOCBQUEUED from ->direct_IO(). in generic_file_direct_write()
3196 written = invalidate_inode_pages2_range(mapping, in generic_file_direct_write()
3203 if (written == -EBUSY) in generic_file_direct_write()
3208 written = mapping->a_ops->direct_IO(iocb, from); in generic_file_direct_write()
3212 * cached by non-direct readahead, or faulted in by get_user_pages() in generic_file_direct_write()
3223 if (mapping->nrpages) in generic_file_direct_write()
3224 invalidate_inode_pages2_range(mapping, in generic_file_direct_write()
3229 write_len -= written; in generic_file_direct_write()
3230 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { in generic_file_direct_write()
3234 iocb->ki_pos = pos; in generic_file_direct_write()
3236 iov_iter_revert(from, write_len - iov_iter_count(from)); in generic_file_direct_write()
3246 struct page *grab_cache_page_write_begin(struct address_space *mapping, in grab_cache_page_write_begin() argument
3255 page = pagecache_get_page(mapping, index, fgp_flags, in grab_cache_page_write_begin()
3256 mapping_gfp_mask(mapping)); in grab_cache_page_write_begin()
3267 struct address_space *mapping = file->f_mapping; in generic_perform_write() local
3268 const struct address_space_operations *a_ops = mapping->a_ops; in generic_perform_write()
3280 offset = (pos & (PAGE_SIZE - 1)); in generic_perform_write()
3281 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3289 * up-to-date. in generic_perform_write()
3296 status = -EFAULT; in generic_perform_write()
3301 status = -EINTR; in generic_perform_write()
3305 status = a_ops->write_begin(file, mapping, pos, bytes, flags, in generic_perform_write()
3310 if (mapping_writably_mapped(mapping)) in generic_perform_write()
3316 status = a_ops->write_end(file, mapping, pos, bytes, copied, in generic_perform_write()
3327 * If we were unable to copy any data at all, we must in generic_perform_write()
3334 bytes = min_t(unsigned long, PAGE_SIZE - offset, in generic_perform_write()
3341 balance_dirty_pages_ratelimited(mapping); in generic_perform_write()
3349 * __generic_file_write_iter - write data to a file
3351 * @from: iov_iter with data to write
3353 * This function does all the work needed for actually writing data to a
3361 * This function does *not* take care of syncing data in case of O_SYNC write.
3367 * * negative error code if no data has been written at all
3371 struct file *file = iocb->ki_filp; in __generic_file_write_iter()
3372 struct address_space * mapping = file->f_mapping; in __generic_file_write_iter() local
3373 struct inode *inode = mapping->host; in __generic_file_write_iter()
3379 current->backing_dev_info = inode_to_bdi(inode); in __generic_file_write_iter()
3388 if (iocb->ki_flags & IOCB_DIRECT) { in __generic_file_write_iter()
3397 * page-cache pages correctly). in __generic_file_write_iter()
3402 status = generic_perform_write(file, from, pos = iocb->ki_pos); in __generic_file_write_iter()
3406 * direct-written, or the error code if that was zero. Note in __generic_file_write_iter()
3407 * that this differs from normal direct-io semantics, which in __generic_file_write_iter()
3408 * will return -EFOO even if some bytes were written. in __generic_file_write_iter()
3419 endbyte = pos + status - 1; in __generic_file_write_iter()
3420 err = filemap_write_and_wait_range(mapping, pos, endbyte); in __generic_file_write_iter()
3422 iocb->ki_pos = endbyte + 1; in __generic_file_write_iter()
3424 invalidate_mapping_pages(mapping, in __generic_file_write_iter()
3430 * the number of bytes which were direct-written in __generic_file_write_iter()
3434 written = generic_perform_write(file, from, iocb->ki_pos); in __generic_file_write_iter()
3436 iocb->ki_pos += written; in __generic_file_write_iter()
3439 current->backing_dev_info = NULL; in __generic_file_write_iter()
3445 * generic_file_write_iter - write data to a file
3447 * @from: iov_iter with data to write
3453 * * negative error code if no data has been written at all of
3459 struct file *file = iocb->ki_filp; in generic_file_write_iter()
3460 struct inode *inode = file->f_mapping->host; in generic_file_write_iter()
3476 * try_to_release_page() - release old fs-specific metadata on a page
3481 * The address_space is to try to release any data against the page
3482 * (presumably at page->private).
3494 struct address_space * const mapping = page->mapping; in try_to_release_page() local
3500 if (mapping && mapping->a_ops->releasepage) in try_to_release_page()
3501 return mapping->a_ops->releasepage(page, gfp_mask); in try_to_release_page()