Lines Matching full:page
27 #include <linux/page-flags.h>
101 #include <asm/page.h>
129 * a zero page mapping on a read fault.
132 * related to the physical page in case of virtualization.
145 /* This function must be updated when the size of struct page grows above 80
152 static inline void __mm_zero_struct_page(struct page *page) in __mm_zero_struct_page() argument
154 unsigned long *_pp = (void *)page; in __mm_zero_struct_page()
156 /* Check that struct page is either 56, 64, 72, or 80 bytes */ in __mm_zero_struct_page()
157 BUILD_BUG_ON(sizeof(struct page) & 7); in __mm_zero_struct_page()
158 BUILD_BUG_ON(sizeof(struct page) < 56); in __mm_zero_struct_page()
159 BUILD_BUG_ON(sizeof(struct page) > 80); in __mm_zero_struct_page()
161 switch (sizeof(struct page)) { in __mm_zero_struct_page()
182 #define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
224 int __add_to_page_cache_locked(struct page *page, struct address_space *mapping,
228 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) argument
230 #define nth_page(page,n) ((page) + (n)) argument
233 /* to align the pointer to the (next) page boundary */
239 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
283 #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
298 #define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
299 #define VM_SYNC 0x00800000 /* Synchronous page faults */
310 #define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
353 # define VM_ARM64_BTI VM_ARCH_1 /* BTI guarded page, a.k.a. GP bit */
433 * low four bits) to a page protection mask..
451 * whether we would allow page faults to retry by specifying these two
454 * (a) ALLOW_RETRY and !TRIED: this means the page fault allows retry, and
457 * (b) ALLOW_RETRY and TRIED: this means the page fault allows retry, and
460 * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry
463 * be used. Note that page faults can be allowed to retry for multiple times,
466 * signals before a retry to make sure the continuous page faults can still be
484 * arch-specific page fault handlers.
500 * Return: true if the page fault allows retry and this is the first
526 * MM layer fills up gfp_mask for page allocations but fault handler might
535 pgoff_t pgoff; /* Logical page offset based on vma */
552 struct page *cow_page; /* Page handler may use for COW fault */
553 struct page *page; /* ->fault handlers should return a member
554 * page here, unless VM_FAULT_NOPAGE
560 * the 'address'. NULL if the page
563 spinlock_t *ptl; /* Page table lock.
564 * Protects pte page table if 'pte'
567 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
568 * vm_ops->map_pages() sets up a page
571 * page table to avoid allocation from
576 /* page entry size for vm->huge_fault() */
586 * to the functions called when a no-page or a wp-page exception occurs.
608 /* notification that a previously read-only page is about to become
652 * page for @addr. This is useful if the default behavior
653 * (using pte_page()) would not find the correct page.
655 struct page *(*find_special_page)(struct vm_area_struct *vma,
730 * Methods to modify the page usage count.
732 * What counts for a page usage:
733 * - cache mapping (page->mapping)
734 * - private data (page->private)
735 * - page mapped in a task's page tables, each mapping
738 * Also, many kernel routines increase the page count before a critical
739 * routine so they can be sure the page doesn't go away from under them.
743 * Drop a ref, return true if the refcount fell to zero (the page has no users)
745 static inline int put_page_testzero(struct page *page) in put_page_testzero() argument
747 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); in put_page_testzero()
748 return page_ref_dec_and_test(page); in put_page_testzero()
752 * Try to grab a ref unless the page has a refcount of zero, return false if
757 static inline int get_page_unless_zero(struct page *page) in get_page_unless_zero() argument
759 return page_ref_add_unless(page, 1, 0); in get_page_unless_zero()
774 struct page *vmalloc_to_page(const void *addr);
836 static inline int head_compound_mapcount(struct page *head) in head_compound_mapcount()
842 * Mapcount of compound page as a whole, does not include mapped sub-pages.
846 static inline int compound_mapcount(struct page *page) in compound_mapcount() argument
848 VM_BUG_ON_PAGE(!PageCompound(page), page); in compound_mapcount()
849 page = compound_head(page); in compound_mapcount()
850 return head_compound_mapcount(page); in compound_mapcount()
854 * The atomic page->_mapcount, starts from -1: so that transitions
858 static inline void page_mapcount_reset(struct page *page) in page_mapcount_reset() argument
860 atomic_set(&(page)->_mapcount, -1); in page_mapcount_reset()
863 int __page_mapcount(struct page *page);
866 * Mapcount of 0-order page; when compound sub-page, includes
871 * They use this place in struct page differently.
873 static inline int page_mapcount(struct page *page) in page_mapcount() argument
875 if (unlikely(PageCompound(page))) in page_mapcount()
876 return __page_mapcount(page); in page_mapcount()
877 return atomic_read(&page->_mapcount) + 1; in page_mapcount()
881 int total_mapcount(struct page *page);
882 int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
884 static inline int total_mapcount(struct page *page) in total_mapcount() argument
886 return page_mapcount(page); in total_mapcount()
888 static inline int page_trans_huge_mapcount(struct page *page, in page_trans_huge_mapcount() argument
891 int mapcount = page_mapcount(page); in page_trans_huge_mapcount()
898 static inline struct page *virt_to_head_page(const void *x) in virt_to_head_page()
900 struct page *page = virt_to_page(x); in virt_to_head_page() local
902 return compound_head(page); in virt_to_head_page()
905 void __put_page(struct page *page);
909 void split_page(struct page *page, unsigned int order);
910 void copy_huge_page(struct page *dst, struct page *src);
915 * These are _only_ valid on the head of a compound page.
917 typedef void compound_page_dtor(struct page *);
933 static inline void set_compound_page_dtor(struct page *page, in set_compound_page_dtor() argument
936 VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page); in set_compound_page_dtor()
937 page[1].compound_dtor = compound_dtor; in set_compound_page_dtor()
940 static inline void destroy_compound_page(struct page *page) in destroy_compound_page() argument
942 VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page); in destroy_compound_page()
943 compound_page_dtors[page[1].compound_dtor](page); in destroy_compound_page()
946 static inline unsigned int compound_order(struct page *page) in compound_order() argument
948 if (!PageHead(page)) in compound_order()
950 return page[1].compound_order; in compound_order()
953 static inline bool hpage_pincount_available(struct page *page) in hpage_pincount_available() argument
956 * Can the page->hpage_pinned_refcount field be used? That field is in in hpage_pincount_available()
957 * the 3rd page of the compound page, so the smallest (2-page) compound in hpage_pincount_available()
960 page = compound_head(page); in hpage_pincount_available()
961 return PageCompound(page) && compound_order(page) > 1; in hpage_pincount_available()
964 static inline int head_compound_pincount(struct page *head) in head_compound_pincount()
969 static inline int compound_pincount(struct page *page) in compound_pincount() argument
971 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); in compound_pincount()
972 page = compound_head(page); in compound_pincount()
973 return head_compound_pincount(page); in compound_pincount()
976 static inline void set_compound_order(struct page *page, unsigned int order) in set_compound_order() argument
978 page[1].compound_order = order; in set_compound_order()
979 page[1].compound_nr = 1U << order; in set_compound_order()
982 /* Returns the number of pages in this potentially compound page. */
983 static inline unsigned long compound_nr(struct page *page) in compound_nr() argument
985 if (!PageHead(page)) in compound_nr()
987 return page[1].compound_nr; in compound_nr()
990 /* Returns the number of bytes in this potentially compound page. */
991 static inline unsigned long page_size(struct page *page) in page_size() argument
993 return PAGE_SIZE << compound_order(page); in page_size()
996 /* Returns the number of bits needed for the number of bytes in a page */
997 static inline unsigned int page_shift(struct page *page) in page_shift() argument
999 return PAGE_SHIFT + compound_order(page); in page_shift()
1002 void free_compound_page(struct page *page);
1018 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page);
1019 void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr);
1026 * Multiple processes may "see" the same page. E.g. for untouched
1027 * mappings of /dev/null, all processes see the same page full of
1031 * For the non-reserved pages, page_count(page) denotes a reference count.
1032 * page_count() == 0 means the page is free. page->lru is then used for
1034 * page_count() > 0 means the page has been allocated.
1038 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
1042 * A page may be used by anyone else who does a __get_free_page().
1044 * be used through the normal accessor functions. The top bits of page->flags
1045 * and page->virtual store page management information, but all other fields
1047 * page is the responsibility of the one who allocated it, and those who have
1054 * A pagecache page contains an opaque `private' member, which belongs to the
1055 * page's address_space. Usually, this is the address of a circular list of
1056 * the page's disk buffers. PG_private must be set to tell the VM to call
1059 * A page may belong to an inode's memory mapping. In this case, page->mapping
1060 * is the pointer to the inode, and page->index is the file offset of the page,
1065 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1068 * reference to the page. Setting PG_private should also increment the
1069 * refcount. The each user mapping also has a reference to the page.
1090 /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
1128 static inline enum zone_type page_zonenum(const struct page *page) in page_zonenum() argument
1130 ASSERT_EXCLUSIVE_BITS(page->flags, ZONES_MASK << ZONES_PGSHIFT); in page_zonenum()
1131 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; in page_zonenum()
1135 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1137 return page_zonenum(page) == ZONE_DEVICE; in is_zone_device_page()
1142 static inline bool is_zone_device_page(const struct page *page) in is_zone_device_page() argument
1148 static inline bool is_zone_movable_page(const struct page *page) in is_zone_movable_page() argument
1150 return page_zonenum(page) == ZONE_MOVABLE; in is_zone_movable_page()
1154 void free_devmap_managed_page(struct page *page);
1157 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1161 if (!is_zone_device_page(page)) in page_is_devmap_managed()
1163 switch (page->pgmap->type) { in page_is_devmap_managed()
1173 void put_devmap_managed_page(struct page *page);
1176 static inline bool page_is_devmap_managed(struct page *page) in page_is_devmap_managed() argument
1181 static inline void put_devmap_managed_page(struct page *page) in put_devmap_managed_page() argument
1186 static inline bool is_device_private_page(const struct page *page) in is_device_private_page() argument
1190 is_zone_device_page(page) && in is_device_private_page()
1191 page->pgmap->type == MEMORY_DEVICE_PRIVATE; in is_device_private_page()
1194 static inline bool is_pci_p2pdma_page(const struct page *page) in is_pci_p2pdma_page() argument
1198 is_zone_device_page(page) && in is_pci_p2pdma_page()
1199 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; in is_pci_p2pdma_page()
1203 #define page_ref_zero_or_close_to_overflow(page) \ argument
1204 ((unsigned int) page_ref_count(page) + 127u <= 127u)
1206 static inline void get_page(struct page *page) in get_page() argument
1208 page = compound_head(page); in get_page()
1210 * Getting a normal page or the head of a compound page in get_page()
1211 * requires to already have an elevated page->_refcount. in get_page()
1213 VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page); in get_page()
1214 page_ref_inc(page); in get_page()
1217 bool __must_check try_grab_page(struct page *page, unsigned int flags);
1218 struct page *try_grab_compound_head(struct page *page, int refs,
1222 static inline __must_check bool try_get_page(struct page *page) in try_get_page() argument
1224 page = compound_head(page); in try_get_page()
1225 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) in try_get_page()
1227 page_ref_inc(page); in try_get_page()
1231 static inline void put_page(struct page *page) in put_page() argument
1233 page = compound_head(page); in put_page()
1237 * 2 to 1, when refcount reach one it means the page is free and we in put_page()
1241 if (page_is_devmap_managed(page)) { in put_page()
1242 put_devmap_managed_page(page); in put_page()
1246 if (put_page_testzero(page)) in put_page()
1247 __put_page(page); in put_page()
1252 * the page's refcount so that two separate items are tracked: the original page
1254 * made against the page. ("gup-pinned" is another term for the latter).
1262 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1264 * simpler, due to the fact that adding an even power of two to the page
1273 * applications that don't have huge page reference counts, this won't be an
1278 * get_user_pages and page_mkclean and other calls that race to set up page
1283 void unpin_user_page(struct page *page);
1284 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1286 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1288 void unpin_user_pages(struct page **pages, unsigned long npages);
1291 * page_maybe_dma_pinned - Report if a page is pinned for DMA.
1292 * @page: The page.
1294 * This function checks if a page has been pinned via a call to
1300 * GUP_PIN_COUNTING_BIAS worth of normal page references".
1302 * False positives are OK, because: a) it's unlikely for a page to get that many
1307 * more tracking data available: the 3rd struct page in the compound page is
1313 * Return: True, if it is likely that the page has been "dma-pinned".
1314 * False, if the page is definitely not dma-pinned.
1316 static inline bool page_maybe_dma_pinned(struct page *page) in page_maybe_dma_pinned() argument
1318 if (hpage_pincount_available(page)) in page_maybe_dma_pinned()
1319 return compound_pincount(page) > 0; in page_maybe_dma_pinned()
1329 return ((unsigned int)page_ref_count(compound_head(page))) >= in page_maybe_dma_pinned()
1340 * should break the cow immediately for a page on the src mm.
1343 struct page *page) in page_needs_cow_for_dma() argument
1351 return page_maybe_dma_pinned(page); in page_needs_cow_for_dma()
1362 * node id available in page flags.
1366 static inline int page_zone_id(struct page *page) in page_zone_id() argument
1368 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK; in page_zone_id()
1372 extern int page_to_nid(const struct page *page);
1374 static inline int page_to_nid(const struct page *page) in page_to_nid() argument
1376 struct page *p = (struct page *)page; in page_to_nid()
1420 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1422 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK); in page_cpupid_xchg_last()
1425 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1427 return page->_last_cpupid; in page_cpupid_last()
1429 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1431 page->_last_cpupid = -1 & LAST_CPUPID_MASK; in page_cpupid_reset_last()
1434 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1436 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; in page_cpupid_last()
1439 extern int page_cpupid_xchg_last(struct page *page, int cpupid);
1441 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1443 page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; in page_cpupid_reset_last()
1447 static inline int page_cpupid_xchg_last(struct page *page, int cpupid) in page_cpupid_xchg_last() argument
1449 return page_to_nid(page); /* XXX */ in page_cpupid_xchg_last()
1452 static inline int page_cpupid_last(struct page *page) in page_cpupid_last() argument
1454 return page_to_nid(page); /* XXX */ in page_cpupid_last()
1482 static inline void page_cpupid_reset_last(struct page *page) in page_cpupid_reset_last() argument
1495 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
1500 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1505 tag = (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK; in page_kasan_tag()
1512 static inline void page_kasan_tag_set(struct page *page, u8 tag) in page_kasan_tag_set() argument
1516 page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT); in page_kasan_tag_set()
1517 page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT; in page_kasan_tag_set()
1521 static inline void page_kasan_tag_reset(struct page *page) in page_kasan_tag_reset() argument
1524 page_kasan_tag_set(page, 0xff); in page_kasan_tag_reset()
1529 static inline u8 page_kasan_tag(const struct page *page) in page_kasan_tag() argument
1534 static inline void page_kasan_tag_set(struct page *page, u8 tag) { } in page_kasan_tag_set() argument
1535 static inline void page_kasan_tag_reset(struct page *page) { } in page_kasan_tag_reset() argument
1539 static inline struct zone *page_zone(const struct page *page) in page_zone() argument
1541 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; in page_zone()
1544 static inline pg_data_t *page_pgdat(const struct page *page) in page_pgdat() argument
1546 return NODE_DATA(page_to_nid(page)); in page_pgdat()
1550 static inline void set_page_section(struct page *page, unsigned long section) in set_page_section() argument
1552 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); in set_page_section()
1553 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; in set_page_section()
1556 static inline unsigned long page_to_section(const struct page *page) in page_to_section() argument
1558 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; in page_to_section()
1564 static inline bool is_pinnable_page(struct page *page) in is_pinnable_page() argument
1566 return !(is_zone_movable_page(page) || is_migrate_cma_page(page)) || in is_pinnable_page()
1567 is_zero_pfn(page_to_pfn(page)); in is_pinnable_page()
1570 static inline bool is_pinnable_page(struct page *page) in is_pinnable_page() argument
1576 static inline void set_page_zone(struct page *page, enum zone_type zone) in set_page_zone() argument
1578 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); in set_page_zone()
1579 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; in set_page_zone()
1582 static inline void set_page_node(struct page *page, unsigned long node) in set_page_node() argument
1584 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); in set_page_node()
1585 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; in set_page_node()
1588 static inline void set_page_links(struct page *page, enum zone_type zone, in set_page_links() argument
1591 set_page_zone(page, zone); in set_page_links()
1592 set_page_node(page, node); in set_page_links()
1594 set_page_section(page, pfn_to_section_nr(pfn)); in set_page_links()
1603 static __always_inline void *lowmem_page_address(const struct page *page) in lowmem_page_address() argument
1605 return page_to_virt(page); in lowmem_page_address()
1613 static inline void *page_address(const struct page *page) in page_address() argument
1615 return page->virtual; in page_address()
1617 static inline void set_page_address(struct page *page, void *address) in set_page_address() argument
1619 page->virtual = address; in set_page_address()
1625 void *page_address(const struct page *page);
1626 void set_page_address(struct page *page, void *virtual);
1631 #define page_address(page) lowmem_page_address(page) argument
1632 #define set_page_address(page, address) do { } while(0) argument
1636 extern void *page_rmapping(struct page *page);
1637 extern struct anon_vma *page_anon_vma(struct page *page);
1638 extern struct address_space *page_mapping(struct page *page);
1640 extern struct address_space *__page_file_mapping(struct page *);
1643 struct address_space *page_file_mapping(struct page *page) in page_file_mapping() argument
1645 if (unlikely(PageSwapCache(page))) in page_file_mapping()
1646 return __page_file_mapping(page); in page_file_mapping()
1648 return page->mapping; in page_file_mapping()
1651 extern pgoff_t __page_file_index(struct page *page);
1654 * Return the pagecache index of the passed page. Regular pagecache pages
1657 static inline pgoff_t page_index(struct page *page) in page_index() argument
1659 if (unlikely(PageSwapCache(page))) in page_index()
1660 return __page_file_index(page); in page_index()
1661 return page->index; in page_index()
1664 bool page_mapped(struct page *page);
1665 struct address_space *page_mapping(struct page *page);
1668 * Return true only if the page has been allocated with
1672 static inline bool page_is_pfmemalloc(const struct page *page) in page_is_pfmemalloc() argument
1675 * lru.next has bit 1 set if the page is allocated from the in page_is_pfmemalloc()
1679 return (uintptr_t)page->lru.next & BIT(1); in page_is_pfmemalloc()
1683 * Only to be called by the page allocator on a freshly allocated
1684 * page.
1686 static inline void set_page_pfmemalloc(struct page *page) in set_page_pfmemalloc() argument
1688 page->lru.next = (void *)BIT(1); in set_page_pfmemalloc()
1691 static inline void clear_page_pfmemalloc(struct page *page) in clear_page_pfmemalloc() argument
1693 page->lru.next = NULL; in clear_page_pfmemalloc()
1702 #define offset_in_thp(page, p) ((unsigned long)(p) & (thp_size(page) - 1)) argument
1724 struct address_space *check_mapping; /* Check page->mapping if set */
1725 pgoff_t first_index; /* Lowest page->index to unmap */
1726 pgoff_t last_index; /* Highest page->index to unmap */
1727 struct page *single_page; /* Locked page to be unmapped */
1730 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1732 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
1764 int truncate_inode_page(struct address_space *mapping, struct page *page);
1765 int generic_error_remove_page(struct address_space *mapping, struct page *page);
1766 int invalidate_inode_page(struct page *page);
1775 void unmap_mapping_page(struct page *page);
1796 static inline void unmap_mapping_page(struct page *page) { } in unmap_mapping_page() argument
1818 unsigned int gup_flags, struct page **pages,
1822 unsigned int gup_flags, struct page **pages,
1825 unsigned int gup_flags, struct page **pages,
1828 unsigned int gup_flags, struct page **pages,
1831 unsigned int gup_flags, struct page **pages, int *locked);
1833 unsigned int gup_flags, struct page **pages, int *locked);
1835 struct page **pages, unsigned int gup_flags);
1837 struct page **pages, unsigned int gup_flags);
1840 unsigned int gup_flags, struct page **pages);
1842 unsigned int gup_flags, struct page **pages);
1850 struct page **pages);
1851 struct page *get_dump_page(unsigned long addr);
1853 extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1854 extern void do_invalidatepage(struct page *page, unsigned int offset,
1858 struct page *page);
1859 void account_page_cleaned(struct page *page, struct address_space *mapping,
1861 int set_page_dirty(struct page *page);
1862 int set_page_dirty_lock(struct page *page);
1863 void __cancel_dirty_page(struct page *page);
1864 static inline void cancel_dirty_page(struct page *page) in cancel_dirty_page() argument
1867 if (PageDirty(page)) in cancel_dirty_page()
1868 __cancel_dirty_page(page); in cancel_dirty_page()
1870 int clear_page_dirty_for_io(struct page *page);
1906 unsigned int gup_flags, struct page **pages);
1908 unsigned int gup_flags, struct page **pages);
1911 unsigned int gup_flags, struct page **pagep) in get_user_page_fast_only()
1956 /* Optimized variant when page is already known not to be PageAnon */
1957 static inline int mm_counter_file(struct page *page) in mm_counter_file() argument
1959 if (PageSwapBacked(page)) in mm_counter_file()
1964 static inline int mm_counter(struct page *page) in mm_counter() argument
1966 if (PageAnon(page)) in mm_counter()
1968 return mm_counter_file(page); in mm_counter()
2181 extern bool ptlock_alloc(struct page *page);
2182 extern void ptlock_free(struct page *page);
2184 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2186 return page->ptl; in ptlock_ptr()
2193 static inline bool ptlock_alloc(struct page *page) in ptlock_alloc() argument
2198 static inline void ptlock_free(struct page *page) in ptlock_free() argument
2202 static inline spinlock_t *ptlock_ptr(struct page *page) in ptlock_ptr() argument
2204 return &page->ptl; in ptlock_ptr()
2213 static inline bool ptlock_init(struct page *page) in ptlock_init() argument
2216 * prep_new_page() initialize page->private (and therefore page->ptl) in ptlock_init()
2219 * It can happen if arch try to use slab for page table allocation: in ptlock_init()
2220 * slab code uses page->slab_cache, which share storage with page->ptl. in ptlock_init()
2222 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page); in ptlock_init()
2223 if (!ptlock_alloc(page)) in ptlock_init()
2225 spin_lock_init(ptlock_ptr(page)); in ptlock_init()
2238 static inline bool ptlock_init(struct page *page) { return true; } in ptlock_init() argument
2239 static inline void ptlock_free(struct page *page) {} in ptlock_free() argument
2248 static inline bool pgtable_pte_page_ctor(struct page *page) in pgtable_pte_page_ctor() argument
2250 if (!ptlock_init(page)) in pgtable_pte_page_ctor()
2252 __SetPageTable(page); in pgtable_pte_page_ctor()
2253 inc_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pte_page_ctor()
2257 static inline void pgtable_pte_page_dtor(struct page *page) in pgtable_pte_page_dtor() argument
2259 ptlock_free(page); in pgtable_pte_page_dtor()
2260 __ClearPageTable(page); in pgtable_pte_page_dtor()
2261 dec_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pte_page_dtor()
2293 static struct page *pmd_to_page(pmd_t *pmd) in pmd_to_page()
2304 static inline bool pmd_ptlock_init(struct page *page) in pmd_ptlock_init() argument
2307 page->pmd_huge_pte = NULL; in pmd_ptlock_init()
2309 return ptlock_init(page); in pmd_ptlock_init()
2312 static inline void pmd_ptlock_free(struct page *page) in pmd_ptlock_free() argument
2315 VM_BUG_ON_PAGE(page->pmd_huge_pte, page); in pmd_ptlock_free()
2317 ptlock_free(page); in pmd_ptlock_free()
2329 static inline bool pmd_ptlock_init(struct page *page) { return true; } in pmd_ptlock_init() argument
2330 static inline void pmd_ptlock_free(struct page *page) {} in pmd_ptlock_free() argument
2343 static inline bool pgtable_pmd_page_ctor(struct page *page) in pgtable_pmd_page_ctor() argument
2345 if (!pmd_ptlock_init(page)) in pgtable_pmd_page_ctor()
2347 __SetPageTable(page); in pgtable_pmd_page_ctor()
2348 inc_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_ctor()
2352 static inline void pgtable_pmd_page_dtor(struct page *page) in pgtable_pmd_page_dtor() argument
2354 pmd_ptlock_free(page); in pgtable_pmd_page_dtor()
2355 __ClearPageTable(page); in pgtable_pmd_page_dtor()
2356 dec_lruvec_page_state(page, NR_PAGETABLE); in pgtable_pmd_page_dtor()
2391 extern void adjust_managed_page_count(struct page *page, long count);
2396 /* Free the reserved page into the buddy system, so it gets managed. */
2397 static inline void free_reserved_page(struct page *page) in free_reserved_page() argument
2399 ClearPageReserved(page); in free_reserved_page()
2400 init_page_count(page); in free_reserved_page()
2401 __free_page(page); in free_reserved_page()
2402 adjust_managed_page_count(page, 1); in free_reserved_page()
2404 #define free_highmem_page(page) free_reserved_page(page) argument
2406 static inline void mark_page_reserved(struct page *page) in mark_page_reserved() argument
2408 SetPageReserved(page); in mark_page_reserved()
2409 adjust_managed_page_count(page, -1); in mark_page_reserved()
2442 * An architecture is expected to register range of page frames backed by
2600 unsigned long flags, struct page **pages);
2662 /* mm/page-writeback.c */
2663 int __must_check write_one_page(struct page *page);
2796 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
2798 struct page **pages, unsigned long *num);
2799 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2801 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2816 unsigned long addr, struct page *page) in vmf_insert_page() argument
2818 int err = vm_insert_page(vma, addr, page); in vmf_insert_page()
2844 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
2848 #define FOLL_TOUCH 0x02 /* mark page accessed */
2849 #define FOLL_GET 0x04 /* do get_page on page */
2854 #define FOLL_POPULATE 0x40 /* fault in page */
2855 #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
2856 #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
2857 #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
2872 * FOLL_LONGTERM indicates that the page will be held for an indefinite time
2880 * pin, delaying writeback, bounce buffer page writeback, etc. As FS DAX was
2892 * that region. And so, CMA attempts to migrate the page before pinning, when
2895 * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
2897 * anything that gets a page reference and then touches page data (for example,
2944 extern void __kernel_poison_pages(struct page *page, int numpages);
2945 extern void __kernel_unpoison_pages(struct page *page, int numpages);
2960 static inline void kernel_poison_pages(struct page *page, int numpages) in kernel_poison_pages() argument
2963 __kernel_poison_pages(page, numpages); in kernel_poison_pages()
2965 static inline void kernel_unpoison_pages(struct page *page, int numpages) in kernel_unpoison_pages() argument
2968 __kernel_unpoison_pages(page, numpages); in kernel_unpoison_pages()
2973 static inline void __kernel_poison_pages(struct page *page, int nunmpages) { } in __kernel_poison_pages() argument
2974 static inline void kernel_poison_pages(struct page *page, int numpages) { } in kernel_poison_pages() argument
2975 static inline void kernel_unpoison_pages(struct page *page, int numpages) { } in kernel_unpoison_pages() argument
3020 extern void __kernel_map_pages(struct page *page, int numpages, int enable);
3022 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) in debug_pagealloc_map_pages() argument
3025 __kernel_map_pages(page, numpages, 1); in debug_pagealloc_map_pages()
3028 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) in debug_pagealloc_unmap_pages() argument
3031 __kernel_map_pages(page, numpages, 0); in debug_pagealloc_unmap_pages()
3034 static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} in debug_pagealloc_map_pages() argument
3035 static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} in debug_pagealloc_unmap_pages() argument
3086 struct page * __populate_section_memmap(unsigned long pfn,
3108 void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
3123 extern void shake_page(struct page *p);
3165 extern void clear_huge_page(struct page *page,
3168 extern void copy_user_huge_page(struct page *dst, struct page *src,
3172 extern long copy_huge_page_from_user(struct page *dst_page,
3178 * vma_is_special_huge - Are transhuge page-table entries considered special?
3181 * Whether transhuge page-table entries are considered "special" following
3184 * Return: true if transhuge page-table entries should be considered special,
3209 static inline bool page_is_guard(struct page *page) in page_is_guard() argument
3214 return PageGuard(page); in page_is_guard()
3219 static inline bool page_is_guard(struct page *page) { return false; } in page_is_guard() argument
3228 extern int memcmp_pages(struct page *page1, struct page *page2);
3230 static inline int pages_identical(struct page *page1, struct page *page2) in pages_identical()