32 static void hpage_pincount_add(struct page *page, int refs)  in hpage_pincount_add()  argument
37 	atomic_add(refs, compound_pincount_ptr(page));  in hpage_pincount_add()
40 static void hpage_pincount_sub(struct page *page, int refs)  in hpage_pincount_sub()  argument
45 	atomic_sub(refs, compound_pincount_ptr(page));  in hpage_pincount_sub()
49 static void put_page_refs(struct page *page, int refs)  in put_page_refs()  argument
52 	if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))  in put_page_refs()
60 	if (refs > 1)  in put_page_refs()
61 		page_ref_sub(page, refs - 1);  in put_page_refs()
69 static inline struct page *try_get_compound_head(struct page *page, int refs)  in try_get_compound_head()  argument
75 	if (unlikely(!page_cache_add_speculative(head, refs)))  in try_get_compound_head()
88 		put_page_refs(head, refs);  in try_get_compound_head()
128 				    int refs, unsigned int flags)  in try_grab_compound_head()  argument
131 		return try_get_compound_head(page, refs);  in try_grab_compound_head()
146 		page = try_get_compound_head(page, refs);  in try_grab_compound_head()
161 			hpage_pincount_add(page, refs);  in try_grab_compound_head()
163 			page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));  in try_grab_compound_head()
166 				    refs);  in try_grab_compound_head()
175 static void put_compound_head(struct page *page, int refs, unsigned int flags)  in put_compound_head()  argument
179 				    refs);  in put_compound_head()
182 			hpage_pincount_sub(page, refs);  in put_compound_head()
184 			refs *= GUP_PIN_COUNTING_BIAS;  in put_compound_head()
187 	put_page_refs(page, refs);  in put_compound_head()
2336 	int refs;  in gup_hugepte()  local
2352 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_hugepte()
2354 	head = try_grab_compound_head(head, refs, flags);  in gup_hugepte()
2359 		put_compound_head(head, refs, flags);  in gup_hugepte()
2363 	*nr += refs;  in gup_hugepte()
2399 	int refs;  in gup_huge_pmd()  local
2412 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pmd()
2414 	head = try_grab_compound_head(pmd_page(orig), refs, flags);  in gup_huge_pmd()
2419 		put_compound_head(head, refs, flags);  in gup_huge_pmd()
2423 	*nr += refs;  in gup_huge_pmd()
2433 	int refs;  in gup_huge_pud()  local
2446 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pud()
2448 	head = try_grab_compound_head(pud_page(orig), refs, flags);  in gup_huge_pud()
2453 		put_compound_head(head, refs, flags);  in gup_huge_pud()
2457 	*nr += refs;  in gup_huge_pud()
2466 	int refs;  in gup_huge_pgd()  local
2475 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pgd()
2477 	head = try_grab_compound_head(pgd_page(orig), refs, flags);  in gup_huge_pgd()
2482 		put_compound_head(head, refs, flags);  in gup_huge_pgd()
2486 	*nr += refs;  in gup_huge_pgd()