803 		unsigned int gup_flags, struct vm_area_struct **vma,  in get_gate_page()  argument
814 	if (gup_flags & FOLL_WRITE)  in get_gate_page()
840 		if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))  in get_gate_page()
844 	if (unlikely(!try_grab_page(*page, gup_flags))) {  in get_gate_page()
914 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)  in check_vma_flags()  argument
917 	int write = (gup_flags & FOLL_WRITE);  in check_vma_flags()
918 	int foreign = (gup_flags & FOLL_REMOTE);  in check_vma_flags()
923 	if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))  in check_vma_flags()
928 			if (!(gup_flags & FOLL_FORCE))  in check_vma_flags()
943 		if (!(gup_flags & FOLL_FORCE))  in check_vma_flags()
1023 		unsigned int gup_flags, struct page **pages,  in __get_user_pages()  argument
1035 	VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));  in __get_user_pages()
1042 	if (!(gup_flags & FOLL_FORCE))  in __get_user_pages()
1043 		gup_flags |= FOLL_NUMA;  in __get_user_pages()
1047 		unsigned int foll_flags = gup_flags;  in __get_user_pages()
1055 						gup_flags, &vma,  in __get_user_pages()
1063 			if (!vma || check_vma_flags(vma, gup_flags)) {  in __get_user_pages()
1070 						gup_flags, locked);  in __get_user_pages()
1077 					BUG_ON(gup_flags & FOLL_NOWAIT);  in __get_user_pages()
1400 	int gup_flags;  in populate_vma_page_range()  local
1408 	gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;  in populate_vma_page_range()
1410 		gup_flags &= ~FOLL_POPULATE;  in populate_vma_page_range()
1417 		gup_flags |= FOLL_WRITE;  in populate_vma_page_range()
1424 		gup_flags |= FOLL_FORCE;  in populate_vma_page_range()
1430 	return __get_user_pages(mm, start, nr_pages, gup_flags,  in populate_vma_page_range()
1596 					unsigned int gup_flags)  in check_and_migrate_cma_pages()  argument
1650 		if (gup_flags & FOLL_PIN)  in check_and_migrate_cma_pages()
1674 						   gup_flags);  in check_and_migrate_cma_pages()
1691 					unsigned int gup_flags)  in check_and_migrate_cma_pages()  argument
1706 				  unsigned int gup_flags)  in __gup_longterm_locked()  argument
1712 	if (gup_flags & FOLL_LONGTERM) {  in __gup_longterm_locked()
1727 				     vmas_tmp, NULL, gup_flags);  in __gup_longterm_locked()
1729 	if (gup_flags & FOLL_LONGTERM) {  in __gup_longterm_locked()
1734 			if (gup_flags & FOLL_PIN)  in __gup_longterm_locked()
1744 						 vmas_tmp, gup_flags);  in __gup_longterm_locked()
1766 static bool is_valid_gup_flags(unsigned int gup_flags)  in is_valid_gup_flags()  argument
1772 	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))  in is_valid_gup_flags()
1779 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))  in is_valid_gup_flags()
1788 				    unsigned int gup_flags, struct page **pages,  in __get_user_pages_remote()  argument
1798 	if (gup_flags & FOLL_LONGTERM) {  in __get_user_pages_remote()
1806 					     vmas, gup_flags | FOLL_TOUCH |  in __get_user_pages_remote()
1812 				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);  in __get_user_pages_remote()
1877 		unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()  argument
1880 	if (!is_valid_gup_flags(gup_flags))  in get_user_pages_remote()
1883 	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,  in get_user_pages_remote()
1891 			   unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()  argument
1899 				    unsigned int gup_flags, struct page **pages,  in __get_user_pages_remote()  argument
1923 		unsigned int gup_flags, struct page **pages,  in get_user_pages()  argument
1926 	if (!is_valid_gup_flags(gup_flags))  in get_user_pages()
1930 				     pages, vmas, gup_flags | FOLL_TOUCH);  in get_user_pages()
1967 			   unsigned int gup_flags, struct page **pages,  in get_user_pages_locked()  argument
1976 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))  in get_user_pages_locked()
1982 	if (WARN_ON_ONCE(gup_flags & FOLL_PIN))  in get_user_pages_locked()
1987 				       gup_flags | FOLL_TOUCH);  in get_user_pages_locked()
2007 			     struct page **pages, unsigned int gup_flags)  in get_user_pages_unlocked()  argument
2019 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))  in get_user_pages_unlocked()
2024 				      &locked, gup_flags | FOLL_TOUCH);  in get_user_pages_unlocked()
2658 				   unsigned int gup_flags, struct page **pages)  in __gup_longterm_unlocked()  argument
2666 	if (gup_flags & FOLL_LONGTERM) {  in __gup_longterm_unlocked()
2670 					    pages, NULL, gup_flags);  in __gup_longterm_unlocked()
2674 					      pages, gup_flags);  in __gup_longterm_unlocked()
2681 					unsigned int gup_flags,  in internal_get_user_pages_fast()  argument
2688 	if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |  in internal_get_user_pages_fast()
2693 	if (gup_flags & FOLL_PIN)  in internal_get_user_pages_fast()
2696 	if (!(gup_flags & FOLL_FAST_ONLY))  in internal_get_user_pages_fast()
2721 		unsigned long fast_flags = gup_flags;  in internal_get_user_pages_fast()
2729 	if (nr_pinned < nr_pages && !(gup_flags & FOLL_FAST_ONLY)) {  in internal_get_user_pages_fast()
2735 					      gup_flags, pages);  in internal_get_user_pages_fast()
2769 			     unsigned int gup_flags, struct page **pages)  in get_user_pages_fast_only()  argument
2779 	gup_flags |= FOLL_GET | FOLL_FAST_ONLY;  in get_user_pages_fast_only()
2781 	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,  in get_user_pages_fast_only()
2814 			unsigned int gup_flags, struct page **pages)  in get_user_pages_fast()  argument
2816 	if (!is_valid_gup_flags(gup_flags))  in get_user_pages_fast()
2825 	gup_flags |= FOLL_GET;  in get_user_pages_fast()
2826 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);  in get_user_pages_fast()
2847 			unsigned int gup_flags, struct page **pages)  in pin_user_pages_fast()  argument
2850 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages_fast()
2853 	gup_flags |= FOLL_PIN;  in pin_user_pages_fast()
2854 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);  in pin_user_pages_fast()
2865 			     unsigned int gup_flags, struct page **pages)  in pin_user_pages_fast_only()  argument
2873 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages_fast_only()
2879 	gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);  in pin_user_pages_fast_only()
2880 	nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,  in pin_user_pages_fast_only()
2919 			   unsigned int gup_flags, struct page **pages,  in pin_user_pages_remote()  argument
2923 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages_remote()
2926 	gup_flags |= FOLL_PIN;  in pin_user_pages_remote()
2927 	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,  in pin_user_pages_remote()
2951 		    unsigned int gup_flags, struct page **pages,  in pin_user_pages()  argument
2955 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages()
2958 	gup_flags |= FOLL_PIN;  in pin_user_pages()
2960 				     pages, vmas, gup_flags);  in pin_user_pages()
2970 			     struct page **pages, unsigned int gup_flags)  in pin_user_pages_unlocked()  argument
2973 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages_unlocked()
2976 	gup_flags |= FOLL_PIN;  in pin_user_pages_unlocked()
2977 	return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);  in pin_user_pages_unlocked()
2987 			   unsigned int gup_flags, struct page **pages,  in pin_user_pages_locked()  argument
2996 	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))  in pin_user_pages_locked()
3000 	if (WARN_ON_ONCE(gup_flags & FOLL_GET))  in pin_user_pages_locked()
3003 	gup_flags |= FOLL_PIN;  in pin_user_pages_locked()
3006 				       gup_flags | FOLL_TOUCH);  in pin_user_pages_locked()