94 static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx)  in pte_basic_tests()  argument
97 	pte_t pte = pfn_pte(args->fixed_pte_pfn, prot);  in pte_basic_tests()
122 static void __init pte_advanced_tests(struct pgtable_debug_args *args)  in pte_advanced_tests()  argument
138 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in pte_advanced_tests()
143 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
144 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
146 	ptep_set_wrprotect(args->mm, args->vaddr, args->ptep);  in pte_advanced_tests()
147 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
149 	ptep_get_and_clear(args->mm, args->vaddr, args->ptep);  in pte_advanced_tests()
150 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
153 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
156 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
160 	ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);  in pte_advanced_tests()
161 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
163 	ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);  in pte_advanced_tests()
164 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
167 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
169 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
171 	ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep);  in pte_advanced_tests()
172 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
176 static void __init pte_savedwrite_tests(struct pgtable_debug_args *args)  in pte_savedwrite_tests()  argument
178 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);  in pte_savedwrite_tests()
189 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx)  in pmd_basic_tests()  argument
199 	pmd = pfn_pmd(args->fixed_pmd_pfn, prot);  in pmd_basic_tests()
227 static void __init pmd_advanced_tests(struct pgtable_debug_args *args)  in pmd_advanced_tests()  argument
231 	unsigned long vaddr = args->vaddr;  in pmd_advanced_tests()
236 	page = (args->pmd_pfn != ULONG_MAX) ? pfn_to_page(args->pmd_pfn) : NULL;  in pmd_advanced_tests()
251 	pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep);  in pmd_advanced_tests()
253 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);  in pmd_advanced_tests()
254 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
256 	pmdp_set_wrprotect(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
257 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
259 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
260 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
263 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);  in pmd_advanced_tests()
266 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
270 	pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);  in pmd_advanced_tests()
271 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
273 	pmdp_huge_get_and_clear_full(args->vma, vaddr, args->pmdp, 1);  in pmd_advanced_tests()
274 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
277 	pmd = pmd_mkhuge(pfn_pmd(args->pmd_pfn, args->page_prot));  in pmd_advanced_tests()
279 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
281 	pmdp_test_and_clear_young(args->vma, vaddr, args->pmdp);  in pmd_advanced_tests()
282 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
286 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
287 	pgtable_trans_huge_withdraw(args->mm, args->pmdp);  in pmd_advanced_tests()
290 static void __init pmd_leaf_tests(struct pgtable_debug_args *args)  in pmd_leaf_tests()  argument
298 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_leaf_tests()
307 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args)  in pmd_savedwrite_tests()  argument
318 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none);  in pmd_savedwrite_tests()
324 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx)  in pud_basic_tests()  argument
334 	pud = pfn_pud(args->fixed_pud_pfn, prot);  in pud_basic_tests()
355 	if (mm_pmd_folded(args->mm))  in pud_basic_tests()
365 static void __init pud_advanced_tests(struct pgtable_debug_args *args)  in pud_advanced_tests()  argument
368 	unsigned long vaddr = args->vaddr;  in pud_advanced_tests()
374 	page = (args->pud_pfn != ULONG_MAX) ? pfn_to_page(args->pud_pfn) : NULL;  in pud_advanced_tests()
389 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
390 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
392 	pudp_set_wrprotect(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
393 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
397 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
398 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
401 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
404 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
408 	pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1);  in pud_advanced_tests()
409 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
413 	pudp_huge_get_and_clear_full(args->mm, vaddr, args->pudp, 1);  in pud_advanced_tests()
414 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
418 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
420 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
422 	pudp_test_and_clear_young(args->vma, vaddr, args->pudp);  in pud_advanced_tests()
423 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
426 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
429 static void __init pud_leaf_tests(struct pgtable_debug_args *args)  in pud_leaf_tests()  argument
437 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_leaf_tests()
445 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pud_basic_tests()  argument
446 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }  in pud_advanced_tests()  argument
447 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }  in pud_leaf_tests()  argument
450 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pmd_basic_tests()  argument
451 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pud_basic_tests()  argument
452 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) { }  in pmd_advanced_tests()  argument
453 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }  in pud_advanced_tests()  argument
454 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) { }  in pmd_leaf_tests()  argument
455 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }  in pud_leaf_tests()  argument
456 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args) { }  in pmd_savedwrite_tests()  argument
460 static void __init pmd_huge_tests(struct pgtable_debug_args *args)  in pmd_huge_tests()  argument
464 	if (!arch_vmap_pmd_supported(args->page_prot))  in pmd_huge_tests()
472 	WRITE_ONCE(*args->pmdp, __pmd(0));  in pmd_huge_tests()
473 	WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot));  in pmd_huge_tests()
474 	WARN_ON(!pmd_clear_huge(args->pmdp));  in pmd_huge_tests()
475 	pmd = READ_ONCE(*args->pmdp);  in pmd_huge_tests()
479 static void __init pud_huge_tests(struct pgtable_debug_args *args)  in pud_huge_tests()  argument
483 	if (!arch_vmap_pud_supported(args->page_prot))  in pud_huge_tests()
491 	WRITE_ONCE(*args->pudp, __pud(0));  in pud_huge_tests()
492 	WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot));  in pud_huge_tests()
493 	WARN_ON(!pud_clear_huge(args->pudp));  in pud_huge_tests()
494 	pud = READ_ONCE(*args->pudp);  in pud_huge_tests()
498 static void __init pmd_huge_tests(struct pgtable_debug_args *args) { }  in pmd_huge_tests()  argument
499 static void __init pud_huge_tests(struct pgtable_debug_args *args) { }  in pud_huge_tests()  argument
502 static void __init p4d_basic_tests(struct pgtable_debug_args *args)  in p4d_basic_tests()  argument
511 static void __init pgd_basic_tests(struct pgtable_debug_args *args)  in pgd_basic_tests()  argument
521 static void __init pud_clear_tests(struct pgtable_debug_args *args)  in pud_clear_tests()  argument
523 	pud_t pud = READ_ONCE(*args->pudp);  in pud_clear_tests()
525 	if (mm_pmd_folded(args->mm))  in pud_clear_tests()
530 	WRITE_ONCE(*args->pudp, pud);  in pud_clear_tests()
531 	pud_clear(args->pudp);  in pud_clear_tests()
532 	pud = READ_ONCE(*args->pudp);  in pud_clear_tests()
536 static void __init pud_populate_tests(struct pgtable_debug_args *args)  in pud_populate_tests()  argument
540 	if (mm_pmd_folded(args->mm))  in pud_populate_tests()
548 	pud_populate(args->mm, args->pudp, args->start_pmdp);  in pud_populate_tests()
549 	pud = READ_ONCE(*args->pudp);  in pud_populate_tests()
553 static void __init pud_clear_tests(struct pgtable_debug_args *args) { }  in pud_clear_tests()  argument
554 static void __init pud_populate_tests(struct pgtable_debug_args *args) { }  in pud_populate_tests()  argument
558 static void __init p4d_clear_tests(struct pgtable_debug_args *args)  in p4d_clear_tests()  argument
560 	p4d_t p4d = READ_ONCE(*args->p4dp);  in p4d_clear_tests()
562 	if (mm_pud_folded(args->mm))  in p4d_clear_tests()
567 	WRITE_ONCE(*args->p4dp, p4d);  in p4d_clear_tests()
568 	p4d_clear(args->p4dp);  in p4d_clear_tests()
569 	p4d = READ_ONCE(*args->p4dp);  in p4d_clear_tests()
573 static void __init p4d_populate_tests(struct pgtable_debug_args *args)  in p4d_populate_tests()  argument
577 	if (mm_pud_folded(args->mm))  in p4d_populate_tests()
585 	pud_clear(args->pudp);  in p4d_populate_tests()
586 	p4d_clear(args->p4dp);  in p4d_populate_tests()
587 	p4d_populate(args->mm, args->p4dp, args->start_pudp);  in p4d_populate_tests()
588 	p4d = READ_ONCE(*args->p4dp);  in p4d_populate_tests()
592 static void __init pgd_clear_tests(struct pgtable_debug_args *args)  in pgd_clear_tests()  argument
594 	pgd_t pgd = READ_ONCE(*(args->pgdp));  in pgd_clear_tests()
596 	if (mm_p4d_folded(args->mm))  in pgd_clear_tests()
601 	WRITE_ONCE(*args->pgdp, pgd);  in pgd_clear_tests()
602 	pgd_clear(args->pgdp);  in pgd_clear_tests()
603 	pgd = READ_ONCE(*args->pgdp);  in pgd_clear_tests()
607 static void __init pgd_populate_tests(struct pgtable_debug_args *args)  in pgd_populate_tests()  argument
611 	if (mm_p4d_folded(args->mm))  in pgd_populate_tests()
619 	p4d_clear(args->p4dp);  in pgd_populate_tests()
620 	pgd_clear(args->pgdp);  in pgd_populate_tests()
621 	pgd_populate(args->mm, args->pgdp, args->start_p4dp);  in pgd_populate_tests()
622 	pgd = READ_ONCE(*args->pgdp);  in pgd_populate_tests()
626 static void __init p4d_clear_tests(struct pgtable_debug_args *args) { }  in p4d_clear_tests()  argument
627 static void __init pgd_clear_tests(struct pgtable_debug_args *args) { }  in pgd_clear_tests()  argument
628 static void __init p4d_populate_tests(struct pgtable_debug_args *args) { }  in p4d_populate_tests()  argument
629 static void __init pgd_populate_tests(struct pgtable_debug_args *args) { }  in pgd_populate_tests()  argument
632 static void __init pte_clear_tests(struct pgtable_debug_args *args)  in pte_clear_tests()  argument
635 	pte_t pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_clear_tests()
637 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in pte_clear_tests()
652 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_clear_tests()
655 	pte_clear(args->mm, args->vaddr, args->ptep);  in pte_clear_tests()
656 	pte = ptep_get(args->ptep);  in pte_clear_tests()
660 static void __init pmd_clear_tests(struct pgtable_debug_args *args)  in pmd_clear_tests()  argument
662 	pmd_t pmd = READ_ONCE(*args->pmdp);  in pmd_clear_tests()
666 	WRITE_ONCE(*args->pmdp, pmd);  in pmd_clear_tests()
667 	pmd_clear(args->pmdp);  in pmd_clear_tests()
668 	pmd = READ_ONCE(*args->pmdp);  in pmd_clear_tests()
672 static void __init pmd_populate_tests(struct pgtable_debug_args *args)  in pmd_populate_tests()  argument
681 	pmd_populate(args->mm, args->pmdp, args->start_ptep);  in pmd_populate_tests()
682 	pmd = READ_ONCE(*args->pmdp);  in pmd_populate_tests()
686 static void __init pte_special_tests(struct pgtable_debug_args *args)  in pte_special_tests()  argument
688 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_special_tests()
697 static void __init pte_protnone_tests(struct pgtable_debug_args *args)  in pte_protnone_tests()  argument
699 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);  in pte_protnone_tests()
710 static void __init pmd_protnone_tests(struct pgtable_debug_args *args)  in pmd_protnone_tests()  argument
721 	pmd = pmd_mkhuge(pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none));  in pmd_protnone_tests()
726 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { }  in pmd_protnone_tests()  argument
730 static void __init pte_devmap_tests(struct pgtable_debug_args *args)  in pte_devmap_tests()  argument
732 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_devmap_tests()
739 static void __init pmd_devmap_tests(struct pgtable_debug_args *args)  in pmd_devmap_tests()  argument
747 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_devmap_tests()
752 static void __init pud_devmap_tests(struct pgtable_debug_args *args)  in pud_devmap_tests()  argument
760 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_devmap_tests()
764 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
767 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }  in pmd_devmap_tests()  argument
768 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
771 static void __init pte_devmap_tests(struct pgtable_debug_args *args) { }  in pte_devmap_tests()  argument
772 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }  in pmd_devmap_tests()  argument
773 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
776 static void __init pte_soft_dirty_tests(struct pgtable_debug_args *args)  in pte_soft_dirty_tests()  argument
778 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_soft_dirty_tests()
788 static void __init pte_swap_soft_dirty_tests(struct pgtable_debug_args *args)  in pte_swap_soft_dirty_tests()  argument
790 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_swap_soft_dirty_tests()
801 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args)  in pmd_soft_dirty_tests()  argument
812 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_soft_dirty_tests()
817 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args)  in pmd_swap_soft_dirty_tests()  argument
829 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_swap_soft_dirty_tests()
834 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) { }  in pmd_soft_dirty_tests()  argument
835 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) { }  in pmd_swap_soft_dirty_tests()  argument
838 static void __init pte_swap_tests(struct pgtable_debug_args *args)  in pte_swap_tests()  argument
844 	pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_swap_tests()
847 	WARN_ON(args->fixed_pte_pfn != pte_pfn(pte));  in pte_swap_tests()
851 static void __init pmd_swap_tests(struct pgtable_debug_args *args)  in pmd_swap_tests()  argument
860 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_swap_tests()
863 	WARN_ON(args->fixed_pmd_pfn != pmd_pfn(pmd));  in pmd_swap_tests()
866 static void __init pmd_swap_tests(struct pgtable_debug_args *args) { }  in pmd_swap_tests()  argument
869 static void __init swap_migration_tests(struct pgtable_debug_args *args)  in swap_migration_tests()  argument
884 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in swap_migration_tests()
910 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args)  in hugetlb_basic_tests()  argument
920 	page = pfn_to_page(args->fixed_pmd_pfn);  in hugetlb_basic_tests()
921 	pte = mk_huge_pte(page, args->page_prot);  in hugetlb_basic_tests()
928 	pte = pfn_pte(args->fixed_pmd_pfn, args->page_prot);  in hugetlb_basic_tests()
934 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) { }  in hugetlb_basic_tests()  argument
938 static void __init pmd_thp_tests(struct pgtable_debug_args *args)  in pmd_thp_tests()  argument
957 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_thp_tests()
967 static void __init pud_thp_tests(struct pgtable_debug_args *args)  in pud_thp_tests()  argument
975 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_thp_tests()
987 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }  in pud_thp_tests()  argument
990 static void __init pmd_thp_tests(struct pgtable_debug_args *args) { }  in pmd_thp_tests()  argument
991 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }  in pud_thp_tests()  argument
1006 static void __init destroy_args(struct pgtable_debug_args *args)  in destroy_args()  argument
1014 	    args->pud_pfn != ULONG_MAX) {  in destroy_args()
1015 		if (args->is_contiguous_page) {  in destroy_args()
1016 			free_contig_range(args->pud_pfn,  in destroy_args()
1019 			page = pfn_to_page(args->pud_pfn);  in destroy_args()
1023 		args->pud_pfn = ULONG_MAX;  in destroy_args()
1024 		args->pmd_pfn = ULONG_MAX;  in destroy_args()
1025 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1030 	    args->pmd_pfn != ULONG_MAX) {  in destroy_args()
1031 		if (args->is_contiguous_page) {  in destroy_args()
1032 			free_contig_range(args->pmd_pfn, (1 << HPAGE_PMD_ORDER));  in destroy_args()
1034 			page = pfn_to_page(args->pmd_pfn);  in destroy_args()
1038 		args->pmd_pfn = ULONG_MAX;  in destroy_args()
1039 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1042 	if (args->pte_pfn != ULONG_MAX) {  in destroy_args()
1043 		page = pfn_to_page(args->pte_pfn);  in destroy_args()
1046 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1050 	if (args->start_ptep) {  in destroy_args()
1051 		pte_free(args->mm, args->start_ptep);  in destroy_args()
1052 		mm_dec_nr_ptes(args->mm);  in destroy_args()
1055 	if (args->start_pmdp) {  in destroy_args()
1056 		pmd_free(args->mm, args->start_pmdp);  in destroy_args()
1057 		mm_dec_nr_pmds(args->mm);  in destroy_args()
1060 	if (args->start_pudp) {  in destroy_args()
1061 		pud_free(args->mm, args->start_pudp);  in destroy_args()
1062 		mm_dec_nr_puds(args->mm);  in destroy_args()
1065 	if (args->start_p4dp)  in destroy_args()
1066 		p4d_free(args->mm, args->start_p4dp);  in destroy_args()
1069 	if (args->vma)  in destroy_args()
1070 		vm_area_free(args->vma);  in destroy_args()
1072 	if (args->mm)  in destroy_args()
1073 		mmdrop(args->mm);  in destroy_args()
1077 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order)  in debug_vm_pgtable_alloc_huge_page()  argument
1086 			args->is_contiguous_page = true;  in debug_vm_pgtable_alloc_huge_page()
1098 static int __init init_args(struct pgtable_debug_args *args)  in init_args()  argument
1110 	memset(args, 0, sizeof(*args));  in init_args()
1111 	args->vaddr              = get_random_vaddr();  in init_args()
1112 	args->page_prot          = vm_get_page_prot(VMFLAGS);  in init_args()
1113 	args->page_prot_none     = __P000;  in init_args()
1114 	args->is_contiguous_page = false;  in init_args()
1115 	args->pud_pfn            = ULONG_MAX;  in init_args()
1116 	args->pmd_pfn            = ULONG_MAX;  in init_args()
1117 	args->pte_pfn            = ULONG_MAX;  in init_args()
1118 	args->fixed_pgd_pfn      = ULONG_MAX;  in init_args()
1119 	args->fixed_p4d_pfn      = ULONG_MAX;  in init_args()
1120 	args->fixed_pud_pfn      = ULONG_MAX;  in init_args()
1121 	args->fixed_pmd_pfn      = ULONG_MAX;  in init_args()
1122 	args->fixed_pte_pfn      = ULONG_MAX;  in init_args()
1125 	args->mm = mm_alloc();  in init_args()
1126 	if (!args->mm) {  in init_args()
1132 	args->vma = vm_area_alloc(args->mm);  in init_args()
1133 	if (!args->vma) {  in init_args()
1144 	args->pgdp = pgd_offset(args->mm, args->vaddr);  in init_args()
1145 	args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr);  in init_args()
1146 	if (!args->p4dp) {  in init_args()
1151 	args->start_p4dp = p4d_offset(args->pgdp, 0UL);  in init_args()
1152 	WARN_ON(!args->start_p4dp);  in init_args()
1154 	args->pudp = pud_alloc(args->mm, args->p4dp, args->vaddr);  in init_args()
1155 	if (!args->pudp) {  in init_args()
1160 	args->start_pudp = pud_offset(args->p4dp, 0UL);  in init_args()
1161 	WARN_ON(!args->start_pudp);  in init_args()
1163 	args->pmdp = pmd_alloc(args->mm, args->pudp, args->vaddr);  in init_args()
1164 	if (!args->pmdp) {  in init_args()
1169 	args->start_pmdp = pmd_offset(args->pudp, 0UL);  in init_args()
1170 	WARN_ON(!args->start_pmdp);  in init_args()
1172 	if (pte_alloc(args->mm, args->pmdp)) {  in init_args()
1177 	args->start_ptep = pmd_pgtable(READ_ONCE(*args->pmdp));  in init_args()
1178 	WARN_ON(!args->start_ptep);  in init_args()
1190 	args->fixed_pgd_pfn = __phys_to_pfn(phys & PGDIR_MASK);  in init_args()
1191 	args->fixed_p4d_pfn = __phys_to_pfn(phys & P4D_MASK);  in init_args()
1192 	args->fixed_pud_pfn = __phys_to_pfn(phys & PUD_MASK);  in init_args()
1193 	args->fixed_pmd_pfn = __phys_to_pfn(phys & PMD_MASK);  in init_args()
1194 	args->fixed_pte_pfn = __phys_to_pfn(phys & PAGE_MASK);  in init_args()
1195 	WARN_ON(!pfn_valid(args->fixed_pte_pfn));  in init_args()
1205 		page = debug_vm_pgtable_alloc_huge_page(args,  in init_args()
1208 			args->pud_pfn = page_to_pfn(page);  in init_args()
1209 			args->pmd_pfn = args->pud_pfn;  in init_args()
1210 			args->pte_pfn = args->pud_pfn;  in init_args()
1217 		page = debug_vm_pgtable_alloc_huge_page(args, HPAGE_PMD_ORDER);  in init_args()
1219 			args->pmd_pfn = page_to_pfn(page);  in init_args()
1220 			args->pte_pfn = args->pmd_pfn;  in init_args()
1227 		args->pte_pfn = page_to_pfn(page);  in init_args()
1232 	destroy_args(args);  in init_args()
1238 	struct pgtable_debug_args args;  in debug_vm_pgtable()  local
1243 	ret = init_args(&args);  in debug_vm_pgtable()
1254 		pte_basic_tests(&args, idx);  in debug_vm_pgtable()
1255 		pmd_basic_tests(&args, idx);  in debug_vm_pgtable()
1256 		pud_basic_tests(&args, idx);  in debug_vm_pgtable()
1266 	p4d_basic_tests(&args);  in debug_vm_pgtable()
1267 	pgd_basic_tests(&args);  in debug_vm_pgtable()
1269 	pmd_leaf_tests(&args);  in debug_vm_pgtable()
1270 	pud_leaf_tests(&args);  in debug_vm_pgtable()
1272 	pte_savedwrite_tests(&args);  in debug_vm_pgtable()
1273 	pmd_savedwrite_tests(&args);  in debug_vm_pgtable()
1275 	pte_special_tests(&args);  in debug_vm_pgtable()
1276 	pte_protnone_tests(&args);  in debug_vm_pgtable()
1277 	pmd_protnone_tests(&args);  in debug_vm_pgtable()
1279 	pte_devmap_tests(&args);  in debug_vm_pgtable()
1280 	pmd_devmap_tests(&args);  in debug_vm_pgtable()
1281 	pud_devmap_tests(&args);  in debug_vm_pgtable()
1283 	pte_soft_dirty_tests(&args);  in debug_vm_pgtable()
1284 	pmd_soft_dirty_tests(&args);  in debug_vm_pgtable()
1285 	pte_swap_soft_dirty_tests(&args);  in debug_vm_pgtable()
1286 	pmd_swap_soft_dirty_tests(&args);  in debug_vm_pgtable()
1288 	pte_swap_tests(&args);  in debug_vm_pgtable()
1289 	pmd_swap_tests(&args);  in debug_vm_pgtable()
1291 	swap_migration_tests(&args);  in debug_vm_pgtable()
1293 	pmd_thp_tests(&args);  in debug_vm_pgtable()
1294 	pud_thp_tests(&args);  in debug_vm_pgtable()
1296 	hugetlb_basic_tests(&args);  in debug_vm_pgtable()
1303 	args.ptep = pte_offset_map_lock(args.mm, args.pmdp, args.vaddr, &ptl);  in debug_vm_pgtable()
1304 	pte_clear_tests(&args);  in debug_vm_pgtable()
1305 	pte_advanced_tests(&args);  in debug_vm_pgtable()
1306 	pte_unmap_unlock(args.ptep, ptl);  in debug_vm_pgtable()
1308 	ptl = pmd_lock(args.mm, args.pmdp);  in debug_vm_pgtable()
1309 	pmd_clear_tests(&args);  in debug_vm_pgtable()
1310 	pmd_advanced_tests(&args);  in debug_vm_pgtable()
1311 	pmd_huge_tests(&args);  in debug_vm_pgtable()
1312 	pmd_populate_tests(&args);  in debug_vm_pgtable()
1315 	ptl = pud_lock(args.mm, args.pudp);  in debug_vm_pgtable()
1316 	pud_clear_tests(&args);  in debug_vm_pgtable()
1317 	pud_advanced_tests(&args);  in debug_vm_pgtable()
1318 	pud_huge_tests(&args);  in debug_vm_pgtable()
1319 	pud_populate_tests(&args);  in debug_vm_pgtable()
1322 	spin_lock(&(args.mm->page_table_lock));  in debug_vm_pgtable()
1323 	p4d_clear_tests(&args);  in debug_vm_pgtable()
1324 	pgd_clear_tests(&args);  in debug_vm_pgtable()
1325 	p4d_populate_tests(&args);  in debug_vm_pgtable()
1326 	pgd_populate_tests(&args);  in debug_vm_pgtable()
1327 	spin_unlock(&(args.mm->page_table_lock));  in debug_vm_pgtable()
1329 	destroy_args(&args);  in debug_vm_pgtable()