94 static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx)  in pte_basic_tests()  argument
97 	pte_t pte = pfn_pte(args->fixed_pte_pfn, prot);  in pte_basic_tests()
122 static void __init pte_advanced_tests(struct pgtable_debug_args *args)  in pte_advanced_tests()  argument
138 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in pte_advanced_tests()
143 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
144 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
146 	ptep_set_wrprotect(args->mm, args->vaddr, args->ptep);  in pte_advanced_tests()
147 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
149 	ptep_get_and_clear(args->mm, args->vaddr, args->ptep);  in pte_advanced_tests()
150 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
153 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
156 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
160 	ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);  in pte_advanced_tests()
161 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
163 	ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);  in pte_advanced_tests()
164 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
167 	pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_advanced_tests()
169 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_advanced_tests()
171 	ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep);  in pte_advanced_tests()
172 	pte = ptep_get(args->ptep);  in pte_advanced_tests()
175 	ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1);  in pte_advanced_tests()
178 static void __init pte_savedwrite_tests(struct pgtable_debug_args *args)  in pte_savedwrite_tests()  argument
180 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);  in pte_savedwrite_tests()
191 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx)  in pmd_basic_tests()  argument
201 	pmd = pfn_pmd(args->fixed_pmd_pfn, prot);  in pmd_basic_tests()
229 static void __init pmd_advanced_tests(struct pgtable_debug_args *args)  in pmd_advanced_tests()  argument
233 	unsigned long vaddr = args->vaddr;  in pmd_advanced_tests()
238 	page = (args->pmd_pfn != ULONG_MAX) ? pfn_to_page(args->pmd_pfn) : NULL;  in pmd_advanced_tests()
253 	pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep);  in pmd_advanced_tests()
255 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);  in pmd_advanced_tests()
256 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
258 	pmdp_set_wrprotect(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
259 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
261 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
262 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
265 	pmd = pfn_pmd(args->pmd_pfn, args->page_prot);  in pmd_advanced_tests()
268 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
272 	pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);  in pmd_advanced_tests()
273 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
275 	pmdp_huge_get_and_clear_full(args->vma, vaddr, args->pmdp, 1);  in pmd_advanced_tests()
276 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
279 	pmd = pmd_mkhuge(pfn_pmd(args->pmd_pfn, args->page_prot));  in pmd_advanced_tests()
281 	set_pmd_at(args->mm, vaddr, args->pmdp, pmd);  in pmd_advanced_tests()
283 	pmdp_test_and_clear_young(args->vma, vaddr, args->pmdp);  in pmd_advanced_tests()
284 	pmd = READ_ONCE(*args->pmdp);  in pmd_advanced_tests()
288 	pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp);  in pmd_advanced_tests()
289 	pgtable_trans_huge_withdraw(args->mm, args->pmdp);  in pmd_advanced_tests()
292 static void __init pmd_leaf_tests(struct pgtable_debug_args *args)  in pmd_leaf_tests()  argument
300 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_leaf_tests()
309 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args)  in pmd_savedwrite_tests()  argument
320 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none);  in pmd_savedwrite_tests()
326 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx)  in pud_basic_tests()  argument
336 	pud = pfn_pud(args->fixed_pud_pfn, prot);  in pud_basic_tests()
357 	if (mm_pmd_folded(args->mm))  in pud_basic_tests()
367 static void __init pud_advanced_tests(struct pgtable_debug_args *args)  in pud_advanced_tests()  argument
370 	unsigned long vaddr = args->vaddr;  in pud_advanced_tests()
376 	page = (args->pud_pfn != ULONG_MAX) ? pfn_to_page(args->pud_pfn) : NULL;  in pud_advanced_tests()
391 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
392 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
394 	pudp_set_wrprotect(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
395 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
399 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
400 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
403 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
406 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
410 	pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1);  in pud_advanced_tests()
411 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
415 	pudp_huge_get_and_clear_full(args->mm, vaddr, args->pudp, 1);  in pud_advanced_tests()
416 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
420 	pud = pfn_pud(args->pud_pfn, args->page_prot);  in pud_advanced_tests()
422 	set_pud_at(args->mm, vaddr, args->pudp, pud);  in pud_advanced_tests()
424 	pudp_test_and_clear_young(args->vma, vaddr, args->pudp);  in pud_advanced_tests()
425 	pud = READ_ONCE(*args->pudp);  in pud_advanced_tests()
428 	pudp_huge_get_and_clear(args->mm, vaddr, args->pudp);  in pud_advanced_tests()
431 static void __init pud_leaf_tests(struct pgtable_debug_args *args)  in pud_leaf_tests()  argument
439 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_leaf_tests()
447 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pud_basic_tests()  argument
448 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }  in pud_advanced_tests()  argument
449 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }  in pud_leaf_tests()  argument
452 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pmd_basic_tests()  argument
453 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { }  in pud_basic_tests()  argument
454 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) { }  in pmd_advanced_tests()  argument
455 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { }  in pud_advanced_tests()  argument
456 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) { }  in pmd_leaf_tests()  argument
457 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { }  in pud_leaf_tests()  argument
458 static void __init pmd_savedwrite_tests(struct pgtable_debug_args *args) { }  in pmd_savedwrite_tests()  argument
462 static void __init pmd_huge_tests(struct pgtable_debug_args *args)  in pmd_huge_tests()  argument
466 	if (!arch_vmap_pmd_supported(args->page_prot))  in pmd_huge_tests()
474 	WRITE_ONCE(*args->pmdp, __pmd(0));  in pmd_huge_tests()
475 	WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot));  in pmd_huge_tests()
476 	WARN_ON(!pmd_clear_huge(args->pmdp));  in pmd_huge_tests()
477 	pmd = READ_ONCE(*args->pmdp);  in pmd_huge_tests()
481 static void __init pud_huge_tests(struct pgtable_debug_args *args)  in pud_huge_tests()  argument
485 	if (!arch_vmap_pud_supported(args->page_prot))  in pud_huge_tests()
493 	WRITE_ONCE(*args->pudp, __pud(0));  in pud_huge_tests()
494 	WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot));  in pud_huge_tests()
495 	WARN_ON(!pud_clear_huge(args->pudp));  in pud_huge_tests()
496 	pud = READ_ONCE(*args->pudp);  in pud_huge_tests()
500 static void __init pmd_huge_tests(struct pgtable_debug_args *args) { }  in pmd_huge_tests()  argument
501 static void __init pud_huge_tests(struct pgtable_debug_args *args) { }  in pud_huge_tests()  argument
504 static void __init p4d_basic_tests(struct pgtable_debug_args *args)  in p4d_basic_tests()  argument
513 static void __init pgd_basic_tests(struct pgtable_debug_args *args)  in pgd_basic_tests()  argument
523 static void __init pud_clear_tests(struct pgtable_debug_args *args)  in pud_clear_tests()  argument
525 	pud_t pud = READ_ONCE(*args->pudp);  in pud_clear_tests()
527 	if (mm_pmd_folded(args->mm))  in pud_clear_tests()
532 	WRITE_ONCE(*args->pudp, pud);  in pud_clear_tests()
533 	pud_clear(args->pudp);  in pud_clear_tests()
534 	pud = READ_ONCE(*args->pudp);  in pud_clear_tests()
538 static void __init pud_populate_tests(struct pgtable_debug_args *args)  in pud_populate_tests()  argument
542 	if (mm_pmd_folded(args->mm))  in pud_populate_tests()
550 	pud_populate(args->mm, args->pudp, args->start_pmdp);  in pud_populate_tests()
551 	pud = READ_ONCE(*args->pudp);  in pud_populate_tests()
555 static void __init pud_clear_tests(struct pgtable_debug_args *args) { }  in pud_clear_tests()  argument
556 static void __init pud_populate_tests(struct pgtable_debug_args *args) { }  in pud_populate_tests()  argument
560 static void __init p4d_clear_tests(struct pgtable_debug_args *args)  in p4d_clear_tests()  argument
562 	p4d_t p4d = READ_ONCE(*args->p4dp);  in p4d_clear_tests()
564 	if (mm_pud_folded(args->mm))  in p4d_clear_tests()
569 	WRITE_ONCE(*args->p4dp, p4d);  in p4d_clear_tests()
570 	p4d_clear(args->p4dp);  in p4d_clear_tests()
571 	p4d = READ_ONCE(*args->p4dp);  in p4d_clear_tests()
575 static void __init p4d_populate_tests(struct pgtable_debug_args *args)  in p4d_populate_tests()  argument
579 	if (mm_pud_folded(args->mm))  in p4d_populate_tests()
587 	pud_clear(args->pudp);  in p4d_populate_tests()
588 	p4d_clear(args->p4dp);  in p4d_populate_tests()
589 	p4d_populate(args->mm, args->p4dp, args->start_pudp);  in p4d_populate_tests()
590 	p4d = READ_ONCE(*args->p4dp);  in p4d_populate_tests()
594 static void __init pgd_clear_tests(struct pgtable_debug_args *args)  in pgd_clear_tests()  argument
596 	pgd_t pgd = READ_ONCE(*(args->pgdp));  in pgd_clear_tests()
598 	if (mm_p4d_folded(args->mm))  in pgd_clear_tests()
603 	WRITE_ONCE(*args->pgdp, pgd);  in pgd_clear_tests()
604 	pgd_clear(args->pgdp);  in pgd_clear_tests()
605 	pgd = READ_ONCE(*args->pgdp);  in pgd_clear_tests()
609 static void __init pgd_populate_tests(struct pgtable_debug_args *args)  in pgd_populate_tests()  argument
613 	if (mm_p4d_folded(args->mm))  in pgd_populate_tests()
621 	p4d_clear(args->p4dp);  in pgd_populate_tests()
622 	pgd_clear(args->pgdp);  in pgd_populate_tests()
623 	pgd_populate(args->mm, args->pgdp, args->start_p4dp);  in pgd_populate_tests()
624 	pgd = READ_ONCE(*args->pgdp);  in pgd_populate_tests()
628 static void __init p4d_clear_tests(struct pgtable_debug_args *args) { }  in p4d_clear_tests()  argument
629 static void __init pgd_clear_tests(struct pgtable_debug_args *args) { }  in pgd_clear_tests()  argument
630 static void __init p4d_populate_tests(struct pgtable_debug_args *args) { }  in p4d_populate_tests()  argument
631 static void __init pgd_populate_tests(struct pgtable_debug_args *args) { }  in pgd_populate_tests()  argument
634 static void __init pte_clear_tests(struct pgtable_debug_args *args)  in pte_clear_tests()  argument
637 	pte_t pte = pfn_pte(args->pte_pfn, args->page_prot);  in pte_clear_tests()
639 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in pte_clear_tests()
654 	set_pte_at(args->mm, args->vaddr, args->ptep, pte);  in pte_clear_tests()
657 	ptep_clear(args->mm, args->vaddr, args->ptep);  in pte_clear_tests()
658 	pte = ptep_get(args->ptep);  in pte_clear_tests()
662 static void __init pmd_clear_tests(struct pgtable_debug_args *args)  in pmd_clear_tests()  argument
664 	pmd_t pmd = READ_ONCE(*args->pmdp);  in pmd_clear_tests()
668 	WRITE_ONCE(*args->pmdp, pmd);  in pmd_clear_tests()
669 	pmd_clear(args->pmdp);  in pmd_clear_tests()
670 	pmd = READ_ONCE(*args->pmdp);  in pmd_clear_tests()
674 static void __init pmd_populate_tests(struct pgtable_debug_args *args)  in pmd_populate_tests()  argument
683 	pmd_populate(args->mm, args->pmdp, args->start_ptep);  in pmd_populate_tests()
684 	pmd = READ_ONCE(*args->pmdp);  in pmd_populate_tests()
688 static void __init pte_special_tests(struct pgtable_debug_args *args)  in pte_special_tests()  argument
690 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_special_tests()
699 static void __init pte_protnone_tests(struct pgtable_debug_args *args)  in pte_protnone_tests()  argument
701 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);  in pte_protnone_tests()
712 static void __init pmd_protnone_tests(struct pgtable_debug_args *args)  in pmd_protnone_tests()  argument
723 	pmd = pmd_mkhuge(pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none));  in pmd_protnone_tests()
728 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { }  in pmd_protnone_tests()  argument
732 static void __init pte_devmap_tests(struct pgtable_debug_args *args)  in pte_devmap_tests()  argument
734 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_devmap_tests()
741 static void __init pmd_devmap_tests(struct pgtable_debug_args *args)  in pmd_devmap_tests()  argument
749 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_devmap_tests()
754 static void __init pud_devmap_tests(struct pgtable_debug_args *args)  in pud_devmap_tests()  argument
762 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_devmap_tests()
766 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
769 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }  in pmd_devmap_tests()  argument
770 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
773 static void __init pte_devmap_tests(struct pgtable_debug_args *args) { }  in pte_devmap_tests()  argument
774 static void __init pmd_devmap_tests(struct pgtable_debug_args *args) { }  in pmd_devmap_tests()  argument
775 static void __init pud_devmap_tests(struct pgtable_debug_args *args) { }  in pud_devmap_tests()  argument
778 static void __init pte_soft_dirty_tests(struct pgtable_debug_args *args)  in pte_soft_dirty_tests()  argument
780 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_soft_dirty_tests()
790 static void __init pte_swap_soft_dirty_tests(struct pgtable_debug_args *args)  in pte_swap_soft_dirty_tests()  argument
792 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_swap_soft_dirty_tests()
803 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args)  in pmd_soft_dirty_tests()  argument
814 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_soft_dirty_tests()
819 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args)  in pmd_swap_soft_dirty_tests()  argument
831 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_swap_soft_dirty_tests()
836 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) { }  in pmd_soft_dirty_tests()  argument
837 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) { }  in pmd_swap_soft_dirty_tests()  argument
840 static void __init pte_swap_exclusive_tests(struct pgtable_debug_args *args)  in pte_swap_exclusive_tests()  argument
843 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_swap_exclusive_tests()
853 static void __init pte_swap_tests(struct pgtable_debug_args *args)  in pte_swap_tests()  argument
859 	pte = pfn_pte(args->fixed_pte_pfn, args->page_prot);  in pte_swap_tests()
862 	WARN_ON(args->fixed_pte_pfn != pte_pfn(pte));  in pte_swap_tests()
866 static void __init pmd_swap_tests(struct pgtable_debug_args *args)  in pmd_swap_tests()  argument
875 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_swap_tests()
878 	WARN_ON(args->fixed_pmd_pfn != pmd_pfn(pmd));  in pmd_swap_tests()
881 static void __init pmd_swap_tests(struct pgtable_debug_args *args) { }  in pmd_swap_tests()  argument
884 static void __init swap_migration_tests(struct pgtable_debug_args *args)  in swap_migration_tests()  argument
899 	page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL;  in swap_migration_tests()
925 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args)  in hugetlb_basic_tests()  argument
935 	page = pfn_to_page(args->fixed_pmd_pfn);  in hugetlb_basic_tests()
936 	pte = mk_huge_pte(page, args->page_prot);  in hugetlb_basic_tests()
943 	pte = pfn_pte(args->fixed_pmd_pfn, args->page_prot);  in hugetlb_basic_tests()
949 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) { }  in hugetlb_basic_tests()  argument
953 static void __init pmd_thp_tests(struct pgtable_debug_args *args)  in pmd_thp_tests()  argument
972 	pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot);  in pmd_thp_tests()
982 static void __init pud_thp_tests(struct pgtable_debug_args *args)  in pud_thp_tests()  argument
990 	pud = pfn_pud(args->fixed_pud_pfn, args->page_prot);  in pud_thp_tests()
1002 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }  in pud_thp_tests()  argument
1005 static void __init pmd_thp_tests(struct pgtable_debug_args *args) { }  in pmd_thp_tests()  argument
1006 static void __init pud_thp_tests(struct pgtable_debug_args *args) { }  in pud_thp_tests()  argument
1021 static void __init destroy_args(struct pgtable_debug_args *args)  in destroy_args()  argument
1029 	    args->pud_pfn != ULONG_MAX) {  in destroy_args()
1030 		if (args->is_contiguous_page) {  in destroy_args()
1031 			free_contig_range(args->pud_pfn,  in destroy_args()
1034 			page = pfn_to_page(args->pud_pfn);  in destroy_args()
1038 		args->pud_pfn = ULONG_MAX;  in destroy_args()
1039 		args->pmd_pfn = ULONG_MAX;  in destroy_args()
1040 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1045 	    args->pmd_pfn != ULONG_MAX) {  in destroy_args()
1046 		if (args->is_contiguous_page) {  in destroy_args()
1047 			free_contig_range(args->pmd_pfn, (1 << HPAGE_PMD_ORDER));  in destroy_args()
1049 			page = pfn_to_page(args->pmd_pfn);  in destroy_args()
1053 		args->pmd_pfn = ULONG_MAX;  in destroy_args()
1054 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1057 	if (args->pte_pfn != ULONG_MAX) {  in destroy_args()
1058 		page = pfn_to_page(args->pte_pfn);  in destroy_args()
1061 		args->pte_pfn = ULONG_MAX;  in destroy_args()
1065 	if (args->start_ptep) {  in destroy_args()
1066 		pte_free(args->mm, args->start_ptep);  in destroy_args()
1067 		mm_dec_nr_ptes(args->mm);  in destroy_args()
1070 	if (args->start_pmdp) {  in destroy_args()
1071 		pmd_free(args->mm, args->start_pmdp);  in destroy_args()
1072 		mm_dec_nr_pmds(args->mm);  in destroy_args()
1075 	if (args->start_pudp) {  in destroy_args()
1076 		pud_free(args->mm, args->start_pudp);  in destroy_args()
1077 		mm_dec_nr_puds(args->mm);  in destroy_args()
1080 	if (args->start_p4dp)  in destroy_args()
1081 		p4d_free(args->mm, args->start_p4dp);  in destroy_args()
1084 	if (args->vma)  in destroy_args()
1085 		vm_area_free(args->vma);  in destroy_args()
1087 	if (args->mm)  in destroy_args()
1088 		mmdrop(args->mm);  in destroy_args()
1092 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order)  in debug_vm_pgtable_alloc_huge_page()  argument
1101 			args->is_contiguous_page = true;  in debug_vm_pgtable_alloc_huge_page()
1113 static int __init init_args(struct pgtable_debug_args *args)  in init_args()  argument
1126 	memset(args, 0, sizeof(*args));  in init_args()
1127 	args->vaddr              = get_random_vaddr();  in init_args()
1128 	args->page_prot          = vm_get_page_prot(VMFLAGS);  in init_args()
1129 	args->page_prot_none     = vm_get_page_prot(VM_NONE);  in init_args()
1130 	args->is_contiguous_page = false;  in init_args()
1131 	args->pud_pfn            = ULONG_MAX;  in init_args()
1132 	args->pmd_pfn            = ULONG_MAX;  in init_args()
1133 	args->pte_pfn            = ULONG_MAX;  in init_args()
1134 	args->fixed_pgd_pfn      = ULONG_MAX;  in init_args()
1135 	args->fixed_p4d_pfn      = ULONG_MAX;  in init_args()
1136 	args->fixed_pud_pfn      = ULONG_MAX;  in init_args()
1137 	args->fixed_pmd_pfn      = ULONG_MAX;  in init_args()
1138 	args->fixed_pte_pfn      = ULONG_MAX;  in init_args()
1141 	args->mm = mm_alloc();  in init_args()
1142 	if (!args->mm) {  in init_args()
1148 	args->vma = vm_area_alloc(args->mm);  in init_args()
1149 	if (!args->vma) {  in init_args()
1160 	args->pgdp = pgd_offset(args->mm, args->vaddr);  in init_args()
1161 	args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr);  in init_args()
1162 	if (!args->p4dp) {  in init_args()
1167 	args->start_p4dp = p4d_offset(args->pgdp, 0UL);  in init_args()
1168 	WARN_ON(!args->start_p4dp);  in init_args()
1170 	args->pudp = pud_alloc(args->mm, args->p4dp, args->vaddr);  in init_args()
1171 	if (!args->pudp) {  in init_args()
1176 	args->start_pudp = pud_offset(args->p4dp, 0UL);  in init_args()
1177 	WARN_ON(!args->start_pudp);  in init_args()
1179 	args->pmdp = pmd_alloc(args->mm, args->pudp, args->vaddr);  in init_args()
1180 	if (!args->pmdp) {  in init_args()
1185 	args->start_pmdp = pmd_offset(args->pudp, 0UL);  in init_args()
1186 	WARN_ON(!args->start_pmdp);  in init_args()
1188 	if (pte_alloc(args->mm, args->pmdp)) {  in init_args()
1193 	args->start_ptep = pmd_pgtable(READ_ONCE(*args->pmdp));  in init_args()
1194 	WARN_ON(!args->start_ptep);  in init_args()
1206 	args->fixed_pgd_pfn = __phys_to_pfn(phys & PGDIR_MASK);  in init_args()
1207 	args->fixed_p4d_pfn = __phys_to_pfn(phys & P4D_MASK);  in init_args()
1208 	args->fixed_pud_pfn = __phys_to_pfn(phys & PUD_MASK);  in init_args()
1209 	args->fixed_pmd_pfn = __phys_to_pfn(phys & PMD_MASK);  in init_args()
1210 	args->fixed_pte_pfn = __phys_to_pfn(phys & PAGE_MASK);  in init_args()
1211 	WARN_ON(!pfn_valid(args->fixed_pte_pfn));  in init_args()
1221 		page = debug_vm_pgtable_alloc_huge_page(args,  in init_args()
1224 			args->pud_pfn = page_to_pfn(page);  in init_args()
1225 			args->pmd_pfn = args->pud_pfn;  in init_args()
1226 			args->pte_pfn = args->pud_pfn;  in init_args()
1233 		page = debug_vm_pgtable_alloc_huge_page(args, HPAGE_PMD_ORDER);  in init_args()
1235 			args->pmd_pfn = page_to_pfn(page);  in init_args()
1236 			args->pte_pfn = args->pmd_pfn;  in init_args()
1243 		args->pte_pfn = page_to_pfn(page);  in init_args()
1248 	destroy_args(args);  in init_args()
1254 	struct pgtable_debug_args args;  in debug_vm_pgtable()  local
1259 	ret = init_args(&args);  in debug_vm_pgtable()
1277 		pte_basic_tests(&args, idx);  in debug_vm_pgtable()
1278 		pmd_basic_tests(&args, idx);  in debug_vm_pgtable()
1279 		pud_basic_tests(&args, idx);  in debug_vm_pgtable()
1289 	p4d_basic_tests(&args);  in debug_vm_pgtable()
1290 	pgd_basic_tests(&args);  in debug_vm_pgtable()
1292 	pmd_leaf_tests(&args);  in debug_vm_pgtable()
1293 	pud_leaf_tests(&args);  in debug_vm_pgtable()
1295 	pte_savedwrite_tests(&args);  in debug_vm_pgtable()
1296 	pmd_savedwrite_tests(&args);  in debug_vm_pgtable()
1298 	pte_special_tests(&args);  in debug_vm_pgtable()
1299 	pte_protnone_tests(&args);  in debug_vm_pgtable()
1300 	pmd_protnone_tests(&args);  in debug_vm_pgtable()
1302 	pte_devmap_tests(&args);  in debug_vm_pgtable()
1303 	pmd_devmap_tests(&args);  in debug_vm_pgtable()
1304 	pud_devmap_tests(&args);  in debug_vm_pgtable()
1306 	pte_soft_dirty_tests(&args);  in debug_vm_pgtable()
1307 	pmd_soft_dirty_tests(&args);  in debug_vm_pgtable()
1308 	pte_swap_soft_dirty_tests(&args);  in debug_vm_pgtable()
1309 	pmd_swap_soft_dirty_tests(&args);  in debug_vm_pgtable()
1311 	pte_swap_exclusive_tests(&args);  in debug_vm_pgtable()
1313 	pte_swap_tests(&args);  in debug_vm_pgtable()
1314 	pmd_swap_tests(&args);  in debug_vm_pgtable()
1316 	swap_migration_tests(&args);  in debug_vm_pgtable()
1318 	pmd_thp_tests(&args);  in debug_vm_pgtable()
1319 	pud_thp_tests(&args);  in debug_vm_pgtable()
1321 	hugetlb_basic_tests(&args);  in debug_vm_pgtable()
1328 	args.ptep = pte_offset_map_lock(args.mm, args.pmdp, args.vaddr, &ptl);  in debug_vm_pgtable()
1329 	pte_clear_tests(&args);  in debug_vm_pgtable()
1330 	pte_advanced_tests(&args);  in debug_vm_pgtable()
1331 	pte_unmap_unlock(args.ptep, ptl);  in debug_vm_pgtable()
1333 	ptl = pmd_lock(args.mm, args.pmdp);  in debug_vm_pgtable()
1334 	pmd_clear_tests(&args);  in debug_vm_pgtable()
1335 	pmd_advanced_tests(&args);  in debug_vm_pgtable()
1336 	pmd_huge_tests(&args);  in debug_vm_pgtable()
1337 	pmd_populate_tests(&args);  in debug_vm_pgtable()
1340 	ptl = pud_lock(args.mm, args.pudp);  in debug_vm_pgtable()
1341 	pud_clear_tests(&args);  in debug_vm_pgtable()
1342 	pud_advanced_tests(&args);  in debug_vm_pgtable()
1343 	pud_huge_tests(&args);  in debug_vm_pgtable()
1344 	pud_populate_tests(&args);  in debug_vm_pgtable()
1347 	spin_lock(&(args.mm->page_table_lock));  in debug_vm_pgtable()
1348 	p4d_clear_tests(&args);  in debug_vm_pgtable()
1349 	pgd_clear_tests(&args);  in debug_vm_pgtable()
1350 	p4d_populate_tests(&args);  in debug_vm_pgtable()
1351 	pgd_populate_tests(&args);  in debug_vm_pgtable()
1352 	spin_unlock(&(args.mm->page_table_lock));  in debug_vm_pgtable()
1354 	destroy_args(&args);  in debug_vm_pgtable()