| /Linux-v4.19/include/asm-generic/ |
| D | pgtable-nop4d-hack.h | 15 typedef struct { pgd_t pgd; } pud_t; member
27 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
28 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
29 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
30 static inline void pgd_clear(pgd_t *pgd) { } in pgd_clear() argument
31 #define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
33 #define pgd_populate(mm, pgd, pud) do { } while (0) argument
40 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) in pud_offset() argument
42 return (pud_t *)pgd; in pud_offset()
45 #define pud_val(x) (pgd_val((x).pgd))
[all …]
|
| D | pgtable-nop4d.h | 9 typedef struct { pgd_t pgd; } p4d_t; member
22 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
23 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
24 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
25 static inline void pgd_clear(pgd_t *pgd) { } in pgd_clear() argument
26 #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
28 #define pgd_populate(mm, pgd, p4d) do { } while (0) argument
35 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) in p4d_offset() argument
37 return (p4d_t *)pgd; in p4d_offset()
40 #define p4d_val(x) (pgd_val((x).pgd))
[all …]
|
| /Linux-v4.19/arch/arm/mm/ |
| D | pgd.c | 24 #define __pgd_free(pgd) kfree(pgd) argument
27 #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) argument
121 pgd_t *pgd; in pgd_free() local
129 pgd = pgd_base + pgd_index(0); in pgd_free()
130 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
133 pud = pud_offset(pgd, 0); in pgd_free()
150 pgd_clear(pgd); in pgd_free()
157 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { in pgd_free()
158 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
160 if (pgd_val(*pgd) & L_PGD_SWAPPER) in pgd_free()
[all …]
|
| /Linux-v4.19/arch/nds32/mm/ |
| D | mm-nds32.c | 41 void pgd_free(struct mm_struct *mm, pgd_t * pgd) in pgd_free() argument
46 if (!pgd) in pgd_free()
49 pmd = (pmd_t *) pgd; in pgd_free()
60 dec_zone_page_state(virt_to_page((unsigned long *)pgd), NR_PAGETABLE); in pgd_free()
65 free_pages((unsigned long)pgd, 0); in pgd_free()
76 pgd_t *pgd; in setup_mm_for_reboot() local
80 if (current->mm && current->mm->pgd) in setup_mm_for_reboot()
81 pgd = current->mm->pgd; in setup_mm_for_reboot()
83 pgd = init_mm.pgd; in setup_mm_for_reboot()
87 pmd = pmd_offset(pgd + i, i << PGDIR_SHIFT); in setup_mm_for_reboot()
|
| /Linux-v4.19/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable-4k.h | 56 #define pgd_none(pgd) (!pgd_val(pgd)) argument
57 #define pgd_bad(pgd) (pgd_val(pgd) == 0) argument
58 #define pgd_present(pgd) (pgd_val(pgd) != 0) argument
59 #define pgd_page_vaddr(pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) argument
68 static inline pte_t pgd_pte(pgd_t pgd) in pgd_pte() argument
70 return __pte(pgd_val(pgd)); in pgd_pte()
77 extern struct page *pgd_page(pgd_t pgd);
|
| /Linux-v4.19/arch/x86/mm/ |
| D | pgtable.c | 102 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
104 struct page *page = virt_to_page(pgd); in pgd_list_add()
109 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
111 struct page *page = virt_to_page(pgd); in pgd_list_del()
122 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) in pgd_set_mm() argument
124 virt_to_page(pgd)->pt_mm = mm; in pgd_set_mm()
132 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) in pgd_ctor() argument
140 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, in pgd_ctor()
147 pgd_set_mm(pgd, mm); in pgd_ctor()
148 pgd_list_add(pgd); in pgd_ctor()
[all …]
|
| D | kasan_init_64.c | 125 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, in kasan_populate_pgd() argument
132 if (pgd_none(*pgd)) { in kasan_populate_pgd()
134 pgd_populate(&init_mm, pgd, p); in kasan_populate_pgd()
137 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
147 pgd_t *pgd; in kasan_populate_shadow() local
152 pgd = pgd_offset_k(addr); in kasan_populate_shadow()
155 kasan_populate_pgd(pgd, addr, next, nid); in kasan_populate_shadow()
156 } while (pgd++, addr = next, addr != end); in kasan_populate_shadow()
173 pgd_t *pgd; in clear_pgds() local
178 pgd = pgd_offset_k(start); in clear_pgds()
[all …]
|
| D | pti.c | 131 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in __pti_set_user_pgtbl() argument
143 return pgd; in __pti_set_user_pgtbl()
149 kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; in __pti_set_user_pgtbl()
164 if ((pgd.pgd & (_PAGE_USER|_PAGE_PRESENT)) == (_PAGE_USER|_PAGE_PRESENT) && in __pti_set_user_pgtbl()
166 pgd.pgd |= _PAGE_NX; in __pti_set_user_pgtbl()
169 return pgd; in __pti_set_user_pgtbl()
180 pgd_t *pgd = kernel_to_user_pgdp(pgd_offset_k(address)); in pti_user_pagetable_walk_p4d() local
188 if (pgd_none(*pgd)) { in pti_user_pagetable_walk_p4d()
193 set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); in pti_user_pagetable_walk_p4d()
195 BUILD_BUG_ON(pgd_large(*pgd) != 0); in pti_user_pagetable_walk_p4d()
[all …]
|
| D | init_64.c | 113 pgd_t *pgd; in sync_global_pgds_l5() local
116 pgd = (pgd_t *)page_address(page) + pgd_index(addr); in sync_global_pgds_l5()
121 if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) in sync_global_pgds_l5()
122 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref)); in sync_global_pgds_l5()
124 if (pgd_none(*pgd)) in sync_global_pgds_l5()
125 set_pgd(pgd, *pgd_ref); in sync_global_pgds_l5()
154 pgd_t *pgd; in sync_global_pgds_l4() local
158 pgd = (pgd_t *)page_address(page) + pgd_index(addr); in sync_global_pgds_l4()
159 p4d = p4d_offset(pgd, addr); in sync_global_pgds_l4()
212 static p4d_t *fill_p4d(pgd_t *pgd, unsigned long vaddr) in fill_p4d() argument
[all …]
|
| /Linux-v4.19/arch/arm/include/asm/ |
| D | stage2_pgtable.h | 22 #define stage2_pgd_none(pgd) pgd_none(pgd) argument
23 #define stage2_pgd_clear(pgd) pgd_clear(pgd) argument
24 #define stage2_pgd_present(pgd) pgd_present(pgd) argument
25 #define stage2_pgd_populate(pgd, pud) pgd_populate(NULL, pgd, pud) argument
26 #define stage2_pud_offset(pgd, address) pud_offset(pgd, address) argument
|
| /Linux-v4.19/arch/sh/mm/ |
| D | hugetlbpage.c | 28 pgd_t *pgd; in huge_pte_alloc() local
33 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
34 if (pgd) { in huge_pte_alloc()
35 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
49 pgd_t *pgd; in huge_pte_offset() local
54 pgd = pgd_offset(mm, addr); in huge_pte_offset()
55 if (pgd) { in huge_pte_offset()
56 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| D | fault.c | 54 pgd_t *pgd; in show_pte() local
57 pgd = mm->pgd; in show_pte()
59 pgd = get_TTB(); in show_pte()
61 if (unlikely(!pgd)) in show_pte()
62 pgd = swapper_pg_dir; in show_pte()
65 printk(KERN_ALERT "pgd = %p\n", pgd); in show_pte()
66 pgd += pgd_index(addr); in show_pte()
68 (u32)(sizeof(*pgd) * 2), (u64)pgd_val(*pgd)); in show_pte()
75 if (pgd_none(*pgd)) in show_pte()
78 if (pgd_bad(*pgd)) { in show_pte()
[all …]
|
| /Linux-v4.19/arch/parisc/include/asm/ |
| D | pgalloc.h | 24 pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, in pgd_alloc() local
26 pgd_t *actual_pgd = pgd; in pgd_alloc()
28 if (likely(pgd != NULL)) { in pgd_alloc()
29 memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); in pgd_alloc()
38 + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); in pgd_alloc()
41 __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); in pgd_alloc()
47 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
50 pgd -= PTRS_PER_PGD; in pgd_free()
52 free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); in pgd_free()
59 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) in pgd_populate() argument
[all …]
|
| /Linux-v4.19/arch/um/kernel/ |
| D | mem.c | 88 pgd_t *pgd; in fixrange_init() local
97 pgd = pgd_base + i; in fixrange_init()
99 for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { in fixrange_init()
100 pud = pud_offset(pgd, vaddr); in fixrange_init()
116 pgd_t *pgd; in fixaddr_user_init() local
132 pgd = swapper_pg_dir + pgd_index(vaddr); in fixaddr_user_init()
133 pud = pud_offset(pgd, vaddr); in fixaddr_user_init()
184 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL); in pgd_alloc() local
186 if (pgd) { in pgd_alloc()
187 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_alloc()
[all …]
|
| /Linux-v4.19/arch/powerpc/include/asm/book3s/64/ |
| D | pgalloc.h | 68 static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) in radix__pgd_free() argument
71 free_page((unsigned long)pgd); in radix__pgd_free()
73 free_pages((unsigned long)pgd, 4); in radix__pgd_free()
79 pgd_t *pgd; in pgd_alloc() local
84 pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc()
91 kmemleak_no_scan(pgd); in pgd_alloc()
102 memset(pgd, 0, PGD_TABLE_SIZE); in pgd_alloc()
104 return pgd; in pgd_alloc()
107 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
110 return radix__pgd_free(mm, pgd); in pgd_free()
[all …]
|
| /Linux-v4.19/arch/arm64/include/asm/ |
| D | stage2_pgtable.h | 80 #define stage2_pgd_none(pgd) pgd_none(pgd) argument
81 #define stage2_pgd_clear(pgd) pgd_clear(pgd) argument
82 #define stage2_pgd_present(pgd) pgd_present(pgd) argument
83 #define stage2_pgd_populate(pgd, pud) pgd_populate(NULL, pgd, pud) argument
84 #define stage2_pud_offset(pgd, address) pud_offset(pgd, address) argument
|
| D | stage2_pgtable-nopud.h | 27 #define stage2_pgd_none(pgd) (0) argument
28 #define stage2_pgd_present(pgd) (1) argument
29 #define stage2_pgd_clear(pgd) do { } while (0) argument
30 #define stage2_pgd_populate(pgd, pud) do { } while (0) argument
32 #define stage2_pud_offset(pgd, address) ((pud_t *)(pgd)) argument
|
| /Linux-v4.19/mm/kasan/ |
| D | kasan_init.c | 38 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
40 return pgd_page(pgd) == virt_to_page(lm_alias(kasan_zero_p4d)); in kasan_p4d_table()
43 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
173 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, in zero_p4d_populate() argument
176 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate()
223 pgd_t *pgd = pgd_offset_k(addr); in kasan_populate_zero_shadow() local
252 pgd_populate(&init_mm, pgd, lm_alias(kasan_zero_p4d)); in kasan_populate_zero_shadow()
254 p4d = p4d_offset(pgd, addr); in kasan_populate_zero_shadow()
263 if (pgd_none(*pgd)) { in kasan_populate_zero_shadow()
267 p = p4d_alloc(&init_mm, pgd, addr); in kasan_populate_zero_shadow()
[all …]
|
| /Linux-v4.19/arch/hexagon/include/asm/ |
| D | pgalloc.h | 36 pgd_t *pgd; in pgd_alloc() local
38 pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); in pgd_alloc()
48 memcpy(pgd, swapper_pg_dir, PTRS_PER_PGD*sizeof(pgd_t)); in pgd_alloc()
52 mm->context.ptbase = __pa(pgd); in pgd_alloc()
54 return pgd; in pgd_alloc()
57 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
59 free_page((unsigned long) pgd); in pgd_free()
135 pmdindex = (pgd_t *)pmd - mm->pgd; in pmd_populate_kernel()
136 ppmd = (pmd_t *)current->active_mm->pgd + pmdindex; in pmd_populate_kernel()
|
| /Linux-v4.19/arch/mips/mm/ |
| D | hugetlbpage.c | 27 pgd_t *pgd; in huge_pte_alloc() local
31 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
32 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
42 pgd_t *pgd; in huge_pte_offset() local
46 pgd = pgd_offset(mm, addr); in huge_pte_offset()
47 if (pgd_present(*pgd)) { in huge_pte_offset()
48 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| /Linux-v4.19/virt/kvm/arm/ |
| D | mmu.c | 151 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr) in clear_stage2_pgd_entry() argument
153 pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL); in clear_stage2_pgd_entry()
154 stage2_pgd_clear(pgd); in clear_stage2_pgd_entry()
157 put_page(virt_to_page(pgd)); in clear_stage2_pgd_entry()
288 static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd, in unmap_stage2_puds() argument
294 start_pud = pud = stage2_pud_offset(pgd, addr); in unmap_stage2_puds()
312 clear_stage2_pgd_entry(kvm, pgd, start_addr); in unmap_stage2_puds()
328 pgd_t *pgd; in unmap_stage2_range() local
335 pgd = kvm->arch.pgd + stage2_pgd_index(addr); in unmap_stage2_range()
342 if (!READ_ONCE(kvm->arch.pgd)) in unmap_stage2_range()
[all …]
|
| /Linux-v4.19/arch/riscv/include/asm/ |
| D | pgalloc.h | 50 pgd_t *pgd; in pgd_alloc() local
52 pgd = (pgd_t *)__get_free_page(GFP_KERNEL); in pgd_alloc()
53 if (likely(pgd != NULL)) { in pgd_alloc()
54 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_alloc()
56 memcpy(pgd + USER_PTRS_PER_PGD, in pgd_alloc()
57 init_mm.pgd + USER_PTRS_PER_PGD, in pgd_alloc()
60 return pgd; in pgd_alloc()
63 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
65 free_page((unsigned long)pgd); in pgd_free()
|
| /Linux-v4.19/arch/x86/platform/efi/ |
| D | efi_64.c | 82 int pgd; in efi_call_phys_prolog() local
103 for (pgd = 0; pgd < n_pgds; pgd++) { in efi_call_phys_prolog()
104 addr_pgd = (unsigned long)(pgd * PGDIR_SIZE); in efi_call_phys_prolog()
105 vaddr = (unsigned long)__va(pgd * PGDIR_SIZE); in efi_call_phys_prolog()
107 save_pgd[pgd] = *pgd_efi; in efi_call_phys_prolog()
138 pgd_offset_k(pgd * PGDIR_SIZE)->pgd &= ~_PAGE_NX; in efi_call_phys_prolog()
154 pgd_t *pgd; in efi_call_phys_epilog() local
166 pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE); in efi_call_phys_epilog()
169 if (!pgd_present(*pgd)) in efi_call_phys_epilog()
173 p4d = p4d_offset(pgd, in efi_call_phys_epilog()
[all …]
|
| /Linux-v4.19/arch/x86/kernel/ |
| D | machine_kexec_32.c | 59 free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER); in machine_kexec_free_page_tables()
60 image->arch.pgd = NULL; in machine_kexec_free_page_tables()
75 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in machine_kexec_alloc_page_tables()
83 if (!image->arch.pgd || in machine_kexec_alloc_page_tables()
94 pgd_t *pgd, pmd_t *pmd, pte_t *pte, in machine_kexec_page_table_set_one() argument
100 pgd += pgd_index(vaddr); in machine_kexec_page_table_set_one()
102 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) in machine_kexec_page_table_set_one()
103 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); in machine_kexec_page_table_set_one()
105 p4d = p4d_offset(pgd, vaddr); in machine_kexec_page_table_set_one()
124 image->arch.pgd, pmd, image->arch.pte0, in machine_kexec_prepare_page_tables()
[all …]
|
| /Linux-v4.19/arch/x86/power/ |
| D | hibernate_32.c | 33 static pmd_t *resume_one_md_table_init(pgd_t *pgd) in resume_one_md_table_init() argument
44 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); in resume_one_md_table_init()
45 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
50 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
87 pgd_t *pgd; in resume_physical_mapping_init() local
93 pgd = pgd_base + pgd_idx; in resume_physical_mapping_init()
96 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { in resume_physical_mapping_init()
97 pmd = resume_one_md_table_init(pgd); in resume_physical_mapping_init()
|
