Home
last modified time | relevance | path

Searched refs:p4d (Results 1 – 25 of 69) sorted by relevance

123

/Linux-v4.19/include/asm-generic/
D5level-fixup.h16 #define pud_alloc(mm, p4d, address) \ argument
17 ((unlikely(pgd_none(*(p4d))) && __pud_alloc(mm, p4d, address)) ? \
18 NULL : pud_offset(p4d, address))
22 #define p4d_none(p4d) 0 argument
23 #define p4d_bad(p4d) 0 argument
24 #define p4d_present(p4d) 1 argument
25 #define p4d_ERROR(p4d) do { } while (0) argument
26 #define p4d_clear(p4d) pgd_clear(p4d) argument
27 #define p4d_val(p4d) pgd_val(p4d) argument
28 #define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud) argument
[all …]
Dpgtable-nopud.h19 typedef struct { p4d_t p4d; } pud_t; member
31 static inline int p4d_none(p4d_t p4d) { return 0; } in p4d_none() argument
32 static inline int p4d_bad(p4d_t p4d) { return 0; } in p4d_bad() argument
33 static inline int p4d_present(p4d_t p4d) { return 1; } in p4d_present() argument
34 static inline void p4d_clear(p4d_t *p4d) { } in p4d_clear() argument
35 #define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
37 #define p4d_populate(mm, p4d, pud) do { } while (0) argument
44 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) in pud_offset() argument
46 return (pud_t *)p4d; in pud_offset()
49 #define pud_val(x) (p4d_val((x).p4d))
[all …]
Dpgtable.h368 #define p4d_access_permitted(p4d, write) \ argument
369 (p4d_present(p4d) && (!(write) || p4d_write(p4d)))
538 static inline int p4d_none_or_clear_bad(p4d_t *p4d) in p4d_none_or_clear_bad() argument
540 if (p4d_none(*p4d)) in p4d_none_or_clear_bad()
542 if (unlikely(p4d_bad(*p4d))) { in p4d_none_or_clear_bad()
543 p4d_clear_bad(p4d); in p4d_none_or_clear_bad()
1005 int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);
1006 int p4d_clear_huge(p4d_t *p4d);
1008 static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) in p4d_set_huge() argument
1012 static inline int p4d_clear_huge(p4d_t *p4d) in p4d_clear_huge() argument
[all …]
/Linux-v4.19/mm/kasan/
Dkasan_init.c50 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
52 return p4d_page(p4d) == virt_to_page(lm_alias(kasan_zero_pud)); in kasan_pud_table()
55 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
138 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
141 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
176 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate() local
185 p4d_populate(&init_mm, p4d, lm_alias(kasan_zero_pud)); in zero_p4d_populate()
186 pud = pud_offset(p4d, addr); in zero_p4d_populate()
194 if (p4d_none(*p4d)) { in zero_p4d_populate()
198 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate()
[all …]
/Linux-v4.19/arch/x86/mm/
Dinit_64.c155 p4d_t *p4d; in sync_global_pgds_l4() local
159 p4d = p4d_offset(pgd, addr); in sync_global_pgds_l4()
164 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) in sync_global_pgds_l4()
165 BUG_ON(p4d_page_vaddr(*p4d) in sync_global_pgds_l4()
168 if (p4d_none(*p4d)) in sync_global_pgds_l4()
169 set_p4d(p4d, *p4d_ref); in sync_global_pgds_l4()
215 p4d_t *p4d = (p4d_t *)spp_getpage(); in fill_p4d() local
216 pgd_populate(&init_mm, pgd, p4d); in fill_p4d()
217 if (p4d != p4d_offset(pgd, 0)) in fill_p4d()
219 p4d, p4d_offset(pgd, 0)); in fill_p4d()
[all …]
Dident_map.c67 p4d_t *p4d = p4d_page + p4d_index(addr); in ident_p4d_init() local
74 if (p4d_present(*p4d)) { in ident_p4d_init()
75 pud = pud_offset(p4d, 0); in ident_p4d_init()
83 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); in ident_p4d_init()
106 p4d_t *p4d; in kernel_ident_mapping_init() local
113 p4d = p4d_offset(pgd, 0); in kernel_ident_mapping_init()
114 result = ident_p4d_init(info, p4d, addr, next); in kernel_ident_mapping_init()
120 p4d = (p4d_t *)info->alloc_pgt_page(info->context); in kernel_ident_mapping_init()
121 if (!p4d) in kernel_ident_mapping_init()
123 result = ident_p4d_init(info, p4d, addr, next); in kernel_ident_mapping_init()
[all …]
Dkasan_init_64.c105 static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, in kasan_populate_p4d() argument
111 if (p4d_none(*p4d)) { in kasan_populate_p4d()
114 p4d_populate(&init_mm, p4d, p); in kasan_populate_p4d()
117 pud = pud_offset(p4d, addr); in kasan_populate_p4d()
129 p4d_t *p4d; in kasan_populate_pgd() local
137 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
140 kasan_populate_p4d(p4d, addr, next, nid); in kasan_populate_pgd()
141 } while (p4d++, addr = next, addr != end); in kasan_populate_pgd()
196 unsigned long p4d; in early_p4d_offset() local
201 p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK; in early_p4d_offset()
[all …]
Dpti.c209 p4d_t *p4d; in pti_user_pagetable_walk_pmd() local
212 p4d = pti_user_pagetable_walk_p4d(address); in pti_user_pagetable_walk_pmd()
213 if (!p4d) in pti_user_pagetable_walk_pmd()
216 BUILD_BUG_ON(p4d_large(*p4d) != 0); in pti_user_pagetable_walk_pmd()
217 if (p4d_none(*p4d)) { in pti_user_pagetable_walk_pmd()
222 set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); in pti_user_pagetable_walk_pmd()
225 pud = pud_offset(p4d, address); in pti_user_pagetable_walk_pmd()
323 p4d_t *p4d; in pti_clone_pgtable() local
333 p4d = p4d_offset(pgd, addr); in pti_clone_pgtable()
334 if (WARN_ON(p4d_none(*p4d))) in pti_clone_pgtable()
[all …]
Dfault.c237 p4d_t *p4d, *p4d_k; in vmalloc_sync_one() local
252 p4d = p4d_offset(pgd, address); in vmalloc_sync_one()
257 pud = pud_offset(p4d, address); in vmalloc_sync_one()
372 p4d_t *p4d; in dump_pagetable() local
385 p4d = p4d_offset(pgd, address); in dump_pagetable()
386 pud = pud_offset(p4d, address); in dump_pagetable()
421 p4d_t *p4d, *p4d_k; in vmalloc_fault() local
452 p4d = p4d_offset(pgd, address); in vmalloc_fault()
457 if (p4d_none(*p4d) && !pgtable_l5_enabled()) { in vmalloc_fault()
458 set_p4d(p4d, *p4d_k); in vmalloc_fault()
[all …]
Dpgtable_32.c31 p4d_t *p4d; in set_pte_vaddr() local
41 p4d = p4d_offset(pgd, vaddr); in set_pte_vaddr()
42 if (p4d_none(*p4d)) { in set_pte_vaddr()
46 pud = pud_offset(p4d, vaddr); in set_pte_vaddr()
Dpageattr.c376 p4d_t *p4d; in lookup_address_in_pgd() local
385 p4d = p4d_offset(pgd, address); in lookup_address_in_pgd()
386 if (p4d_none(*p4d)) in lookup_address_in_pgd()
390 if (p4d_large(*p4d) || !p4d_present(*p4d)) in lookup_address_in_pgd()
391 return (pte_t *)p4d; in lookup_address_in_pgd()
393 pud = pud_offset(p4d, address); in lookup_address_in_pgd()
445 p4d_t *p4d; in lookup_pmd_address() local
452 p4d = p4d_offset(pgd, address); in lookup_pmd_address()
453 if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d)) in lookup_pmd_address()
456 pud = pud_offset(p4d, address); in lookup_pmd_address()
[all …]
Dmem_encrypt_identity.c97 p4d_t *p4d; in sme_prepare_pgd() local
103 p4d = ppd->pgtable_area; in sme_prepare_pgd()
104 memset(p4d, 0, sizeof(*p4d) * PTRS_PER_P4D); in sme_prepare_pgd()
105 ppd->pgtable_area += sizeof(*p4d) * PTRS_PER_P4D; in sme_prepare_pgd()
106 set_pgd(pgd, __pgd(PGD_FLAGS | __pa(p4d))); in sme_prepare_pgd()
109 p4d = p4d_offset(pgd, ppd->vaddr); in sme_prepare_pgd()
110 if (p4d_none(*p4d)) { in sme_prepare_pgd()
114 set_p4d(p4d, __p4d(P4D_FLAGS | __pa(pud))); in sme_prepare_pgd()
117 pud = pud_offset(p4d, ppd->vaddr); in sme_prepare_pgd()
/Linux-v4.19/arch/x86/include/asm/
Dpgalloc.h138 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument
141 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); in p4d_populate()
168 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
172 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); in pgd_populate()
173 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); in pgd_populate()
185 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) in p4d_free() argument
190 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); in p4d_free()
191 free_page((unsigned long)p4d); in p4d_free()
194 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
196 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, in __p4d_free_tlb() argument
[all …]
Dpgtable_types.h309 typedef struct { p4dval_t p4d; } p4d_t; member
316 static inline p4dval_t native_p4d_val(p4d_t p4d) in native_p4d_val() argument
318 return p4d.p4d; in native_p4d_val()
328 static inline p4dval_t native_p4d_val(p4d_t p4d) in native_p4d_val() argument
330 return native_pgd_val(p4d.pgd); in native_p4d_val()
351 return (pud_t) { .p4d.pgd = native_make_pgd(val) }; in native_make_pud()
356 return native_pgd_val(pud.p4d.pgd); in native_pud_val()
377 return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; in native_make_pmd()
382 return native_pgd_val(pmd.pud.p4d.pgd); in native_pmd_val()
386 static inline p4dval_t p4d_pfn_mask(p4d_t p4d) in p4d_pfn_mask() argument
[all …]
Dpgtable.h75 # define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d) argument
79 #define p4d_clear(p4d) native_p4d_clear(p4d) argument
216 static inline unsigned long p4d_pfn(p4d_t p4d) in p4d_pfn() argument
218 return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT; in p4d_pfn()
226 static inline int p4d_large(p4d_t p4d) in p4d_large() argument
884 static inline int p4d_none(p4d_t p4d) in p4d_none() argument
886 return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; in p4d_none()
889 static inline int p4d_present(p4d_t p4d) in p4d_present() argument
891 return p4d_flags(p4d) & _PAGE_PRESENT; in p4d_present()
894 static inline unsigned long p4d_page_vaddr(p4d_t p4d) in p4d_page_vaddr() argument
[all …]
/Linux-v4.19/arch/x86/power/
Dhibernate_64.c53 p4d_t *p4d = NULL; in set_up_temporary_text_mapping() local
76 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); in set_up_temporary_text_mapping()
77 if (!p4d) in set_up_temporary_text_mapping()
93 if (p4d) { in set_up_temporary_text_mapping()
95 pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); in set_up_temporary_text_mapping()
97 set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); in set_up_temporary_text_mapping()
151 p4d_t *p4d; in relocate_restore_code() local
165 p4d = p4d_offset(pgd, relocated_restore_code); in relocate_restore_code()
166 if (p4d_large(*p4d)) { in relocate_restore_code()
167 set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX)); in relocate_restore_code()
[all …]
Dhibernate_32.c35 p4d_t *p4d; in resume_one_md_table_init() local
45 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
46 pud = pud_offset(p4d, 0); in resume_one_md_table_init()
50 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
51 pud = pud_offset(p4d, 0); in resume_one_md_table_init()
/Linux-v4.19/mm/
Dsparse-vmemmap.c180 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) in vmemmap_pud_populate() argument
182 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate()
194 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local
195 if (p4d_none(*p4d)) { in vmemmap_p4d_populate()
199 p4d_populate(&init_mm, p4d, p); in vmemmap_p4d_populate()
201 return p4d; in vmemmap_p4d_populate()
221 p4d_t *p4d; in vmemmap_populate_basepages() local
230 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_basepages()
231 if (!p4d) in vmemmap_populate_basepages()
233 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_basepages()
/Linux-v4.19/lib/
Dioremap.c106 static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, in ioremap_pud_range() argument
113 pud = pud_alloc(&init_mm, p4d, addr); in ioremap_pud_range()
136 p4d_t *p4d; in ioremap_p4d_range() local
140 p4d = p4d_alloc(&init_mm, pgd, addr); in ioremap_p4d_range()
141 if (!p4d) in ioremap_p4d_range()
149 if (p4d_set_huge(p4d, phys_addr + addr, prot)) in ioremap_p4d_range()
153 if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot)) in ioremap_p4d_range()
155 } while (p4d++, addr = next, addr != end); in ioremap_p4d_range()
/Linux-v4.19/arch/x86/kernel/
Dhead64.c120 p4dval_t *p4d; in __startup_64() local
161 p4d = fixup_pointer(&level4_kernel_pgt, physaddr); in __startup_64()
162 p4d[511] += load_delta; in __startup_64()
187 p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr); in __startup_64()
190 pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
191 pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
194 p4d[i + 0] = (pgdval_t)pud + pgtable_flags; in __startup_64()
195 p4d[i + 1] = (pgdval_t)pud + pgtable_flags; in __startup_64()
284 p4dval_t p4d, *p4d_p; in __early_make_pgtable() local
316 p4d = *p4d_p; in __early_make_pgtable()
[all …]
Dmachine_kexec_64.c41 free_page((unsigned long)image->arch.p4d); in free_transition_pgtable()
42 image->arch.p4d = NULL; in free_transition_pgtable()
53 p4d_t *p4d; in init_transition_pgtable() local
64 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); in init_transition_pgtable()
65 if (!p4d) in init_transition_pgtable()
67 image->arch.p4d = p4d; in init_transition_pgtable()
68 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE)); in init_transition_pgtable()
70 p4d = p4d_offset(pgd, vaddr); in init_transition_pgtable()
71 if (!p4d_present(*p4d)) { in init_transition_pgtable()
76 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); in init_transition_pgtable()
[all …]
/Linux-v4.19/arch/s390/mm/
Dpage-states.c126 static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end) in mark_kernel_pud() argument
133 pud = pud_offset(p4d, addr); in mark_kernel_pud()
151 p4d_t *p4d; in mark_kernel_p4d() local
154 p4d = p4d_offset(pgd, addr); in mark_kernel_p4d()
157 if (p4d_none(*p4d)) in mark_kernel_p4d()
159 if (!p4d_folded(*p4d)) { in mark_kernel_p4d()
160 page = virt_to_page(p4d_val(*p4d)); in mark_kernel_p4d()
164 mark_kernel_pud(p4d, addr, next); in mark_kernel_p4d()
165 } while (p4d++, addr = next, addr != end); in mark_kernel_p4d()
Dgup.c169 static inline int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, in gup_pud_range() argument
176 if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) in gup_pud_range()
177 pudp = (pud_t *) p4d_deref(p4d); in gup_pud_range()
201 p4d_t *p4dp, p4d; in gup_p4d_range() local
208 p4d = *p4dp; in gup_p4d_range()
211 if (p4d_none(p4d)) in gup_p4d_range()
213 if (!gup_pud_range(p4dp, p4d, addr, next, write, pages, nr)) in gup_p4d_range()
/Linux-v4.19/arch/s390/include/asm/
Dpgalloc.h59 #define p4d_free(mm, p4d) crst_table_free(mm, (unsigned long *) p4d) argument
90 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
92 pgd_val(*pgd) = _REGION1_ENTRY | __pa(p4d); in pgd_populate()
95 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument
97 p4d_val(*p4d) = _REGION2_ENTRY | __pa(pud); in p4d_populate()
Dpgtable.h602 static inline int p4d_folded(p4d_t p4d) in p4d_folded() argument
604 return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2; in p4d_folded()
607 static inline int p4d_present(p4d_t p4d) in p4d_present() argument
609 if (p4d_folded(p4d)) in p4d_present()
611 return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL; in p4d_present()
614 static inline int p4d_none(p4d_t p4d) in p4d_none() argument
616 if (p4d_folded(p4d)) in p4d_none()
618 return p4d_val(p4d) == _REGION2_ENTRY_EMPTY; in p4d_none()
621 static inline unsigned long p4d_pfn(p4d_t p4d) in p4d_pfn() argument
626 return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT; in p4d_pfn()
[all …]

123