Home
last modified time | relevance | path

Searched refs:pudp (Results 1 – 25 of 59) sorted by relevance

123

/Linux-v5.4/arch/arm64/include/asm/
Dpgalloc.h47 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
49 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate()
52 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument
54 __pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE); in pud_populate()
57 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
70 static inline void pud_free(struct mm_struct *mm, pud_t *pudp) in pud_free() argument
72 BUG_ON((unsigned long)pudp & (PAGE_SIZE-1)); in pud_free()
73 free_page((unsigned long)pudp); in pud_free()
76 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) in __pgd_populate() argument
78 set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot)); in __pgd_populate()
[all …]
Dkvm_mmu.h174 #define kvm_mk_pgd(pudp) \
175 __pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE)
177 #define kvm_set_pud(pudp, pud) set_pud(pudp, pud)
262 static inline void kvm_set_s2pud_readonly(pud_t *pudp)
264 kvm_set_s2pte_readonly((pte_t *)pudp);
267 static inline bool kvm_s2pud_readonly(pud_t *pudp)
269 return kvm_s2pte_readonly((pte_t *)pudp);
272 static inline bool kvm_s2pud_exec(pud_t *pudp)
274 return !(READ_ONCE(pud_val(*pudp)) & PUD_S2_XN);
296 #define hyp_pud_table_empty(pudp) (0)
[all …]
Dtlb.h63 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, in __pud_free_tlb() argument
66 tlb_remove_table(tlb, virt_to_page(pudp)); in __pud_free_tlb()
/Linux-v5.4/arch/powerpc/mm/nohash/
Dbook3e_pgtable.c76 pud_t *pudp; in map_kernel_page() local
83 pudp = pud_alloc(&init_mm, pgdp, ea); in map_kernel_page()
84 if (!pudp) in map_kernel_page()
86 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
96 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); in map_kernel_page()
97 pgd_populate(&init_mm, pgdp, pudp); in map_kernel_page()
100 pudp = pud_offset(pgdp, ea); in map_kernel_page()
101 if (pud_none(*pudp)) { in map_kernel_page()
103 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
105 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
/Linux-v5.4/arch/arm64/mm/
Dmmu.c199 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, in init_pmd() argument
206 pmdp = pmd_set_fixmap_offset(pudp, addr); in init_pmd()
236 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, in alloc_init_cont_pmd() argument
242 pud_t pud = READ_ONCE(*pudp); in alloc_init_cont_pmd()
252 __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); in alloc_init_cont_pmd()
253 pud = READ_ONCE(*pudp); in alloc_init_cont_pmd()
267 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); in alloc_init_cont_pmd()
291 pud_t *pudp; in alloc_init_pud() local
303 pudp = pud_set_fixmap_offset(pgdp, addr); in alloc_init_pud()
305 pud_t old_pud = READ_ONCE(*pudp); in alloc_init_pud()
[all …]
Dhugetlbpage.c70 pud_t *pudp; in find_num_contig() local
74 pudp = pud_offset(pgdp, addr); in find_num_contig()
75 pmdp = pmd_offset(pudp, addr); in find_num_contig()
220 pud_t *pudp; in huge_pte_alloc() local
225 pudp = pud_alloc(mm, pgdp, addr); in huge_pte_alloc()
226 if (!pudp) in huge_pte_alloc()
230 ptep = (pte_t *)pudp; in huge_pte_alloc()
232 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc()
245 pud_none(READ_ONCE(*pudp))) in huge_pte_alloc()
246 ptep = huge_pmd_share(mm, addr, pudp); in huge_pte_alloc()
[all …]
Dkasan_init.c74 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, in kasan_pmd_offset() argument
77 if (pud_none(READ_ONCE(*pudp))) { in kasan_pmd_offset()
81 __pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE); in kasan_pmd_offset()
84 return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); in kasan_pmd_offset()
117 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument
121 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate()
133 pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early); in kasan_pud_populate() local
137 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate()
138 } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); in kasan_pud_populate()
Dpageattr.c201 pud_t *pudp, pud; in kernel_page_present() local
213 pudp = pud_offset(pgdp, addr); in kernel_page_present()
214 pud = READ_ONCE(*pudp); in kernel_page_present()
220 pmdp = pmd_offset(pudp, addr); in kernel_page_present()
Ddump.c306 static void walk_pmd(struct pg_state *st, pud_t *pudp, unsigned long start, in walk_pmd() argument
310 pmd_t *pmdp = pmd_offset(pudp, start); in walk_pmd()
329 pud_t *pudp = pud_offset(pgdp, start); in walk_pud() local
332 pud_t pud = READ_ONCE(*pudp); in walk_pud()
339 walk_pmd(st, pudp, addr, next); in walk_pud()
341 } while (pudp++, addr = next, addr != end); in walk_pud()
/Linux-v5.4/arch/s390/mm/
Dhugetlbpage.c177 pud_t *pudp = (pud_t *) ptep; in huge_ptep_get_and_clear() local
180 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); in huge_ptep_get_and_clear()
191 pud_t *pudp; in huge_pte_alloc() local
197 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
198 if (pudp) { in huge_pte_alloc()
200 return (pte_t *) pudp; in huge_pte_alloc()
202 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc()
213 pud_t *pudp; in huge_pte_offset() local
220 pudp = pud_offset(p4dp, addr); in huge_pte_offset()
221 if (pud_present(*pudp)) { in huge_pte_offset()
[all …]
Dpageattr.c155 static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, in walk_pmd_level() argument
162 pmdp = pmd_offset(pudp, addr); in walk_pmd_level()
187 static int split_pud_page(pud_t *pudp, unsigned long addr) in split_pud_page() argument
197 pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; in split_pud_page()
198 ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); in split_pud_page()
199 nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); in split_pud_page()
210 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in split_pud_page()
216 static void modify_pud_page(pud_t *pudp, unsigned long addr, in modify_pud_page() argument
219 pud_t new = *pudp; in modify_pud_page()
229 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in modify_pud_page()
[all …]
Dpgtable.c460 unsigned long addr, pud_t *pudp) in pudp_idte_local() argument
463 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, in pudp_idte_local()
466 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); in pudp_idte_local()
470 unsigned long addr, pud_t *pudp) in pudp_idte_global() argument
473 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, in pudp_idte_global()
476 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); in pudp_idte_global()
482 __pmdp_csp((pmd_t *) pudp); in pudp_idte_global()
486 unsigned long addr, pud_t *pudp) in pudp_flush_direct() argument
490 old = *pudp; in pudp_flush_direct()
496 pudp_idte_local(mm, addr, pudp); in pudp_flush_direct()
[all …]
/Linux-v5.4/arch/x86/include/asm/
Dpgtable-3level.h101 static inline void native_set_pud(pud_t *pudp, pud_t pud) in native_set_pud() argument
104 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); in native_set_pud()
106 set_64bit((unsigned long long *)(pudp), native_pud_val(pud)); in native_set_pud()
130 static inline void native_pud_clear(pud_t *pudp) in native_pud_clear() argument
134 static inline void pud_clear(pud_t *pudp) in pud_clear() argument
136 set_pud(pudp, __pud(0)); in pud_clear()
230 static inline pud_t native_pudp_get_and_clear(pud_t *pudp) in native_pudp_get_and_clear() argument
232 union split_pud res, *orig = (union split_pud *)pudp; in native_pudp_get_and_clear()
235 pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0)); in native_pudp_get_and_clear()
/Linux-v5.4/arch/riscv/include/asm/
Dpgtable-64.h46 static inline void set_pud(pud_t *pudp, pud_t pud) in set_pud() argument
48 *pudp = pud; in set_pud()
51 static inline void pud_clear(pud_t *pudp) in pud_clear() argument
53 set_pud(pudp, __pud(0)); in pud_clear()
/Linux-v5.4/arch/powerpc/mm/book3s64/
Dradix_pgtable.c66 pud_t *pudp; in early_map_kernel_page() local
72 pudp = early_alloc_pgtable(PUD_TABLE_SIZE, nid, in early_map_kernel_page()
74 pgd_populate(&init_mm, pgdp, pudp); in early_map_kernel_page()
76 pudp = pud_offset(pgdp, ea); in early_map_kernel_page()
78 ptep = (pte_t *)pudp; in early_map_kernel_page()
81 if (pud_none(*pudp)) { in early_map_kernel_page()
84 pud_populate(&init_mm, pudp, pmdp); in early_map_kernel_page()
86 pmdp = pmd_offset(pudp, ea); in early_map_kernel_page()
116 pud_t *pudp; in __map_kernel_page() local
138 pudp = pud_alloc(&init_mm, pgdp, ea); in __map_kernel_page()
[all …]
Dhash_pgtable.c151 pud_t *pudp; in hash__map_kernel_page() local
158 pudp = pud_alloc(&init_mm, pgdp, ea); in hash__map_kernel_page()
159 if (!pudp) in hash__map_kernel_page()
161 pmdp = pmd_alloc(&init_mm, pudp, ea); in hash__map_kernel_page()
/Linux-v5.4/include/asm-generic/
Dpgtable.h41 unsigned long address, pud_t *pudp,
52 unsigned long address, pud_t *pudp, in pudp_set_access_flags() argument
149 pud_t *pudp) in pudp_huge_get_and_clear() argument
151 pud_t pud = *pudp; in pudp_huge_get_and_clear()
153 pud_clear(pudp); in pudp_huge_get_and_clear()
171 unsigned long address, pud_t *pudp, in pudp_huge_get_and_clear_full() argument
174 return pudp_huge_get_and_clear(mm, address, pudp); in pudp_huge_get_and_clear_full()
217 pud_t *pudp);
272 unsigned long address, pud_t *pudp) in pudp_set_wrprotect() argument
274 pud_t old_pud = *pudp; in pudp_set_wrprotect()
[all …]
Dtlb.h539 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) argument
542 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ argument
546 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \
589 #define pud_free_tlb(tlb, pudp, address) \ argument
594 __pud_free_tlb(tlb, pudp, address); \
601 #define p4d_free_tlb(tlb, pudp, address) \ argument
605 __p4d_free_tlb(tlb, pudp, address); \
/Linux-v5.4/include/trace/events/
Dxen.h223 TP_PROTO(pud_t *pudp, pud_t pudval),
224 TP_ARGS(pudp, pudval),
226 __field(pud_t *, pudp)
229 TP_fast_assign(__entry->pudp = pudp;
232 __entry->pudp,
258 TP_PROTO(pud_t *pudp, pud_t pudval),
259 TP_ARGS(pudp, pudval),
261 __field(pud_t *, pudp)
264 TP_fast_assign(__entry->pudp = pudp;
267 __entry->pudp,
/Linux-v5.4/arch/arm64/kernel/
Dhibernate.c206 pud_t *pudp; in create_safe_exec_page() local
227 pudp = allocator(mask); in create_safe_exec_page()
228 if (!pudp) { in create_safe_exec_page()
232 pgd_populate(&init_mm, pgdp, pudp); in create_safe_exec_page()
235 pudp = pud_offset(pgdp, dst_addr); in create_safe_exec_page()
236 if (pud_none(READ_ONCE(*pudp))) { in create_safe_exec_page()
242 pud_populate(&init_mm, pudp, pmdp); in create_safe_exec_page()
245 pmdp = pmd_offset(pudp, dst_addr); in create_safe_exec_page()
/Linux-v5.4/arch/arm/include/asm/
Dpgtable-3level.h138 #define pud_clear(pudp) \ argument
140 *pudp = __pud(0); \
141 clean_pmd_entry(pudp); \
144 #define set_pud(pudp, pud) \ argument
146 *pudp = pud; \
147 flush_pmd_entry(pudp); \
/Linux-v5.4/arch/powerpc/include/asm/nohash/64/
Dpgtable.h150 static inline void pud_set(pud_t *pudp, unsigned long val) in pud_set() argument
152 *pudp = __pud(val); in pud_set()
155 static inline void pud_clear(pud_t *pudp) in pud_clear() argument
157 *pudp = __pud(0); in pud_clear()
193 #define pmd_offset(pudp,addr) \ argument
194 (((pmd_t *) pud_page_vaddr(*(pudp))) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)))
/Linux-v5.4/mm/
Dpgtable-generic.c138 pud_t *pudp) in pudp_huge_clear_flush() argument
143 VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp)); in pudp_huge_clear_flush()
144 pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp); in pudp_huge_clear_flush()
/Linux-v5.4/virt/kvm/arm/
Dmmu.c113 static void stage2_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) in stage2_dissolve_pud() argument
115 if (!stage2_pud_huge(kvm, *pudp)) in stage2_dissolve_pud()
118 stage2_pud_clear(kvm, pudp); in stage2_dissolve_pud()
120 put_page(virt_to_page(pudp)); in stage2_dissolve_pud()
201 static inline void kvm_pud_populate(pud_t *pudp, pmd_t *pmdp) in kvm_pud_populate() argument
203 WRITE_ONCE(*pudp, kvm_mk_pud(pmdp)); in kvm_pud_populate()
207 static inline void kvm_pgd_populate(pgd_t *pgdp, pud_t *pudp) in kvm_pgd_populate() argument
209 WRITE_ONCE(*pgdp, kvm_mk_pgd(pudp)); in kvm_pgd_populate()
1117 pud_t *pudp, old_pud; in stage2_set_pud_huge() local
1120 pudp = stage2_get_pud(kvm, cache, addr); in stage2_set_pud_huge()
[all …]
/Linux-v5.4/arch/x86/mm/
Dpgtable.c181 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) in pud_populate() argument
187 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); in pud_populate()
514 pud_t *pudp, pud_t entry, int dirty) in pudp_set_access_flags() argument
516 int changed = !pud_same(*pudp, entry); in pudp_set_access_flags()
521 set_pud(pudp, entry); in pudp_set_access_flags()
559 unsigned long addr, pud_t *pudp) in pudp_test_and_clear_young() argument
563 if (pud_young(*pudp)) in pudp_test_and_clear_young()
565 (unsigned long *)pudp); in pudp_test_and_clear_young()

123