| /Linux-v4.19/arch/arm64/mm/ |
| D | mmu.c | 274 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, in alloc_init_pud() argument
281 pgd_t pgd = READ_ONCE(*pgdp); in alloc_init_pud()
287 __pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE); in alloc_init_pud()
288 pgd = READ_ONCE(*pgdp); in alloc_init_pud()
292 pudp = pud_set_fixmap_offset(pgdp, addr); in alloc_init_pud()
331 pgd_t *pgdp = pgd_offset_raw(pgdir, virt); in __create_pgd_mapping() local
347 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, in __create_pgd_mapping()
350 } while (pgdp++, addr = next, addr != end); in __create_pgd_mapping()
412 static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, in __map_memblock() argument
415 __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, in __map_memblock()
[all …]
|
| D | kasan_init.c | 72 static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node, in kasan_pud_offset() argument
75 if (pgd_none(READ_ONCE(*pgdp))) { in kasan_pud_offset()
78 __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE); in kasan_pud_offset()
81 return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr); in kasan_pud_offset()
110 static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr, in kasan_pud_populate() argument
114 pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early); in kasan_pud_populate()
126 pgd_t *pgdp; in kasan_pgd_populate() local
128 pgdp = pgd_offset_k(addr); in kasan_pgd_populate()
131 kasan_pud_populate(pgdp, addr, next, node, early); in kasan_pgd_populate()
132 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate()
[all …]
|
| D | hugetlbpage.c | 57 pgd_t *pgdp = pgd_offset(mm, addr); in find_num_contig() local
62 pudp = pud_offset(pgdp, addr); in find_num_contig()
207 pgd_t *pgdp; in huge_pte_alloc() local
212 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
213 pudp = pud_alloc(mm, pgdp, addr); in huge_pte_alloc()
249 pgd_t *pgdp; in huge_pte_offset() local
253 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
254 if (!pgd_present(READ_ONCE(*pgdp))) in huge_pte_offset()
257 pudp = pud_offset(pgdp, addr); in huge_pte_offset()
|
| D | pageattr.c | 159 pgd_t *pgdp; in kernel_page_present() local
165 pgdp = pgd_offset_k(addr); in kernel_page_present()
166 if (pgd_none(READ_ONCE(*pgdp))) in kernel_page_present()
169 pudp = pud_offset(pgdp, addr); in kernel_page_present()
|
| D | dump.c | 320 static void walk_pud(struct pg_state *st, pgd_t *pgdp, unsigned long start) in walk_pud() argument
322 pud_t *pudp = pud_offset(pgdp, 0UL); in walk_pud()
342 pgd_t *pgdp = pgd_offset(mm, 0UL); in walk_pgd() local
346 for (i = 0; i < PTRS_PER_PGD; i++, pgdp++) { in walk_pgd()
347 pgd_t pgd = READ_ONCE(*pgdp); in walk_pgd()
354 walk_pud(st, pgdp, addr); in walk_pgd()
|
| /Linux-v4.19/arch/powerpc/mm/ |
| D | pgtable-book3e.c | 75 pgd_t *pgdp; in map_kernel_page() local
82 pgdp = pgd_offset_k(ea); in map_kernel_page()
83 pudp = pud_alloc(&init_mm, pgdp, ea); in map_kernel_page()
95 pgdp = pgd_offset_k(ea); in map_kernel_page()
97 if (pgd_none(*pgdp)) { in map_kernel_page()
100 pgd_populate(&init_mm, pgdp, pudp); in map_kernel_page()
103 pudp = pud_offset(pgdp, ea); in map_kernel_page()
|
| D | pgtable-radix.c | 83 pgd_t *pgdp; in early_map_kernel_page() local
88 pgdp = pgd_offset_k(ea); in early_map_kernel_page()
89 if (pgd_none(*pgdp)) { in early_map_kernel_page()
92 pgd_populate(&init_mm, pgdp, pudp); in early_map_kernel_page()
94 pudp = pud_offset(pgdp, ea); in early_map_kernel_page()
133 pgd_t *pgdp; in __map_kernel_page() local
151 pgdp = pgd_offset_k(ea); in __map_kernel_page()
152 pudp = pud_alloc(&init_mm, pgdp, ea); in __map_kernel_page()
188 pgd_t *pgdp; in radix__change_memory_range() local
200 pgdp = pgd_offset_k(idx); in radix__change_memory_range()
[all …]
|
| /Linux-v4.19/arch/sh/mm/ |
| D | gup.c | 173 pgd_t *pgdp; in __get_user_pages_fast() local
189 pgdp = pgd_offset(mm, addr); in __get_user_pages_fast()
191 pgd_t pgd = *pgdp; in __get_user_pages_fast()
198 } while (pgdp++, addr = next, addr != end); in __get_user_pages_fast()
226 pgd_t *pgdp; in get_user_pages_fast() local
238 pgdp = pgd_offset(mm, addr); in get_user_pages_fast()
240 pgd_t pgd = *pgdp; in get_user_pages_fast()
247 } while (pgdp++, addr = next, addr != end); in get_user_pages_fast()
|
| /Linux-v4.19/arch/arm64/include/asm/ |
| D | pgalloc.h | 74 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) in __pgd_populate() argument
76 set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot)); in __pgd_populate()
79 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp) in pgd_populate() argument
81 __pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE); in pgd_populate()
84 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) in __pgd_populate() argument
91 extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
|
| /Linux-v4.19/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable-4k.h | 63 static inline void pgd_clear(pgd_t *pgdp) in pgd_clear() argument
65 *pgdp = __pgd(0); in pgd_clear()
81 #define pud_offset(pgdp, addr) \ argument
82 (((pud_t *) pgd_page_vaddr(*(pgdp))) + \
|
| /Linux-v4.19/arch/sparc/mm/ |
| D | gup.c | 206 pgd_t *pgdp; in __get_user_pages_fast() local
233 pgdp = pgd_offset(mm, addr); in __get_user_pages_fast()
235 pgd_t pgd = *pgdp; in __get_user_pages_fast()
242 } while (pgdp++, addr = next, addr != end); in __get_user_pages_fast()
254 pgd_t *pgdp; in get_user_pages_fast() local
299 pgdp = pgd_offset(mm, addr); in get_user_pages_fast()
301 pgd_t pgd = *pgdp; in get_user_pages_fast()
308 } while (pgdp++, addr = next, addr != end); in get_user_pages_fast()
|
| D | srmmu.c | 110 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) in srmmu_ctxd_set() argument
114 pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4))); in srmmu_ctxd_set()
510 pgd_t *pgdp; in srmmu_mapioaddr() local
516 pgdp = pgd_offset_k(virt_addr); in srmmu_mapioaddr()
517 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_mapioaddr()
545 pgd_t *pgdp; in srmmu_unmapioaddr() local
549 pgdp = pgd_offset_k(virt_addr); in srmmu_unmapioaddr()
550 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
687 pgd_t *pgdp; in srmmu_early_allocate_ptable_skeleton() local
692 pgdp = pgd_offset_k(start); in srmmu_early_allocate_ptable_skeleton()
[all …]
|
| /Linux-v4.19/arch/mips/mm/ |
| D | gup.c | 191 pgd_t *pgdp; in __get_user_pages_fast() local
220 pgdp = pgd_offset(mm, addr); in __get_user_pages_fast()
222 pgd_t pgd = *pgdp; in __get_user_pages_fast()
229 } while (pgdp++, addr = next, addr != end); in __get_user_pages_fast()
257 pgd_t *pgdp; in get_user_pages_fast() local
270 pgdp = pgd_offset(mm, addr); in get_user_pages_fast()
272 pgd_t pgd = *pgdp; in get_user_pages_fast()
279 } while (pgdp++, addr = next, addr != end); in get_user_pages_fast()
|
| D | tlb-r8k.c | 186 pgd_t *pgdp; in __update_tlb() local
203 pgdp = pgd_offset(vma->vm_mm, address); in __update_tlb()
204 pmdp = pmd_offset(pgdp, address); in __update_tlb()
|
| D | c-r3k.c | 243 pgd_t *pgdp; in r3k_flush_cache_page() local
255 pgdp = pgd_offset(mm, addr); in r3k_flush_cache_page()
256 pudp = pud_offset(pgdp, addr); in r3k_flush_cache_page()
|
| /Linux-v4.19/arch/s390/mm/ |
| D | hugetlbpage.c | 189 pgd_t *pgdp; in huge_pte_alloc() local
194 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
195 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
211 pgd_t *pgdp; in huge_pte_offset() local
216 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
217 if (pgd_present(*pgdp)) { in huge_pte_offset()
218 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
|
| D | gup.c | 197 static inline int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, in gup_p4d_range() argument
203 p4dp = (p4d_t *) pgdp; in gup_p4d_range()
232 pgd_t *pgdp, pgd; in __get_user_pages_fast() local
249 pgdp = pgd_offset(mm, addr); in __get_user_pages_fast()
251 pgd = *pgdp; in __get_user_pages_fast()
256 if (!gup_p4d_range(pgdp, pgd, addr, next, write, pages, &nr)) in __get_user_pages_fast()
258 } while (pgdp++, addr = next, addr != end); in __get_user_pages_fast()
|
| D | pageattr.c | 289 pgd_t *pgdp; in change_page_attr() local
296 pgdp = pgd_offset_k(addr); in change_page_attr()
298 if (pgd_none(*pgdp)) in change_page_attr()
301 rc = walk_p4d_level(pgdp, addr, next, flags); in change_page_attr()
305 } while (pgdp++, addr = next, addr < end && !rc); in change_page_attr()
|
| /Linux-v4.19/arch/arm64/kernel/ |
| D | hibernate.c | 205 pgd_t *pgdp; in create_safe_exec_page() local
219 pgdp = pgd_offset_raw(allocator(mask), dst_addr); in create_safe_exec_page()
220 if (pgd_none(READ_ONCE(*pgdp))) { in create_safe_exec_page()
226 pgd_populate(&init_mm, pgdp, pudp); in create_safe_exec_page()
229 pudp = pud_offset(pgdp, dst_addr); in create_safe_exec_page()
266 write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); in create_safe_exec_page()
|
| /Linux-v4.19/arch/x86/mm/ |
| D | pgtable.c | 271 static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) in mop_up_one_pmd() argument
273 pgd_t pgd = *pgdp; in mop_up_one_pmd()
278 pgd_clear(pgdp); in mop_up_one_pmd()
286 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) in pgd_mop_up_pmds() argument
291 mop_up_one_pmd(mm, &pgdp[i]); in pgd_mop_up_pmds()
298 pgdp = kernel_to_user_pgdp(pgdp); in pgd_mop_up_pmds()
301 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); in pgd_mop_up_pmds()
|
| /Linux-v4.19/arch/sparc/include/asm/ |
| D | pgalloc_32.h | 31 static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument
35 set_pte((pte_t *)pgdp, __pte((SRMMU_ET_PTD | (pa >> 4)))); in pgd_set()
|
| /Linux-v4.19/arch/x86/include/asm/ |
| D | pgtable_64.h | 155 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
157 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
|
| D | pgtable.h | 70 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) argument
668 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
675 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
679 return __pti_set_user_pgtbl(pgdp, pgd); in pti_set_user_pgtbl()
682 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
1247 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) in kernel_to_user_pgdp() argument
1249 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_pgdp()
1252 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) in user_to_kernel_pgdp() argument
1254 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_pgdp()
|
| /Linux-v4.19/arch/m68k/include/asm/ |
| D | motorola_pgtable.h | 120 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
122 pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pgd_set()
153 #define pgd_clear(pgdp) ({ pgd_val(*pgdp) = 0; }) argument
|
| /Linux-v4.19/arch/alpha/include/asm/ |
| D | pgtable.h | 229 extern inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument
230 { pgd_val(*pgdp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } in pgd_set()
262 extern inline void pgd_clear(pgd_t * pgdp) { pgd_val(*pgdp) = 0; } in pgd_clear() argument
|
