/Linux-v5.4/arch/arm64/mm/ |
D | mmu.c | 63 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) in set_swapper_pgd() argument 68 fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); in set_swapper_pgd() 285 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, in alloc_init_pud() argument 292 pgd_t pgd = READ_ONCE(*pgdp); in alloc_init_pud() 298 __pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE); in alloc_init_pud() 299 pgd = READ_ONCE(*pgdp); in alloc_init_pud() 303 pudp = pud_set_fixmap_offset(pgdp, addr); in alloc_init_pud() 342 pgd_t *pgdp = pgd_offset_raw(pgdir, virt); in __create_pgd_mapping() local 358 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, in __create_pgd_mapping() 361 } while (pgdp++, addr = next, addr != end); in __create_pgd_mapping() [all …]
|
D | kasan_init.c | 87 static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node, in kasan_pud_offset() argument 90 if (pgd_none(READ_ONCE(*pgdp))) { in kasan_pud_offset() 94 __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE); in kasan_pud_offset() 97 return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr); in kasan_pud_offset() 129 static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr, in kasan_pud_populate() argument 133 pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early); in kasan_pud_populate() 145 pgd_t *pgdp; in kasan_pgd_populate() local 147 pgdp = pgd_offset_k(addr); in kasan_pgd_populate() 150 kasan_pud_populate(pgdp, addr, next, node, early); in kasan_pgd_populate() 151 } while (pgdp++, addr = next, addr != end); in kasan_pgd_populate() [all …]
|
D | hugetlbpage.c | 69 pgd_t *pgdp = pgd_offset(mm, addr); in find_num_contig() local 74 pudp = pud_offset(pgdp, addr); in find_num_contig() 219 pgd_t *pgdp; in huge_pte_alloc() local 224 pgdp = pgd_offset(mm, addr); in huge_pte_alloc() 225 pudp = pud_alloc(mm, pgdp, addr); in huge_pte_alloc() 261 pgd_t *pgdp; in huge_pte_offset() local 265 pgdp = pgd_offset(mm, addr); in huge_pte_offset() 266 if (!pgd_present(READ_ONCE(*pgdp))) in huge_pte_offset() 269 pudp = pud_offset(pgdp, addr); in huge_pte_offset()
|
D | pageattr.c | 200 pgd_t *pgdp; in kernel_page_present() local 209 pgdp = pgd_offset_k(addr); in kernel_page_present() 210 if (pgd_none(READ_ONCE(*pgdp))) in kernel_page_present() 213 pudp = pud_offset(pgdp, addr); in kernel_page_present()
|
D | dump.c | 325 static void walk_pud(struct pg_state *st, pgd_t *pgdp, unsigned long start, in walk_pud() argument 329 pud_t *pudp = pud_offset(pgdp, start); in walk_pud() 349 pgd_t *pgdp = pgd_offset(mm, start); in walk_pgd() local 352 pgd_t pgd = READ_ONCE(*pgdp); in walk_pgd() 359 walk_pud(st, pgdp, addr, next); in walk_pgd() 361 } while (pgdp++, addr = next, addr != end); in walk_pgd()
|
/Linux-v5.4/arch/powerpc/mm/nohash/ |
D | book3e_pgtable.c | 75 pgd_t *pgdp; in map_kernel_page() local 82 pgdp = pgd_offset_k(ea); in map_kernel_page() 83 pudp = pud_alloc(&init_mm, pgdp, ea); in map_kernel_page() 93 pgdp = pgd_offset_k(ea); in map_kernel_page() 95 if (pgd_none(*pgdp)) { in map_kernel_page() 97 pgd_populate(&init_mm, pgdp, pudp); in map_kernel_page() 100 pudp = pud_offset(pgdp, ea); in map_kernel_page()
|
/Linux-v5.4/arch/arm64/include/asm/ |
D | pgalloc.h | 76 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) in __pgd_populate() argument 78 set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot)); in __pgd_populate() 81 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp) in pgd_populate() argument 83 __pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE); in pgd_populate() 86 static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot) in __pgd_populate() argument 93 extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
|
D | mmu_context.h | 138 static inline void cpu_replace_ttbr1(pgd_t *pgdp) in cpu_replace_ttbr1() argument 145 phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); in cpu_replace_ttbr1() 147 if (system_supports_cnp() && !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) { in cpu_replace_ttbr1()
|
D | pgtable.h | 465 extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); 597 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) in set_pgd() argument 599 if (in_swapper_pgdir(pgdp)) { in set_pgd() 600 set_swapper_pgd(pgdp, pgd); in set_pgd() 604 WRITE_ONCE(*pgdp, pgd); in set_pgd() 609 static inline void pgd_clear(pgd_t *pgdp) in pgd_clear() argument 611 set_pgd(pgdp, __pgd(0)); in pgd_clear() 640 #define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) argument
|
/Linux-v5.4/arch/powerpc/include/asm/nohash/64/ |
D | pgtable-4k.h | 63 static inline void pgd_clear(pgd_t *pgdp) in pgd_clear() argument 65 *pgdp = __pgd(0); in pgd_clear() 81 #define pud_offset(pgdp, addr) \ argument 82 (((pud_t *) pgd_page_vaddr(*(pgdp))) + \
|
/Linux-v5.4/arch/s390/mm/ |
D | hugetlbpage.c | 189 pgd_t *pgdp; in huge_pte_alloc() local 194 pgdp = pgd_offset(mm, addr); in huge_pte_alloc() 195 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc() 211 pgd_t *pgdp; in huge_pte_offset() local 216 pgdp = pgd_offset(mm, addr); in huge_pte_offset() 217 if (pgd_present(*pgdp)) { in huge_pte_offset() 218 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
|
D | pageattr.c | 289 pgd_t *pgdp; in change_page_attr() local 296 pgdp = pgd_offset_k(addr); in change_page_attr() 298 if (pgd_none(*pgdp)) in change_page_attr() 301 rc = walk_p4d_level(pgdp, addr, next, flags); in change_page_attr() 305 } while (pgdp++, addr = next, addr < end && !rc); in change_page_attr()
|
/Linux-v5.4/arch/sparc/mm/ |
D | srmmu.c | 110 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) in srmmu_ctxd_set() argument 114 pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4))); in srmmu_ctxd_set() 518 pgd_t *pgdp; in srmmu_mapioaddr() local 524 pgdp = pgd_offset_k(virt_addr); in srmmu_mapioaddr() 525 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_mapioaddr() 553 pgd_t *pgdp; in srmmu_unmapioaddr() local 557 pgdp = pgd_offset_k(virt_addr); in srmmu_unmapioaddr() 558 pmdp = pmd_offset(pgdp, virt_addr); in srmmu_unmapioaddr() 695 pgd_t *pgdp; in srmmu_early_allocate_ptable_skeleton() local 700 pgdp = pgd_offset_k(start); in srmmu_early_allocate_ptable_skeleton() [all …]
|
/Linux-v5.4/arch/arm64/kernel/ |
D | hibernate.c | 205 pgd_t *pgdp; in create_safe_exec_page() local 225 pgdp = pgd_offset_raw(trans_pgd, dst_addr); in create_safe_exec_page() 226 if (pgd_none(READ_ONCE(*pgdp))) { in create_safe_exec_page() 232 pgd_populate(&init_mm, pgdp, pudp); in create_safe_exec_page() 235 pudp = pud_offset(pgdp, dst_addr); in create_safe_exec_page() 272 write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); in create_safe_exec_page()
|
/Linux-v5.4/arch/x86/mm/ |
D | pgtable.c | 255 static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) in mop_up_one_pmd() argument 257 pgd_t pgd = *pgdp; in mop_up_one_pmd() 262 pgd_clear(pgdp); in mop_up_one_pmd() 270 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) in pgd_mop_up_pmds() argument 275 mop_up_one_pmd(mm, &pgdp[i]); in pgd_mop_up_pmds() 282 pgdp = kernel_to_user_pgdp(pgdp); in pgd_mop_up_pmds() 285 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); in pgd_mop_up_pmds()
|
/Linux-v5.4/arch/powerpc/mm/book3s64/ |
D | radix_pgtable.c | 65 pgd_t *pgdp; in early_map_kernel_page() local 70 pgdp = pgd_offset_k(ea); in early_map_kernel_page() 71 if (pgd_none(*pgdp)) { in early_map_kernel_page() 74 pgd_populate(&init_mm, pgdp, pudp); in early_map_kernel_page() 76 pudp = pud_offset(pgdp, ea); in early_map_kernel_page() 115 pgd_t *pgdp; in __map_kernel_page() local 137 pgdp = pgd_offset_k(ea); in __map_kernel_page() 138 pudp = pud_alloc(&init_mm, pgdp, ea); in __map_kernel_page() 174 pgd_t *pgdp; in radix__change_memory_range() local 186 pgdp = pgd_offset_k(idx); in radix__change_memory_range() [all …]
|
D | hash_pgtable.c | 150 pgd_t *pgdp; in hash__map_kernel_page() local 157 pgdp = pgd_offset_k(ea); in hash__map_kernel_page() 158 pudp = pud_alloc(&init_mm, pgdp, ea); in hash__map_kernel_page()
|
/Linux-v5.4/arch/riscv/mm/ |
D | init.c | 288 static void __init create_pgd_mapping(pgd_t *pgdp, in create_pgd_mapping() argument 297 if (pgd_val(pgdp[pgd_index]) == 0) in create_pgd_mapping() 298 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); in create_pgd_mapping() 302 if (pgd_val(pgdp[pgd_index]) == 0) { in create_pgd_mapping() 304 pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); in create_pgd_mapping() 308 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); in create_pgd_mapping()
|
/Linux-v5.4/arch/powerpc/mm/ |
D | pgtable.c | 315 pgd_t pgd, *pgdp; in __find_linux_pte() local 328 pgdp = pgdir + pgd_index(ea); in __find_linux_pte() 329 pgd = READ_ONCE(*pgdp); in __find_linux_pte() 340 ret_pte = (pte_t *)pgdp; in __find_linux_pte()
|
/Linux-v5.4/arch/sparc/include/asm/ |
D | pgalloc_32.h | 29 static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument 33 set_pte((pte_t *)pgdp, __pte((SRMMU_ET_PTD | (pa >> 4)))); in pgd_set()
|
/Linux-v5.4/arch/x86/include/asm/ |
D | pgtable_64.h | 155 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument 157 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
|
D | pgtable.h | 72 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) argument 684 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd); 691 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument 695 return __pti_set_user_pgtbl(pgdp, pgd); in pti_set_user_pgtbl() 698 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument 1266 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) in kernel_to_user_pgdp() argument 1268 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_pgdp() 1271 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) in user_to_kernel_pgdp() argument 1273 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_pgdp()
|
/Linux-v5.4/arch/mips/mm/ |
D | c-r3k.c | 243 pgd_t *pgdp; in r3k_flush_cache_page() local 255 pgdp = pgd_offset(mm, addr); in r3k_flush_cache_page() 256 pudp = pud_offset(pgdp, addr); in r3k_flush_cache_page()
|
/Linux-v5.4/arch/alpha/include/asm/ |
D | pgtable.h | 229 extern inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) in pgd_set() argument 230 { pgd_val(*pgdp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } in pgd_set() 262 extern inline void pgd_clear(pgd_t * pgdp) { pgd_val(*pgdp) = 0; } in pgd_clear() argument
|
/Linux-v5.4/arch/m68k/include/asm/ |
D | motorola_pgtable.h | 120 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument 122 pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pgd_set() 153 #define pgd_clear(pgdp) ({ pgd_val(*pgdp) = 0; }) argument
|