| /Linux-v5.4/include/asm-generic/ |
| D | pgtable-nop4d.h | 9 typedef struct { pgd_t pgd; } p4d_t; member
22 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
23 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
24 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
25 static inline void pgd_clear(pgd_t *pgd) { } in pgd_clear() argument
26 #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
28 #define pgd_populate(mm, pgd, p4d) do { } while (0) argument
29 #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) argument
36 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) in p4d_offset() argument
38 return (p4d_t *)pgd; in p4d_offset()
[all …]
|
| D | pgtable-nop4d-hack.h | 15 typedef struct { pgd_t pgd; } pud_t; member
27 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
28 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
29 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
30 static inline void pgd_clear(pgd_t *pgd) { } in pgd_clear() argument
31 #define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
33 #define pgd_populate(mm, pgd, pud) do { } while (0) argument
34 #define pgd_populate_safe(mm, pgd, pud) do { } while (0) argument
41 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) in pud_offset() argument
43 return (pud_t *)pgd; in pud_offset()
[all …]
|
| /Linux-v5.4/arch/arm/mm/ |
| D | pgd.c | 21 #define __pgd_free(pgd) kfree(pgd) argument
24 #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) argument
118 pgd_t *pgd; in pgd_free() local
126 pgd = pgd_base + pgd_index(0); in pgd_free()
127 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
130 pud = pud_offset(pgd, 0); in pgd_free()
147 pgd_clear(pgd); in pgd_free()
154 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { in pgd_free()
155 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
157 if (pgd_val(*pgd) & L_PGD_SWAPPER) in pgd_free()
[all …]
|
| /Linux-v5.4/arch/nds32/mm/ |
| D | mm-nds32.c | 41 void pgd_free(struct mm_struct *mm, pgd_t * pgd) in pgd_free() argument
46 if (!pgd) in pgd_free()
49 pmd = (pmd_t *) pgd; in pgd_free()
60 dec_zone_page_state(virt_to_page((unsigned long *)pgd), NR_PAGETABLE); in pgd_free()
65 free_pages((unsigned long)pgd, 0); in pgd_free()
76 pgd_t *pgd; in setup_mm_for_reboot() local
80 if (current->mm && current->mm->pgd) in setup_mm_for_reboot()
81 pgd = current->mm->pgd; in setup_mm_for_reboot()
83 pgd = init_mm.pgd; in setup_mm_for_reboot()
87 pmd = pmd_offset(pgd + i, i << PGDIR_SHIFT); in setup_mm_for_reboot()
|
| /Linux-v5.4/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable-4k.h | 56 #define pgd_none(pgd) (!pgd_val(pgd)) argument
57 #define pgd_bad(pgd) (pgd_val(pgd) == 0) argument
58 #define pgd_present(pgd) (pgd_val(pgd) != 0) argument
59 #define pgd_page_vaddr(pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) argument
68 static inline pte_t pgd_pte(pgd_t pgd) in pgd_pte() argument
70 return __pte(pgd_val(pgd)); in pgd_pte()
77 extern struct page *pgd_page(pgd_t pgd);
|
| /Linux-v5.4/arch/powerpc/include/asm/book3s/64/ |
| D | pgalloc.h | 41 static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) in radix__pgd_free() argument
44 free_page((unsigned long)pgd); in radix__pgd_free()
46 free_pages((unsigned long)pgd, 4); in radix__pgd_free()
52 pgd_t *pgd; in pgd_alloc() local
57 pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc()
59 if (unlikely(!pgd)) in pgd_alloc()
60 return pgd; in pgd_alloc()
67 kmemleak_no_scan(pgd); in pgd_alloc()
78 memset(pgd, 0, PGD_TABLE_SIZE); in pgd_alloc()
80 return pgd; in pgd_alloc()
[all …]
|
| /Linux-v5.4/arch/x86/mm/ |
| D | pgtable.c | 86 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() argument
88 struct page *page = virt_to_page(pgd); in pgd_list_add()
93 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() argument
95 struct page *page = virt_to_page(pgd); in pgd_list_del()
106 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) in pgd_set_mm() argument
108 virt_to_page(pgd)->pt_mm = mm; in pgd_set_mm()
116 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) in pgd_ctor() argument
124 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, in pgd_ctor()
131 pgd_set_mm(pgd, mm); in pgd_ctor()
132 pgd_list_add(pgd); in pgd_ctor()
[all …]
|
| D | kasan_init_64.c | 126 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, in kasan_populate_pgd() argument
133 if (pgd_none(*pgd)) { in kasan_populate_pgd()
135 pgd_populate(&init_mm, pgd, p); in kasan_populate_pgd()
138 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
148 pgd_t *pgd; in kasan_populate_shadow() local
153 pgd = pgd_offset_k(addr); in kasan_populate_shadow()
156 kasan_populate_pgd(pgd, addr, next, nid); in kasan_populate_shadow()
157 } while (pgd++, addr = next, addr != end); in kasan_populate_shadow()
174 pgd_t *pgd; in clear_pgds() local
179 pgd = pgd_offset_k(start); in clear_pgds()
[all …]
|
| D | pti.c | 125 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in __pti_set_user_pgtbl() argument
137 return pgd; in __pti_set_user_pgtbl()
143 kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; in __pti_set_user_pgtbl()
158 if ((pgd.pgd & (_PAGE_USER|_PAGE_PRESENT)) == (_PAGE_USER|_PAGE_PRESENT) && in __pti_set_user_pgtbl()
160 pgd.pgd |= _PAGE_NX; in __pti_set_user_pgtbl()
163 return pgd; in __pti_set_user_pgtbl()
174 pgd_t *pgd = kernel_to_user_pgdp(pgd_offset_k(address)); in pti_user_pagetable_walk_p4d() local
182 if (pgd_none(*pgd)) { in pti_user_pagetable_walk_p4d()
187 set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); in pti_user_pagetable_walk_p4d()
189 BUILD_BUG_ON(pgd_large(*pgd) != 0); in pti_user_pagetable_walk_p4d()
[all …]
|
| /Linux-v5.4/arch/parisc/mm/ |
| D | hugetlbpage.c | 51 pgd_t *pgd; in huge_pte_alloc() local
63 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
64 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
76 pgd_t *pgd; in huge_pte_offset() local
83 pgd = pgd_offset(mm, addr); in huge_pte_offset()
84 if (!pgd_none(*pgd)) { in huge_pte_offset()
85 pud = pud_offset(pgd, addr); in huge_pte_offset()
142 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags); in set_huge_pte_at()
144 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags); in set_huge_pte_at()
154 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags); in huge_ptep_get_and_clear()
[all …]
|
| /Linux-v5.4/arch/parisc/include/asm/ |
| D | pgalloc.h | 26 pgd_t *pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, in pgd_alloc() local
28 pgd_t *actual_pgd = pgd; in pgd_alloc()
30 if (likely(pgd != NULL)) { in pgd_alloc()
31 memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); in pgd_alloc()
40 + (__u32)(__pa((unsigned long)pgd) >> PxD_VALUE_SHIFT)); in pgd_alloc()
43 __pgd_val_set(*pgd, PxD_FLAG_ATTACHED); in pgd_alloc()
50 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
53 pgd -= PTRS_PER_PGD; in pgd_free()
55 free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); in pgd_free()
62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) in pgd_populate() argument
[all …]
|
| D | pgtable.h | 93 spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);\
97 spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);\
346 #define pgd_page_vaddr(pgd) ((unsigned long) __va(pgd_address(pgd))) argument
347 #define pgd_page(pgd) virt_to_page((void *)pgd_page_vaddr(pgd)) argument
354 static inline void pgd_clear(pgd_t *pgd) { in pgd_clear() argument
356 if(pgd_flag(*pgd) & PxD_FLAG_ATTACHED) in pgd_clear()
361 __pgd_val_set(*pgd, 0); in pgd_clear()
369 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument
370 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument
371 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument
[all …]
|
| /Linux-v5.4/arch/um/kernel/ |
| D | mem.c | 98 pgd_t *pgd; in fixrange_init() local
107 pgd = pgd_base + i; in fixrange_init()
109 for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { in fixrange_init()
110 pud = pud_offset(pgd, vaddr); in fixrange_init()
126 pgd_t *pgd; in fixaddr_user_init() local
146 pgd = swapper_pg_dir + pgd_index(vaddr); in fixaddr_user_init()
147 pud = pud_offset(pgd, vaddr); in fixaddr_user_init()
196 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL); in pgd_alloc() local
198 if (pgd) { in pgd_alloc()
199 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_alloc()
[all …]
|
| /Linux-v5.4/arch/arm/include/asm/ |
| D | stage2_pgtable.h | 19 #define stage2_pgd_none(kvm, pgd) pgd_none(pgd) argument
20 #define stage2_pgd_clear(kvm, pgd) pgd_clear(pgd) argument
21 #define stage2_pgd_present(kvm, pgd) pgd_present(pgd) argument
22 #define stage2_pgd_populate(kvm, pgd, pud) pgd_populate(NULL, pgd, pud) argument
23 #define stage2_pud_offset(kvm, pgd, address) pud_offset(pgd, address) argument
|
| /Linux-v5.4/arch/hexagon/include/asm/ |
| D | pgalloc.h | 23 pgd_t *pgd; in pgd_alloc() local
25 pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); in pgd_alloc()
35 memcpy(pgd, swapper_pg_dir, PTRS_PER_PGD*sizeof(pgd_t)); in pgd_alloc()
39 mm->context.ptbase = __pa(pgd); in pgd_alloc()
41 return pgd; in pgd_alloc()
44 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
46 free_page((unsigned long) pgd); in pgd_free()
88 pmdindex = (pgd_t *)pmd - mm->pgd; in pmd_populate_kernel()
89 ppmd = (pmd_t *)current->active_mm->pgd + pmdindex; in pmd_populate_kernel()
|
| /Linux-v5.4/arch/sh/mm/ |
| D | hugetlbpage.c | 28 pgd_t *pgd; in huge_pte_alloc() local
33 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
34 if (pgd) { in huge_pte_alloc()
35 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
49 pgd_t *pgd; in huge_pte_offset() local
54 pgd = pgd_offset(mm, addr); in huge_pte_offset()
55 if (pgd) { in huge_pte_offset()
56 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| D | fault.c | 39 pgd_t *pgd; in show_pte() local
42 pgd = mm->pgd; in show_pte()
44 pgd = get_TTB(); in show_pte()
46 if (unlikely(!pgd)) in show_pte()
47 pgd = swapper_pg_dir; in show_pte()
50 printk(KERN_ALERT "pgd = %p\n", pgd); in show_pte()
51 pgd += pgd_index(addr); in show_pte()
53 (u32)(sizeof(*pgd) * 2), (u64)pgd_val(*pgd)); in show_pte()
60 if (pgd_none(*pgd)) in show_pte()
63 if (pgd_bad(*pgd)) { in show_pte()
[all …]
|
| /Linux-v5.4/arch/riscv/include/asm/ |
| D | pgalloc.h | 44 pgd_t *pgd; in pgd_alloc() local
46 pgd = (pgd_t *)__get_free_page(GFP_KERNEL); in pgd_alloc()
47 if (likely(pgd != NULL)) { in pgd_alloc()
48 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_alloc()
50 memcpy(pgd + USER_PTRS_PER_PGD, in pgd_alloc()
51 init_mm.pgd + USER_PTRS_PER_PGD, in pgd_alloc()
54 return pgd; in pgd_alloc()
57 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
59 free_page((unsigned long)pgd); in pgd_free()
|
| /Linux-v5.4/arch/x86/power/ |
| D | hibernate_32.c | 30 static pmd_t *resume_one_md_table_init(pgd_t *pgd) in resume_one_md_table_init() argument
41 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); in resume_one_md_table_init()
42 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
47 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
84 pgd_t *pgd; in resume_physical_mapping_init() local
90 pgd = pgd_base + pgd_idx; in resume_physical_mapping_init()
93 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { in resume_physical_mapping_init()
94 pmd = resume_one_md_table_init(pgd); in resume_physical_mapping_init()
147 pgd_t *pgd; in set_up_temporary_text_mapping() local
151 pgd = pgd_base + pgd_index(restore_jump_address); in set_up_temporary_text_mapping()
[all …]
|
| D | hibernate_64.c | 28 static int set_up_temporary_text_mapping(pgd_t *pgd) in set_up_temporary_text_mapping() argument
77 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); in set_up_temporary_text_mapping()
81 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); in set_up_temporary_text_mapping()
100 pgd_t *pgd; in set_up_temporary_mappings() local
104 pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); in set_up_temporary_mappings()
105 if (!pgd) in set_up_temporary_mappings()
109 result = set_up_temporary_text_mapping(pgd); in set_up_temporary_mappings()
118 result = kernel_ident_mapping_init(&info, pgd, mstart, mend); in set_up_temporary_mappings()
123 temp_pgt = __pa(pgd); in set_up_temporary_mappings()
|
| /Linux-v5.4/mm/kasan/ |
| D | init.c | 38 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
40 return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); in kasan_p4d_table()
43 static inline bool kasan_p4d_table(pgd_t pgd) in kasan_p4d_table() argument
183 static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, in zero_p4d_populate() argument
186 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate()
235 pgd_t *pgd = pgd_offset_k(addr); in kasan_populate_early_shadow() local
264 pgd_populate(&init_mm, pgd, in kasan_populate_early_shadow()
267 p4d = p4d_offset(pgd, addr); in kasan_populate_early_shadow()
279 if (pgd_none(*pgd)) { in kasan_populate_early_shadow()
283 p = p4d_alloc(&init_mm, pgd, addr); in kasan_populate_early_shadow()
[all …]
|
| /Linux-v5.4/arch/mips/mm/ |
| D | hugetlbpage.c | 27 pgd_t *pgd; in huge_pte_alloc() local
31 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
32 pud = pud_alloc(mm, pgd, addr); in huge_pte_alloc()
42 pgd_t *pgd; in huge_pte_offset() local
46 pgd = pgd_offset(mm, addr); in huge_pte_offset()
47 if (pgd_present(*pgd)) { in huge_pte_offset()
48 pud = pud_offset(pgd, addr); in huge_pte_offset()
|
| /Linux-v5.4/virt/kvm/arm/ |
| D | mmu.c | 155 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr) in clear_stage2_pgd_entry() argument
157 pud_t *pud_table __maybe_unused = stage2_pud_offset(kvm, pgd, 0UL); in clear_stage2_pgd_entry()
158 stage2_pgd_clear(kvm, pgd); in clear_stage2_pgd_entry()
161 put_page(virt_to_page(pgd)); in clear_stage2_pgd_entry()
292 static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd, in unmap_stage2_puds() argument
298 start_pud = pud = stage2_pud_offset(kvm, pgd, addr); in unmap_stage2_puds()
316 clear_stage2_pgd_entry(kvm, pgd, start_addr); in unmap_stage2_puds()
332 pgd_t *pgd; in unmap_stage2_range() local
339 pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr); in unmap_stage2_range()
346 if (!READ_ONCE(kvm->arch.pgd)) in unmap_stage2_range()
[all …]
|
| /Linux-v5.4/arch/x86/platform/efi/ |
| D | efi_64.c | 82 int pgd; in efi_call_phys_prolog() local
87 return efi_mm.pgd; in efi_call_phys_prolog()
105 for (pgd = 0; pgd < n_pgds; pgd++) { in efi_call_phys_prolog()
106 addr_pgd = (unsigned long)(pgd * PGDIR_SIZE); in efi_call_phys_prolog()
107 vaddr = (unsigned long)__va(pgd * PGDIR_SIZE); in efi_call_phys_prolog()
109 save_pgd[pgd] = *pgd_efi; in efi_call_phys_prolog()
140 pgd_offset_k(pgd * PGDIR_SIZE)->pgd &= ~_PAGE_NX; in efi_call_phys_prolog()
157 pgd_t *pgd; in efi_call_phys_epilog() local
169 pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE); in efi_call_phys_epilog()
172 if (!pgd_present(*pgd)) in efi_call_phys_epilog()
[all …]
|
| /Linux-v5.4/arch/x86/kernel/ |
| D | machine_kexec_32.c | 57 free_pages((unsigned long)image->arch.pgd, PGD_ALLOCATION_ORDER); in machine_kexec_free_page_tables()
58 image->arch.pgd = NULL; in machine_kexec_free_page_tables()
73 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in machine_kexec_alloc_page_tables()
81 if (!image->arch.pgd || in machine_kexec_alloc_page_tables()
92 pgd_t *pgd, pmd_t *pmd, pte_t *pte, in machine_kexec_page_table_set_one() argument
98 pgd += pgd_index(vaddr); in machine_kexec_page_table_set_one()
100 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) in machine_kexec_page_table_set_one()
101 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); in machine_kexec_page_table_set_one()
103 p4d = p4d_offset(pgd, vaddr); in machine_kexec_page_table_set_one()
122 image->arch.pgd, pmd, image->arch.pte0, in machine_kexec_prepare_page_tables()
[all …]
|
