Home
last modified time | relevance | path

Searched refs:pmdp (Results 1 – 25 of 122) sorted by relevance

12345

/Linux-v4.19/mm/
Dpgtable-generic.c96 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
99 int changed = !pmd_same(*pmdp, entry); in pmdp_set_access_flags()
102 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_set_access_flags()
111 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
115 young = pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
124 pmd_t *pmdp) in pmdp_huge_clear_flush() argument
128 VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_clear_flush()
129 !pmd_devmap(*pmdp)) || !pmd_present(*pmdp)); in pmdp_huge_clear_flush()
130 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); in pmdp_huge_clear_flush()
151 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
[all …]
/Linux-v4.19/arch/powerpc/mm/
Dpgtable-hash64.c149 pmd_t *pmdp; in hash__map_kernel_page() local
158 pmdp = pmd_alloc(&init_mm, pudp, ea); in hash__map_kernel_page()
159 if (!pmdp) in hash__map_kernel_page()
161 ptep = pte_alloc_kernel(pmdp, ea); in hash__map_kernel_page()
188 pmd_t *pmdp, unsigned long clr, in hash__pmd_hugepage_update() argument
195 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in hash__pmd_hugepage_update()
196 assert_spin_locked(pmd_lockptr(mm, pmdp)); in hash__pmd_hugepage_update()
207 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp) in hash__pmd_hugepage_update()
208 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), in hash__pmd_hugepage_update()
216 hpte_do_hugepage_flush(mm, addr, pmdp, old); in hash__pmd_hugepage_update()
[all …]
Dpgtable-book3e.c77 pmd_t *pmdp; in map_kernel_page() local
86 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
87 if (!pmdp) in map_kernel_page()
89 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
105 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page()
106 BUG_ON(pmdp == NULL); in map_kernel_page()
107 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
109 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
110 if (!pmd_present(*pmdp)) { in map_kernel_page()
113 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
[all …]
D40x_mmu.c109 pmd_t *pmdp; in mmu_mapin_ram() local
112 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
113 *pmdp++ = __pmd(val); in mmu_mapin_ram()
114 *pmdp++ = __pmd(val); in mmu_mapin_ram()
115 *pmdp++ = __pmd(val); in mmu_mapin_ram()
116 *pmdp++ = __pmd(val); in mmu_mapin_ram()
124 pmd_t *pmdp; in mmu_mapin_ram() local
127 pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v); in mmu_mapin_ram()
128 *pmdp = __pmd(val); in mmu_mapin_ram()
Dpgtable-radix.c85 pmd_t *pmdp; in early_map_kernel_page() local
100 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE, nid, in early_map_kernel_page()
102 pud_populate(&init_mm, pudp, pmdp); in early_map_kernel_page()
104 pmdp = pmd_offset(pudp, ea); in early_map_kernel_page()
106 ptep = pmdp_ptep(pmdp); in early_map_kernel_page()
109 if (!pmd_present(*pmdp)) { in early_map_kernel_page()
112 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_map_kernel_page()
114 ptep = pte_offset_kernel(pmdp, ea); in early_map_kernel_page()
135 pmd_t *pmdp; in __map_kernel_page() local
159 pmdp = pmd_alloc(&init_mm, pudp, ea); in __map_kernel_page()
[all …]
Dpgtable-book3s64.c40 pmd_t *pmdp, pmd_t entry, int dirty) in pmdp_set_access_flags() argument
44 WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in pmdp_set_access_flags()
45 assert_spin_locked(pmd_lockptr(vma->vm_mm, pmdp)); in pmdp_set_access_flags()
47 changed = !pmd_same(*(pmdp), entry); in pmdp_set_access_flags()
53 __ptep_set_access_flags(vma, pmdp_ptep(pmdp), in pmdp_set_access_flags()
60 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument
62 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp); in pmdp_test_and_clear_young()
69 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
72 WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp))); in set_pmd_at()
73 assert_spin_locked(pmd_lockptr(mm, pmdp)); in set_pmd_at()
[all …]
Dhugepage-hash64.c22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument
36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp()
52 } while (!pmd_xchg(pmdp, __pmd(old_pmd), __pmd(new_pmd))); in __hash_page_thp()
74 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp()
82 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp()
159 *pmdp = __pmd(old_pmd); in __hash_page_thp()
183 *pmdp = __pmd(new_pmd & ~H_PAGE_BUSY); in __hash_page_thp()
/Linux-v4.19/arch/arm64/include/asm/
Dpgalloc.h39 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp) in pmd_free() argument
41 BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1)); in pmd_free()
42 free_page((unsigned long)pmdp); in pmd_free()
45 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
47 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate()
50 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument
52 __pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE); in pud_populate()
55 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
129 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, in __pmd_populate() argument
132 set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); in __pmd_populate()
[all …]
/Linux-v4.19/arch/sparc/mm/
Dtlb.c220 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
222 pmd_t orig = *pmdp; in set_pmd_at()
224 *pmdp = pmd; in set_pmd_at()
229 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument
234 old = *pmdp; in pmdp_establish()
235 } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd); in pmdp_establish()
245 pmd_t *pmdp) in pmdp_invalidate() argument
249 entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); in pmdp_invalidate()
250 old = pmdp_establish(vma, address, pmdp, entry); in pmdp_invalidate()
265 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
[all …]
/Linux-v4.19/arch/s390/mm/
Dpgtable.c343 unsigned long addr, pmd_t *pmdp) in pmdp_idte_local() argument
346 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, in pmdp_idte_local()
349 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); in pmdp_idte_local()
355 unsigned long addr, pmd_t *pmdp) in pmdp_idte_global() argument
358 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, in pmdp_idte_global()
363 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); in pmdp_idte_global()
367 __pmdp_csp(pmdp); in pmdp_idte_global()
374 unsigned long addr, pmd_t *pmdp) in pmdp_flush_direct() argument
378 old = *pmdp; in pmdp_flush_direct()
384 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
[all …]
Dpageattr.c84 static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, in walk_pte_level() argument
89 ptep = pte_offset(pmdp, addr); in walk_pte_level()
110 static int split_pmd_page(pmd_t *pmdp, unsigned long addr) in split_pmd_page() argument
120 pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; in split_pmd_page()
121 ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); in split_pmd_page()
122 nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); in split_pmd_page()
133 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in split_pmd_page()
139 static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, in modify_pmd_page() argument
142 pmd_t new = *pmdp; in modify_pmd_page()
152 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in modify_pmd_page()
[all …]
Dgmap.c907 pmd_t *pmdp; in gmap_pmd_op_walk() local
911 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); in gmap_pmd_op_walk()
913 if (!pmdp || pmd_none(*pmdp)) { in gmap_pmd_op_walk()
919 if (!pmd_large(*pmdp)) in gmap_pmd_op_walk()
921 return pmdp; in gmap_pmd_op_walk()
929 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) in gmap_pmd_op_end() argument
931 if (pmd_large(*pmdp)) in gmap_pmd_op_end()
950 pmd_t *pmdp, int prot, unsigned long bits) in gmap_protect_pmd() argument
952 int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID; in gmap_protect_pmd()
953 int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT; in gmap_protect_pmd()
[all …]
/Linux-v4.19/arch/arm64/mm/
Dmmu.c129 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, in init_pte() argument
134 ptep = pte_set_fixmap_offset(pmdp, addr); in init_pte()
153 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, in alloc_init_cont_pte() argument
160 pmd_t pmd = READ_ONCE(*pmdp); in alloc_init_cont_pte()
167 __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); in alloc_init_cont_pte()
168 pmd = READ_ONCE(*pmdp); in alloc_init_cont_pte()
182 init_pte(pmdp, addr, next, phys, __prot); in alloc_init_cont_pte()
193 pmd_t *pmdp; in init_pmd() local
195 pmdp = pmd_set_fixmap_offset(pudp, addr); in init_pmd()
197 pmd_t old_pmd = READ_ONCE(*pmdp); in init_pmd()
[all …]
Dkasan_init.c47 static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, in kasan_pte_offset() argument
50 if (pmd_none(READ_ONCE(*pmdp))) { in kasan_pte_offset()
53 __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); in kasan_pte_offset()
56 return early ? pte_offset_kimg(pmdp, addr) in kasan_pte_offset()
57 : pte_offset_kernel(pmdp, addr); in kasan_pte_offset()
84 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument
88 pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); in kasan_pte_populate()
102 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() local
106 kasan_pte_populate(pmdp, addr, next, node, early); in kasan_pmd_populate()
107 } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); in kasan_pmd_populate()
/Linux-v4.19/include/asm-generic/
Dpgtable.h38 unsigned long address, pmd_t *pmdp,
45 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
80 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
82 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young()
87 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); in pmdp_test_and_clear_young()
93 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
109 unsigned long address, pmd_t *pmdp);
116 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
139 pmd_t *pmdp) in pmdp_huge_get_and_clear() argument
141 pmd_t pmd = *pmdp; in pmdp_huge_get_and_clear()
[all …]
/Linux-v4.19/arch/m68k/include/asm/
Dmotorola_pgtable.h109 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) in pmd_set() argument
112 unsigned long *ptr = pmdp->pmd; in pmd_set()
120 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
122 pgd_val(*pgdp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pgd_set()
141 #define pmd_clear(pmdp) ({ \ argument
142 unsigned long *__ptr = pmdp->pmd; \
218 static inline pte_t *pte_offset_kernel(pmd_t *pmdp, unsigned long address) in pte_offset_kernel() argument
220 return (pte_t *)__pmd_page(*pmdp) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); in pte_offset_kernel()
223 #define pte_offset_map(pmdp,address) ((pte_t *)__pmd_page(*pmdp) + (((address) >> PAGE_SHIFT) & (PT… argument
242 pmd_t *pmdp; in nocache_page() local
[all …]
Dmcf_pgtable.h165 #define pmd_set(pmdp, ptep) do {} while (0) argument
167 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
169 pgd_val(*pgdp) = virt_to_phys(pmdp); in pgd_set()
199 static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; } in pmd_clear() argument
363 pmd_t *pmdp; in nocache_page() local
368 pmdp = pmd_offset(dir, addr); in nocache_page()
369 ptep = pte_offset_kernel(pmdp, addr); in nocache_page()
379 pmd_t *pmdp; in cache_page() local
384 pmdp = pmd_offset(dir, addr); in cache_page()
385 ptep = pte_offset_kernel(pmdp, addr); in cache_page()
[all …]
/Linux-v4.19/arch/unicore32/include/asm/
Dpgalloc.h82 static inline void __pmd_populate(pmd_t *pmdp, unsigned long pmdval) in __pmd_populate() argument
84 set_pmd(pmdp, __pmd(pmdval)); in __pmd_populate()
85 flush_pmd_entry(pmdp); in __pmd_populate()
93 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
101 __pmd_populate(pmdp, __pa(pte_ptr) | _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
105 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
107 __pmd_populate(pmdp, in pmd_populate()
/Linux-v4.19/arch/powerpc/include/asm/book3s/32/
Dpgalloc.h55 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
58 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
61 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
64 *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT); in pmd_populate()
70 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
73 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
76 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
79 *pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT); in pmd_populate()
/Linux-v4.19/arch/powerpc/include/asm/nohash/32/
Dpgalloc.h55 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
58 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
61 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
64 *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_USER | in pmd_populate()
71 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
74 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
77 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
80 *pmdp = __pmd((unsigned long)lowmem_page_address(pte_page) | _PMD_PRESENT); in pmd_populate()
/Linux-v4.19/arch/arm/include/asm/
Dpgalloc.h131 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
135 pmdp[0] = __pmd(pmdval); in __pmd_populate()
137 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate()
139 flush_pmd_entry(pmdp); in __pmd_populate()
149 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
154 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
158 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
168 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
/Linux-v4.19/arch/x86/include/asm/
Dpgtable-3level.h73 static inline pmd_t pmd_read_atomic(pmd_t *pmdp) in pmd_read_atomic() argument
76 u32 *tmp = (u32 *)pmdp; in pmd_read_atomic()
96 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
98 set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd)); in native_set_pmd()
172 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) in native_pmdp_get_and_clear() argument
174 union split_pmd res, *orig = (union split_pmd *)pmdp; in native_pmdp_get_and_clear()
190 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument
202 ptr = (union split_pmd *)pmdp; in pmdp_establish()
214 old = *pmdp; in pmdp_establish()
215 } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd); in pmdp_establish()
/Linux-v4.19/arch/s390/include/asm/
Dpgtable.h862 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
864 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; in pmd_clear()
1357 static inline void __pmdp_csp(pmd_t *pmdp) in __pmdp_csp() argument
1359 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), in __pmdp_csp()
1360 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); in __pmdp_csp()
1370 static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp, in __pmdp_idte() argument
1376 sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t); in __pmdp_idte()
1381 : "+m" (*pmdp) in __pmdp_idte()
1389 : "+m" (*pmdp) in __pmdp_idte()
1430 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
[all …]
/Linux-v4.19/arch/powerpc/include/asm/book3s/64/
Dhash-4k.h93 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
136 unsigned long addr, pmd_t *pmdp,
139 unsigned long address, pmd_t *pmdp);
140 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
142 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
144 unsigned long addr, pmd_t *pmdp);
Dpgtable.h861 static inline void pmd_set(pmd_t *pmdp, unsigned long val) in pmd_set() argument
863 *pmdp = __pmd(val); in pmd_set()
866 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
868 *pmdp = __pmd(0); in pmd_clear()
1109 pmd_t *pmdp, pmd_t pmd);
1122 pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, in pmd_hugepage_update() argument
1126 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set); in pmd_hugepage_update()
1127 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set); in pmd_hugepage_update()
1144 unsigned long addr, pmd_t *pmdp) in __pmdp_test_and_clear_young() argument
1148 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) in __pmdp_test_and_clear_young()
[all …]

12345