| /Linux-v4.19/arch/s390/mm/ |
| D | pgtable.c | 30 pte_t *ptep, int nodat) in ptep_ipte_local() argument
43 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local()
45 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
50 pte_t *ptep, int nodat) in ptep_ipte_global() argument
63 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global()
65 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
70 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument
75 old = *ptep; in ptep_flush_direct()
81 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
83 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
[all …]
|
| /Linux-v4.19/arch/arm64/mm/ |
| D | hugetlbpage.c | 55 pte_t *ptep, size_t *pgsize) in find_num_contig() argument
64 if ((pte_t *)pmdp == ptep) { in find_num_contig()
107 pte_t *ptep, in get_clear_flush() argument
111 pte_t orig_pte = huge_ptep_get(ptep); in get_clear_flush()
115 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { in get_clear_flush()
116 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_flush()
148 pte_t *ptep, in clear_flush() argument
155 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) in clear_flush()
156 pte_clear(mm, addr, ptep); in clear_flush()
162 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
[all …]
|
| /Linux-v4.19/arch/arm/include/asm/ |
| D | hugetlb-3level.h | 32 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
34 pte_t retval = *ptep; in huge_ptep_get()
41 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
43 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
49 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
53 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
55 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
59 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
61 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
[all …]
|
| /Linux-v4.19/arch/s390/include/asm/ |
| D | hugetlb.h | 21 pte_t *ptep, pte_t pte);
22 pte_t huge_ptep_get(pte_t *ptep);
24 unsigned long addr, pte_t *ptep);
46 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument
48 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) in huge_pte_clear()
49 pte_val(*ptep) = _REGION3_ENTRY_EMPTY; in huge_pte_clear()
51 pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY; in huge_pte_clear()
55 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument
57 huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush()
61 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
[all …]
|
| /Linux-v4.19/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable.h | 181 pte_t *ptep, unsigned long clr, in pte_update() argument
194 : "=&r" (old), "=&r" (tmp), "=m" (*ptep) in pte_update()
195 : "r" (ptep), "r" (clr), "m" (*ptep), "r" (set) in pte_update()
198 unsigned long old = pte_val(*ptep); in pte_update()
199 *ptep = __pte((old & ~clr) | set); in pte_update()
214 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
218 if (pte_young(*ptep)) in __ptep_test_and_clear_young()
220 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young()
233 pte_t *ptep) in ptep_set_wrprotect() argument
236 if ((pte_val(*ptep) & _PAGE_RW) == 0) in ptep_set_wrprotect()
[all …]
|
| /Linux-v4.19/arch/x86/include/asm/ |
| D | hugetlb.h | 40 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
42 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
46 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
48 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
52 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
54 ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
68 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
70 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
74 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
77 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
[all …]
|
| D | pgtable-3level.h | 30 static inline void native_set_pte(pte_t *ptep, pte_t pte) in native_set_pte() argument
32 ptep->pte_high = pte.pte_high; in native_set_pte()
34 ptep->pte_low = pte.pte_low; in native_set_pte()
91 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) in native_set_pte_atomic() argument
93 set_64bit((unsigned long long *)(ptep), native_pte_val(pte)); in native_set_pte_atomic()
115 pte_t *ptep) in native_pte_clear() argument
117 ptep->pte_low = 0; in native_pte_clear()
119 ptep->pte_high = 0; in native_pte_clear()
151 static inline pte_t native_ptep_get_and_clear(pte_t *ptep) in native_ptep_get_and_clear() argument
155 res.pte = (pteval_t)arch_atomic64_xchg((atomic64_t *)ptep, 0); in native_ptep_get_and_clear()
[all …]
|
| /Linux-v4.19/arch/ia64/include/asm/ |
| D | hugetlb.h | 25 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
27 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
31 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
33 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
37 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
52 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
54 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
58 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
61 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
64 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /Linux-v4.19/arch/mips/include/asm/ |
| D | hugetlb.h | 51 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
53 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
57 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
60 pte_t pte = *ptep; in huge_ptep_get_and_clear()
63 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear()
68 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
85 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
87 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
92 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument
95 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
[all …]
|
| /Linux-v4.19/arch/sh/include/asm/ |
| D | hugetlb.h | 39 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument
41 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
45 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
47 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
51 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
66 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
68 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
72 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
75 return ptep_set_access_flags(vma, addr, ptep, pte, dirty); in huge_ptep_set_access_flags()
78 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /Linux-v4.19/arch/sparc/include/asm/ |
| D | hugetlb.h | 17 pte_t *ptep, pte_t pte);
20 pte_t *ptep);
45 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
60 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
62 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect()
63 set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect()
67 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
70 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
72 set_huge_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
78 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
[all …]
|
| /Linux-v4.19/arch/powerpc/include/asm/book3s/32/ |
| D | pgtable.h | 109 #define pte_clear(mm, addr, ptep) \ argument
110 do { pte_update(ptep, ~_PAGE_HASHPTE, 0); } while (0)
133 extern void flush_hash_entry(struct mm_struct *mm, pte_t *ptep,
198 static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
201 old = pte_update(ptep, _PAGE_ACCESSED, 0); in __ptep_test_and_clear_young()
203 unsigned long ptephys = __pa(ptep) & PAGE_MASK; in __ptep_test_and_clear_young()
213 pte_t *ptep) in ptep_get_and_clear() argument
215 return __pte(pte_update(ptep, ~_PAGE_HASHPTE, 0)); in ptep_get_and_clear()
220 pte_t *ptep) in ptep_set_wrprotect() argument
222 pte_update(ptep, (_PAGE_RW | _PAGE_HWWRITE), _PAGE_RO); in ptep_set_wrprotect()
[all …]
|
| /Linux-v4.19/arch/parisc/mm/ |
| D | hugetlbpage.c | 118 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument
127 set_pte(ptep, entry); in __set_huge_pte_at()
128 ptep++; in __set_huge_pte_at()
138 pte_t *ptep, pte_t entry) in set_huge_pte_at() argument
143 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at()
149 pte_t *ptep) in huge_ptep_get_and_clear() argument
155 entry = *ptep; in huge_ptep_get_and_clear()
156 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear()
164 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
170 old_pte = *ptep; in huge_ptep_set_wrprotect()
[all …]
|
| /Linux-v4.19/arch/powerpc/kvm/ |
| D | book3s_64_mmu_radix.c | 156 static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep, in kvmppc_radix_update_pte() argument
160 return __radix_pte_update(ptep, clr, set); in kvmppc_radix_update_pte()
164 pte_t *ptep, pte_t pte) in kvmppc_radix_set_pte_at() argument
166 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0); in kvmppc_radix_set_pte_at()
177 static void kvmppc_pte_free(pte_t *ptep) in kvmppc_pte_free() argument
179 kmem_cache_free(kvm_pte_cache, ptep); in kvmppc_pte_free()
365 pte_t *ptep, *new_ptep = NULL; in kvmppc_create_pte() local
496 ptep = pte_offset_kernel(pmd, gpa); in kvmppc_create_pte()
497 if (pte_present(*ptep)) { in kvmppc_create_pte()
499 if (pte_raw(*ptep) == pte_raw(pte)) { in kvmppc_create_pte()
[all …]
|
| /Linux-v4.19/arch/powerpc/include/asm/book3s/64/ |
| D | hash.h | 99 pte_t *ptep, unsigned long pte, int huge);
104 pte_t *ptep, unsigned long clr, in hash__pte_update() argument
119 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in hash__pte_update()
120 : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), in hash__pte_update()
129 hpte_need_flush(mm, addr, ptep, old, huge); in hash__pte_update()
137 static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) in hash__ptep_set_access_flags() argument
153 :"=&r" (old), "=&r" (tmp), "=m" (*ptep) in hash__ptep_set_access_flags()
154 :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) in hash__ptep_set_access_flags()
177 pte_t *ptep, pte_t pte, int percpu) in hash__set_pte_at() argument
183 *ptep = pte; in hash__set_pte_at()
|
| D | radix.h | 126 extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
130 static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, in __radix_pte_update() argument
141 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in __radix_pte_update()
142 : "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr)) in __radix_pte_update()
150 pte_t *ptep, unsigned long clr, in radix__pte_update() argument
156 old_pte = __radix_pte_update(ptep, clr, set); in radix__pte_update()
165 pte_t *ptep, int full) in radix__ptep_get_and_clear_full() argument
170 old_pte = pte_val(*ptep); in radix__ptep_get_and_clear_full()
171 *ptep = __pte(0); in radix__ptep_get_and_clear_full()
173 old_pte = radix__pte_update(mm, addr, ptep, ~0ul, 0, 0); in radix__ptep_get_and_clear_full()
[all …]
|
| /Linux-v4.19/arch/arm64/include/asm/ |
| D | hugetlb.h | 23 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
25 return READ_ONCE(*ptep); in huge_ptep_get()
74 pte_t *ptep, pte_t pte);
76 unsigned long addr, pte_t *ptep,
79 unsigned long addr, pte_t *ptep);
81 unsigned long addr, pte_t *ptep);
83 unsigned long addr, pte_t *ptep);
85 pte_t *ptep, unsigned long sz);
88 pte_t *ptep, pte_t pte, unsigned long sz);
|
| /Linux-v4.19/arch/x86/xen/ |
| D | p2m.c | 211 pte_t *ptep; in xen_build_mfn_list_list() local
239 ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), in xen_build_mfn_list_list()
241 BUG_ON(!ptep || level != PG_LEVEL_4K); in xen_build_mfn_list_list()
242 mfn = pte_mfn(*ptep); in xen_build_mfn_list_list()
243 ptep = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); in xen_build_mfn_list_list()
249 if (ptep == p2m_missing_pte || ptep == p2m_identity_pte) { in xen_build_mfn_list_list()
327 pte_t *ptep; in xen_rebuild_p2m_list() local
379 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
380 set_pte(ptep, in xen_rebuild_p2m_list()
389 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
[all …]
|
| /Linux-v4.19/drivers/iommu/ |
| D | io-pgtable-arm.c | 278 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, in __arm_lpae_sync_pte() argument
281 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte()
282 sizeof(*ptep), DMA_TO_DEVICE); in __arm_lpae_sync_pte()
285 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, in __arm_lpae_set_pte() argument
288 *ptep = pte; in __arm_lpae_set_pte()
291 __arm_lpae_sync_pte(ptep, cfg); in __arm_lpae_set_pte()
296 arm_lpae_iopte *ptep);
300 int lvl, arm_lpae_iopte *ptep) in __arm_lpae_init_pte() argument
315 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); in __arm_lpae_init_pte()
321 arm_lpae_iopte *ptep) in arm_lpae_init_pte() argument
[all …]
|
| D | io-pgtable-arm-v7s.c | 250 static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, in __arm_v7s_pte_sync() argument
256 dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), in __arm_v7s_pte_sync()
257 num_entries * sizeof(*ptep), DMA_TO_DEVICE); in __arm_v7s_pte_sync()
259 static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte, in __arm_v7s_set_pte() argument
265 ptep[i] = pte; in __arm_v7s_set_pte()
267 __arm_v7s_pte_sync(ptep, num_entries, cfg); in __arm_v7s_set_pte()
370 int lvl, int num_entries, arm_v7s_iopte *ptep) in arm_v7s_init_pte() argument
377 if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) { in arm_v7s_init_pte()
385 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl); in arm_v7s_init_pte()
389 } else if (ptep[i]) { in arm_v7s_init_pte()
[all …]
|
| /Linux-v4.19/arch/riscv/include/asm/ |
| D | pgtable.h | 273 unsigned long address, pte_t *ptep) in update_mmu_cache() argument
296 static inline void set_pte(pte_t *ptep, pte_t pteval) in set_pte() argument
298 *ptep = pteval; in set_pte()
304 unsigned long addr, pte_t *ptep, pte_t pteval) in set_pte_at() argument
309 set_pte(ptep, pteval); in set_pte_at()
313 unsigned long addr, pte_t *ptep) in pte_clear() argument
315 set_pte_at(mm, addr, ptep, __pte(0)); in pte_clear()
320 unsigned long address, pte_t *ptep, in ptep_set_access_flags() argument
323 if (!pte_same(*ptep, entry)) in ptep_set_access_flags()
324 set_pte_at(vma->vm_mm, address, ptep, entry); in ptep_set_access_flags()
[all …]
|
| /Linux-v4.19/arch/powerpc/mm/ |
| D | pgtable-book3e.c | 78 pte_t *ptep; in map_kernel_page() local
89 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
90 if (!ptep) in map_kernel_page()
92 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, in map_kernel_page()
111 ptep = early_alloc_pgtable(PAGE_SIZE); in map_kernel_page()
112 BUG_ON(ptep == NULL); in map_kernel_page()
113 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
115 ptep = pte_offset_kernel(pmdp, ea); in map_kernel_page()
116 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, in map_kernel_page()
|
| /Linux-v4.19/arch/parisc/include/asm/ |
| D | hugetlb.h | 10 pte_t *ptep, pte_t pte);
13 pte_t *ptep);
44 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
59 unsigned long addr, pte_t *ptep);
62 unsigned long addr, pte_t *ptep,
65 static inline pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
67 return *ptep; in huge_ptep_get()
|
| /Linux-v4.19/arch/xtensa/include/asm/ |
| D | pgtable.h | 263 #define pte_clear(mm,addr,ptep) \ argument
264 do { update_pte(ptep, __pte(_PAGE_CA_INVALID | _PAGE_USER)); } while (0)
314 static inline void update_pte(pte_t *ptep, pte_t pteval) in update_pte() argument
316 *ptep = pteval; in update_pte()
318 __asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep)); in update_pte()
326 set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval) in set_pte_at() argument
328 update_pte(ptep, pteval); in set_pte_at()
331 static inline void set_pte(pte_t *ptep, pte_t pteval) in set_pte() argument
333 update_pte(ptep, pteval); in set_pte()
346 pte_t *ptep) in ptep_test_and_clear_young() argument
[all …]
|
| /Linux-v4.19/include/trace/events/ |
| D | xen.h | 131 TP_PROTO(pte_t *ptep, pte_t pteval),
132 TP_ARGS(ptep, pteval),
134 __field(pte_t *, ptep)
137 TP_fast_assign(__entry->ptep = ptep;
140 __entry->ptep,
147 TP_PROTO(pte_t *ptep, pte_t pteval), \
148 TP_ARGS(ptep, pteval))
154 pte_t *ptep, pte_t pteval),
155 TP_ARGS(mm, addr, ptep, pteval),
159 __field(pte_t *, ptep)
[all …]
|
