1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Based on arch/arm/include/asm/tlbflush.h
4 *
5 * Copyright (C) 1999-2003 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8 #ifndef __ASM_TLBFLUSH_H
9 #define __ASM_TLBFLUSH_H
10
11 #ifndef __ASSEMBLY__
12
13 #include <linux/bitfield.h>
14 #include <linux/mm_types.h>
15 #include <linux/sched.h>
16 #include <asm/cputype.h>
17 #include <asm/mmu.h>
18
19 /*
20 * Raw TLBI operations.
21 *
22 * Where necessary, use the __tlbi() macro to avoid asm()
23 * boilerplate. Drivers and most kernel code should use the TLB
24 * management routines in preference to the macro below.
25 *
26 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
27 * on whether a particular TLBI operation takes an argument or
28 * not. The macros handles invoking the asm with or without the
29 * register argument as appropriate.
30 */
31 #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \
32 "tlbi " #op "\n" \
33 ALTERNATIVE("nop\n nop", \
34 "dsb ish\n tlbi " #op, \
35 ARM64_WORKAROUND_REPEAT_TLBI, \
36 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
37 : : )
38
39 #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \
40 "tlbi " #op ", %0\n" \
41 ALTERNATIVE("nop\n nop", \
42 "dsb ish\n tlbi " #op ", %0", \
43 ARM64_WORKAROUND_REPEAT_TLBI, \
44 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
45 : : "r" (arg))
46
47 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)
48
49 #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0)
50
51 #define __tlbi_user(op, arg) do { \
52 if (arm64_kernel_unmapped_at_el0()) \
53 __tlbi(op, (arg) | USER_ASID_FLAG); \
54 } while (0)
55
56 /* This macro creates a properly formatted VA operand for the TLBI */
57 #define __TLBI_VADDR(addr, asid) \
58 ({ \
59 unsigned long __ta = (addr) >> 12; \
60 __ta &= GENMASK_ULL(43, 0); \
61 __ta |= (unsigned long)(asid) << 48; \
62 __ta; \
63 })
64
65 /*
66 * Get translation granule of the system, which is decided by
67 * PAGE_SIZE. Used by TTL.
68 * - 4KB : 1
69 * - 16KB : 2
70 * - 64KB : 3
71 */
72 #define TLBI_TTL_TG_4K 1
73 #define TLBI_TTL_TG_16K 2
74 #define TLBI_TTL_TG_64K 3
75
get_trans_granule(void)76 static inline unsigned long get_trans_granule(void)
77 {
78 switch (PAGE_SIZE) {
79 case SZ_4K:
80 return TLBI_TTL_TG_4K;
81 case SZ_16K:
82 return TLBI_TTL_TG_16K;
83 case SZ_64K:
84 return TLBI_TTL_TG_64K;
85 default:
86 return 0;
87 }
88 }
89
90 /*
91 * Level-based TLBI operations.
92 *
93 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
94 * the level at which the invalidation must take place. If the level is
95 * wrong, no invalidation may take place. In the case where the level
96 * cannot be easily determined, a 0 value for the level parameter will
97 * perform a non-hinted invalidation.
98 *
99 * For Stage-2 invalidation, use the level values provided to that effect
100 * in asm/stage2_pgtable.h.
101 */
102 #define TLBI_TTL_MASK GENMASK_ULL(47, 44)
103
104 #define __tlbi_level(op, addr, level) do { \
105 u64 arg = addr; \
106 \
107 if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) && \
108 level) { \
109 u64 ttl = level & 3; \
110 ttl |= get_trans_granule() << 2; \
111 arg &= ~TLBI_TTL_MASK; \
112 arg |= FIELD_PREP(TLBI_TTL_MASK, ttl); \
113 } \
114 \
115 __tlbi(op, arg); \
116 } while(0)
117
118 #define __tlbi_user_level(op, arg, level) do { \
119 if (arm64_kernel_unmapped_at_el0()) \
120 __tlbi_level(op, (arg | USER_ASID_FLAG), level); \
121 } while (0)
122
123 /*
124 * This macro creates a properly formatted VA operand for the TLB RANGE.
125 * The value bit assignments are:
126 *
127 * +----------+------+-------+-------+-------+----------------------+
128 * | ASID | TG | SCALE | NUM | TTL | BADDR |
129 * +-----------------+-------+-------+-------+----------------------+
130 * |63 48|47 46|45 44|43 39|38 37|36 0|
131 *
132 * The address range is determined by below formula:
133 * [BADDR, BADDR + (NUM + 1) * 2^(5*SCALE + 1) * PAGESIZE)
134 *
135 */
136 #define __TLBI_VADDR_RANGE(addr, asid, scale, num, ttl) \
137 ({ \
138 unsigned long __ta = (addr) >> PAGE_SHIFT; \
139 __ta &= GENMASK_ULL(36, 0); \
140 __ta |= (unsigned long)(ttl) << 37; \
141 __ta |= (unsigned long)(num) << 39; \
142 __ta |= (unsigned long)(scale) << 44; \
143 __ta |= get_trans_granule() << 46; \
144 __ta |= (unsigned long)(asid) << 48; \
145 __ta; \
146 })
147
148 /* These macros are used by the TLBI RANGE feature. */
149 #define __TLBI_RANGE_PAGES(num, scale) \
150 ((unsigned long)((num) + 1) << (5 * (scale) + 1))
151 #define MAX_TLBI_RANGE_PAGES __TLBI_RANGE_PAGES(31, 3)
152
153 /*
154 * Generate 'num' values from -1 to 30 with -1 rejected by the
155 * __flush_tlb_range() loop below.
156 */
157 #define TLBI_RANGE_MASK GENMASK_ULL(4, 0)
158 #define __TLBI_RANGE_NUM(pages, scale) \
159 ((((pages) >> (5 * (scale) + 1)) & TLBI_RANGE_MASK) - 1)
160
161 /*
162 * TLB Invalidation
163 * ================
164 *
165 * This header file implements the low-level TLB invalidation routines
166 * (sometimes referred to as "flushing" in the kernel) for arm64.
167 *
168 * Every invalidation operation uses the following template:
169 *
170 * DSB ISHST // Ensure prior page-table updates have completed
171 * TLBI ... // Invalidate the TLB
172 * DSB ISH // Ensure the TLB invalidation has completed
173 * if (invalidated kernel mappings)
174 * ISB // Discard any instructions fetched from the old mapping
175 *
176 *
177 * The following functions form part of the "core" TLB invalidation API,
178 * as documented in Documentation/core-api/cachetlb.rst:
179 *
180 * flush_tlb_all()
181 * Invalidate the entire TLB (kernel + user) on all CPUs
182 *
183 * flush_tlb_mm(mm)
184 * Invalidate an entire user address space on all CPUs.
185 * The 'mm' argument identifies the ASID to invalidate.
186 *
187 * flush_tlb_range(vma, start, end)
188 * Invalidate the virtual-address range '[start, end)' on all
189 * CPUs for the user address space corresponding to 'vma->mm'.
190 * Note that this operation also invalidates any walk-cache
191 * entries associated with translations for the specified address
192 * range.
193 *
194 * flush_tlb_kernel_range(start, end)
195 * Same as flush_tlb_range(..., start, end), but applies to
196 * kernel mappings rather than a particular user address space.
197 * Whilst not explicitly documented, this function is used when
198 * unmapping pages from vmalloc/io space.
199 *
200 * flush_tlb_page(vma, addr)
201 * Invalidate a single user mapping for address 'addr' in the
202 * address space corresponding to 'vma->mm'. Note that this
203 * operation only invalidates a single, last-level page-table
204 * entry and therefore does not affect any walk-caches.
205 *
206 *
207 * Next, we have some undocumented invalidation routines that you probably
208 * don't want to call unless you know what you're doing:
209 *
210 * local_flush_tlb_all()
211 * Same as flush_tlb_all(), but only applies to the calling CPU.
212 *
213 * __flush_tlb_kernel_pgtable(addr)
214 * Invalidate a single kernel mapping for address 'addr' on all
215 * CPUs, ensuring that any walk-cache entries associated with the
216 * translation are also invalidated.
217 *
218 * __flush_tlb_range(vma, start, end, stride, last_level)
219 * Invalidate the virtual-address range '[start, end)' on all
220 * CPUs for the user address space corresponding to 'vma->mm'.
221 * The invalidation operations are issued at a granularity
222 * determined by 'stride' and only affect any walk-cache entries
223 * if 'last_level' is equal to false.
224 *
225 *
226 * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
227 * on top of these routines, since that is our interface to the mmu_gather
228 * API as used by munmap() and friends.
229 */
local_flush_tlb_all(void)230 static inline void local_flush_tlb_all(void)
231 {
232 dsb(nshst);
233 __tlbi(vmalle1);
234 dsb(nsh);
235 isb();
236 }
237
flush_tlb_all(void)238 static inline void flush_tlb_all(void)
239 {
240 dsb(ishst);
241 __tlbi(vmalle1is);
242 dsb(ish);
243 isb();
244 }
245
flush_tlb_mm(struct mm_struct * mm)246 static inline void flush_tlb_mm(struct mm_struct *mm)
247 {
248 unsigned long asid = __TLBI_VADDR(0, ASID(mm));
249
250 dsb(ishst);
251 __tlbi(aside1is, asid);
252 __tlbi_user(aside1is, asid);
253 dsb(ish);
254 }
255
flush_tlb_page_nosync(struct vm_area_struct * vma,unsigned long uaddr)256 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
257 unsigned long uaddr)
258 {
259 unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
260
261 dsb(ishst);
262 __tlbi(vale1is, addr);
263 __tlbi_user(vale1is, addr);
264 }
265
flush_tlb_page(struct vm_area_struct * vma,unsigned long uaddr)266 static inline void flush_tlb_page(struct vm_area_struct *vma,
267 unsigned long uaddr)
268 {
269 flush_tlb_page_nosync(vma, uaddr);
270 dsb(ish);
271 }
272
273 /*
274 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
275 * necessarily a performance improvement.
276 */
277 #define MAX_TLBI_OPS PTRS_PER_PTE
278
__flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long stride,bool last_level,int tlb_level)279 static inline void __flush_tlb_range(struct vm_area_struct *vma,
280 unsigned long start, unsigned long end,
281 unsigned long stride, bool last_level,
282 int tlb_level)
283 {
284 int num = 0;
285 int scale = 0;
286 unsigned long asid = ASID(vma->vm_mm);
287 unsigned long addr;
288 unsigned long pages;
289
290 start = round_down(start, stride);
291 end = round_up(end, stride);
292 pages = (end - start) >> PAGE_SHIFT;
293
294 /*
295 * When not uses TLB range ops, we can handle up to
296 * (MAX_TLBI_OPS - 1) pages;
297 * When uses TLB range ops, we can handle up to
298 * (MAX_TLBI_RANGE_PAGES - 1) pages.
299 */
300 if ((!system_supports_tlb_range() &&
301 (end - start) >= (MAX_TLBI_OPS * stride)) ||
302 pages >= MAX_TLBI_RANGE_PAGES) {
303 flush_tlb_mm(vma->vm_mm);
304 return;
305 }
306
307 dsb(ishst);
308
309 /*
310 * When the CPU does not support TLB range operations, flush the TLB
311 * entries one by one at the granularity of 'stride'. If the the TLB
312 * range ops are supported, then:
313 *
314 * 1. If 'pages' is odd, flush the first page through non-range
315 * operations;
316 *
317 * 2. For remaining pages: the minimum range granularity is decided
318 * by 'scale', so multiple range TLBI operations may be required.
319 * Start from scale = 0, flush the corresponding number of pages
320 * ((num+1)*2^(5*scale+1) starting from 'addr'), then increase it
321 * until no pages left.
322 *
323 * Note that certain ranges can be represented by either num = 31 and
324 * scale or num = 0 and scale + 1. The loop below favours the latter
325 * since num is limited to 30 by the __TLBI_RANGE_NUM() macro.
326 */
327 while (pages > 0) {
328 if (!system_supports_tlb_range() ||
329 pages % 2 == 1) {
330 addr = __TLBI_VADDR(start, asid);
331 if (last_level) {
332 __tlbi_level(vale1is, addr, tlb_level);
333 __tlbi_user_level(vale1is, addr, tlb_level);
334 } else {
335 __tlbi_level(vae1is, addr, tlb_level);
336 __tlbi_user_level(vae1is, addr, tlb_level);
337 }
338 start += stride;
339 pages -= stride >> PAGE_SHIFT;
340 continue;
341 }
342
343 num = __TLBI_RANGE_NUM(pages, scale);
344 if (num >= 0) {
345 addr = __TLBI_VADDR_RANGE(start, asid, scale,
346 num, tlb_level);
347 if (last_level) {
348 __tlbi(rvale1is, addr);
349 __tlbi_user(rvale1is, addr);
350 } else {
351 __tlbi(rvae1is, addr);
352 __tlbi_user(rvae1is, addr);
353 }
354 start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;
355 pages -= __TLBI_RANGE_PAGES(num, scale);
356 }
357 scale++;
358 }
359 dsb(ish);
360 }
361
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)362 static inline void flush_tlb_range(struct vm_area_struct *vma,
363 unsigned long start, unsigned long end)
364 {
365 /*
366 * We cannot use leaf-only invalidation here, since we may be invalidating
367 * table entries as part of collapsing hugepages or moving page tables.
368 * Set the tlb_level to 0 because we can not get enough information here.
369 */
370 __flush_tlb_range(vma, start, end, PAGE_SIZE, false, 0);
371 }
372
flush_tlb_kernel_range(unsigned long start,unsigned long end)373 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
374 {
375 unsigned long addr;
376
377 if ((end - start) > (MAX_TLBI_OPS * PAGE_SIZE)) {
378 flush_tlb_all();
379 return;
380 }
381
382 start = __TLBI_VADDR(start, 0);
383 end = __TLBI_VADDR(end, 0);
384
385 dsb(ishst);
386 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
387 __tlbi(vaale1is, addr);
388 dsb(ish);
389 isb();
390 }
391
392 /*
393 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
394 * table levels (pgd/pud/pmd).
395 */
__flush_tlb_kernel_pgtable(unsigned long kaddr)396 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
397 {
398 unsigned long addr = __TLBI_VADDR(kaddr, 0);
399
400 dsb(ishst);
401 __tlbi(vaae1is, addr);
402 dsb(ish);
403 isb();
404 }
405 #endif
406
407 #endif
408
