1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/mm.h>
4 #include <linux/smp.h>
5 #include <linux/sched.h>
6 #include <asm/sbi.h>
7 #include <asm/mmu_context.h>
8
local_flush_tlb_all_asid(unsigned long asid)9 static inline void local_flush_tlb_all_asid(unsigned long asid)
10 {
11 __asm__ __volatile__ ("sfence.vma x0, %0"
12 :
13 : "r" (asid)
14 : "memory");
15 }
16
local_flush_tlb_page_asid(unsigned long addr,unsigned long asid)17 static inline void local_flush_tlb_page_asid(unsigned long addr,
18 unsigned long asid)
19 {
20 __asm__ __volatile__ ("sfence.vma %0, %1"
21 :
22 : "r" (addr), "r" (asid)
23 : "memory");
24 }
25
flush_tlb_all(void)26 void flush_tlb_all(void)
27 {
28 sbi_remote_sfence_vma(NULL, 0, -1);
29 }
30
__sbi_tlb_flush_range(struct mm_struct * mm,unsigned long start,unsigned long size,unsigned long stride)31 static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
32 unsigned long size, unsigned long stride)
33 {
34 struct cpumask *cmask = mm_cpumask(mm);
35 unsigned int cpuid;
36 bool broadcast;
37
38 if (cpumask_empty(cmask))
39 return;
40
41 cpuid = get_cpu();
42 /* check if the tlbflush needs to be sent to other CPUs */
43 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
44 if (static_branch_unlikely(&use_asid_allocator)) {
45 unsigned long asid = atomic_long_read(&mm->context.id);
46
47 if (broadcast) {
48 sbi_remote_sfence_vma_asid(cmask, start, size, asid);
49 } else if (size <= stride) {
50 local_flush_tlb_page_asid(start, asid);
51 } else {
52 local_flush_tlb_all_asid(asid);
53 }
54 } else {
55 if (broadcast) {
56 sbi_remote_sfence_vma(cmask, start, size);
57 } else if (size <= stride) {
58 local_flush_tlb_page(start);
59 } else {
60 local_flush_tlb_all();
61 }
62 }
63
64 put_cpu();
65 }
66
flush_tlb_mm(struct mm_struct * mm)67 void flush_tlb_mm(struct mm_struct *mm)
68 {
69 __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE);
70 }
71
flush_tlb_page(struct vm_area_struct * vma,unsigned long addr)72 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
73 {
74 __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
75 }
76
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)77 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
78 unsigned long end)
79 {
80 __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE);
81 }
82 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
flush_pmd_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)83 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
84 unsigned long end)
85 {
86 __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE);
87 }
88 #endif
89
