1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * TLB flush routines for radix kernels.
5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
15 #include <asm/ppc-opcode.h>
16 #include <asm/tlb.h>
26  * i.e., r=1 and is=01 or is=10 or is=11
39 		     : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)  in tlbiel_radix_set_isa300()
50 	 * Flush the first set of the TLB, and the entire Page Walk Cache  in tlbiel_all_isa300()
51 	 * and partition table entries. Then flush the remaining sets of the  in tlbiel_all_isa300()
52 	 * TLB.  in tlbiel_all_isa300()
95 		WARN(1, "%s called on pre-POWER9 CPU\n", __func__);  in radix__tlbiel_all()
112 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbiel_pid()
126 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_pid()
142 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_pid_lpid()
155 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_lpid()
169 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_lpid_guest()
185 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbiel_va()
201 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_va()
218 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_va_lpid()
234 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");  in __tlbie_lpid_va()
289 	unsigned long va = ((1UL << 52) - 1);  in fixup_tlbie_pid()
308 	unsigned long va = ((1UL << 52) - 1);  in fixup_tlbie_pid_lpid()
342 	unsigned long va = ((1UL << 52) - 1);  in fixup_tlbie_lpid()
377 		 * Flush the first set of the TLB, and if  in _tlbiel_pid()
385 		/* For the remaining sets, just flush the TLB */  in _tlbiel_pid()
400 	 * must be a compile-time constraint to match the "i" constraint  in _tlbie_pid()
426 	 * must be a compile-time contraint to match the "i" constraint  in _tlbie_pid_lpid()
453 	if (t->ric == RIC_FLUSH_TLB)  in do_tlbiel_pid()
454 		_tlbiel_pid(t->pid, RIC_FLUSH_TLB);  in do_tlbiel_pid()
455 	else if (t->ric == RIC_FLUSH_PWC)  in do_tlbiel_pid()
456 		_tlbiel_pid(t->pid, RIC_FLUSH_PWC);  in do_tlbiel_pid()
458 		_tlbiel_pid(t->pid, RIC_FLUSH_ALL);  in do_tlbiel_pid()
473 	if (atomic_read(&mm->context.copros) > 0)  in _tlbiel_pid_multicast()
483 	 * must be a compile-time contraint to match the "i" constraint  in _tlbie_lpid()
506 	 * must be a compile-time contraint to match the "i" constraint  in _tlbie_lpid_guest()
566 	fixup_tlbie_va_range(addr - page_size, pid, ap);  in __tlbie_va_range()
580 	fixup_tlbie_va_range_lpid(addr - page_size, pid, lpid, ap);  in __tlbie_va_range_lpid()
605 	if (t->ric == RIC_FLUSH_TLB)  in do_tlbiel_va()
606 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_TLB);  in do_tlbiel_va()
607 	else if (t->ric == RIC_FLUSH_PWC)  in do_tlbiel_va()
608 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_PWC);  in do_tlbiel_va()
610 		_tlbiel_va(t->va, t->pid, t->psize, RIC_FLUSH_ALL);  in do_tlbiel_va()
620 	if (atomic_read(&mm->context.copros) > 0)  in _tlbiel_va_multicast()
637 	_tlbiel_va_range(t->start, t->end, t->pid, t->page_size,  in do_tlbiel_va_range()
638 				    t->psize, t->also_pwc);  in do_tlbiel_va_range()
686 	if (atomic_read(&mm->context.copros) > 0)  in _tlbiel_va_range_multicast()
691  * Base TLB flushing operations:
693  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
694  *  - flush_tlb_page(vma, vmaddr) flushes one page
695  *  - flush_tlb_range(vma, start, end) flushes a range of pages
696  *  - flush_tlb_kernel_range(start, end) flushes kernel pages
698  *  - local_* variants of page and mm only apply to the current
706 	pid = mm->context.id;  in radix__local_flush_tlb_mm()
719 	pid = mm->context.id;  in radix__local_flush_all_mm()
738 	pid = mm->context.id;  in radix__local_flush_tlb_page_psize()
751 	radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);  in radix__local_flush_tlb_page()
763 	 * this workaround is required - escalate all RIC=0 IS=1/2/3 flushes  in mm_needs_flush_escalation()
770 	if (atomic_read(&mm->context.copros) > 0)  in mm_needs_flush_escalation()
781 	unsigned long pid = mm->context.id;  in exit_lazy_flush_tlb()
787 	 * interrupted here. In that case, current->mm will be set to mm,  in exit_lazy_flush_tlb()
788 	 * because kthread_use_mm() setting ->mm and switching to the mm is  in exit_lazy_flush_tlb()
791 	if (current->mm == mm)  in exit_lazy_flush_tlb()
794 	if (current->active_mm == mm) {  in exit_lazy_flush_tlb()
795 		WARN_ON_ONCE(current->mm != NULL);  in exit_lazy_flush_tlb()
796 		/* Is a kernel thread and is using mm as the lazy tlb */  in exit_lazy_flush_tlb()
798 		current->active_mm = &init_mm;  in exit_lazy_flush_tlb()
808 	 * not otherwise be a problem, but the TLB must be flushed because  in exit_lazy_flush_tlb()
812 		atomic_dec(&mm->context.active_cpus);  in exit_lazy_flush_tlb()
850  * mm_cpumask can be trimmed for the case where it's not a single-threaded
852  * flushes. Don't want this to be so low that it adds noticable cost to TLB
875 	int active_cpus = atomic_read(&mm->context.active_cpus);  in flush_type_needed()
881 		if (current->mm != mm) {  in flush_type_needed()
896 	if (atomic_read(&mm->context.copros) > 0)  in flush_type_needed()
908 	 * If we are running the only thread of a single-threaded process,  in flush_type_needed()
913 	if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) {  in flush_type_needed()
919 		 * and the TLB still gets flushed by the IPI, so this CPU  in flush_type_needed()
931 		if (current->mm == mm)  in flush_type_needed()
947 	pid = mm->context.id;  in radix__flush_tlb_mm()
965 			if (atomic_read(&mm->context.copros) > 0)  in radix__flush_tlb_mm()
968 					       H_RPTI_PAGE_ALL, 0, -1UL);  in radix__flush_tlb_mm()
987 	pid = mm->context.id;  in __flush_all_mm()
1002 			if (atomic_read(&mm->context.copros) > 0)  in __flush_all_mm()
1005 					       H_RPTI_PAGE_ALL, 0, -1UL);  in __flush_all_mm()
1026 	pid = mm->context.id;  in radix__flush_tlb_page_psize()
1043 			if (atomic_read(&mm->context.copros) > 0)  in radix__flush_tlb_page_psize()
1062 	radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);  in radix__flush_tlb_page()
1107 #define TLB_FLUSH_ALL -1UL
1114  * It also does not iterate over sets like the local tlbiel variant when
1116  * individual page flushes to full-pid flushes.
1127 	unsigned long nr_pages = (end - start) >> page_shift;  in __radix__flush_tlb_range()
1132 	pid = mm->context.id;  in __radix__flush_tlb_range()
1153 	if (!flush_pid && (end - start) >= PMD_SIZE)  in __radix__flush_tlb_range()
1163 		if (atomic_read(&mm->context.copros) > 0)  in __radix__flush_tlb_range()
1186 			hstart = (start + PMD_SIZE - 1) & PMD_MASK;  in __radix__flush_tlb_range()
1232 	__radix__flush_tlb_range(vma->vm_mm, start, end);  in radix__flush_tlb_range()
1247 		return -1;  in radix_get_mmu_psize()
1290 void radix__tlb_flush(struct mmu_gather *tlb)  in radix__tlb_flush()  argument
1293 	struct mm_struct *mm = tlb->mm;  in radix__tlb_flush()
1294 	int page_size = tlb->page_size;  in radix__tlb_flush()
1295 	unsigned long start = tlb->start;  in radix__tlb_flush()
1296 	unsigned long end = tlb->end;  in radix__tlb_flush()
1305 	if (tlb->fullmm || tlb->need_flush_all) {  in radix__tlb_flush()
1307 	} else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {  in radix__tlb_flush()
1308 		if (!tlb->freed_tables)  in radix__tlb_flush()
1313 		if (!tlb->freed_tables)  in radix__tlb_flush()
1327 	unsigned long nr_pages = (end - start) >> page_shift;  in __radix__flush_tlb_range_psize()
1332 	pid = mm->context.id;  in __radix__flush_tlb_range_psize()
1358 		if (atomic_read(&mm->context.copros) > 0)  in __radix__flush_tlb_range_psize()
1408 	pid = mm->context.id;  in radix__flush_tlb_collapsed_pmd()
1435 			if (atomic_read(&mm->context.copros) > 0)  in radix__flush_tlb_collapsed_pmd()
1453 	radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);  in radix__flush_pmd_tlb_range()
1465 	rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */  in radix__flush_tlb_all()
1472 		     : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");  in radix__flush_tlb_all()
1477 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");  in radix__flush_tlb_all()
1483  * Performs process-scoped invalidations for a given LPID
1507 	if (start == 0 && end == -1)  in do_h_rpt_invalidate_prt()
1513 		if (!(pg_sizes & def->h_rpt_pgsize))  in do_h_rpt_invalidate_prt()
1516 		nr_pages = (end - start) >> def->shift;  in do_h_rpt_invalidate_prt()
1530 				     (1UL << def->shift), psize, false);  in do_h_rpt_invalidate_prt()