Lines Matching +full:i +full:- +full:tlb +full:- +full:sets
1 // SPDX-License-Identifier: GPL-2.0-or-later
23 #include <asm/tlb.h>
27 #include <asm/ppc-opcode.h>
28 #include <asm/feature-fixups.h>
30 #include <misc/cxl-base.h>
57 * i.e., r=1 and is=01 or is=10 or is=11
71 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "i"(r) in tlbiel_hash_set_isa300()
107 * * PRS=1, R=0, and RIC!=2 (The only process-scoped in tlbiel_all_isa300()
113 * Then flush the sets of the TLB proper. Hash mode uses in tlbiel_all_isa300()
114 * partition scoped TLB translations, which may be flushed in tlbiel_all_isa300()
147 WARN(1, "%s called on pre-POWER7 CPU\n", __func__); in hash__tlbiel_all()
176 va &= ~((1ul << (64 - 52)) - 1); in ___tlbie()
181 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) in ___tlbie()
185 /* We need 14 to 14 + i bits of va */ in ___tlbie()
187 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); in ___tlbie()
200 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) in ___tlbie()
223 * re-order the tlbie in fixup_tlbie_vpn()
227 : : "r"(rb), "i"(r), "i"(prs), in fixup_tlbie_vpn()
228 "i"(ric), "r"(rs) : "memory"); in fixup_tlbie_vpn()
266 va &= ~((1ul << (64 - 52)) - 1); in __tlbiel()
271 : : "r" (va), "i" (CPU_FTR_ARCH_206) in __tlbiel()
275 /* We need 14 to 14 + i bits of va */ in __tlbiel()
277 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); in __tlbiel()
290 : : "r" (va), "i" (CPU_FTR_ARCH_206) in __tlbiel()
325 unsigned long *word = (unsigned long *)&hptep->v; in native_lock_hpte()
339 unsigned long *word = (unsigned long *)&hptep->v; in native_unlock_hpte()
350 int i; in native_hpte_insert() local
358 for (i = 0; i < HPTES_PER_GROUP; i++) { in native_hpte_insert()
359 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) { in native_hpte_insert()
362 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) in native_hpte_insert()
370 if (i == HPTES_PER_GROUP) in native_hpte_insert()
371 return -1; in native_hpte_insert()
377 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", in native_hpte_insert()
378 i, hpte_v, hpte_r); in native_hpte_insert()
386 hptep->r = cpu_to_be64(hpte_r); in native_hpte_insert()
393 hptep->v = cpu_to_be64(hpte_v); in native_hpte_insert()
397 return i | (!!(vflags & HPTE_V_SECONDARY) << 3); in native_hpte_insert()
403 int i; in native_hpte_remove() local
412 for (i = 0; i < HPTES_PER_GROUP; i++) { in native_hpte_remove()
414 hpte_v = be64_to_cpu(hptep->v); in native_hpte_remove()
419 hpte_v = be64_to_cpu(hptep->v); in native_hpte_remove()
430 if (i == HPTES_PER_GROUP) in native_hpte_remove()
431 return -1; in native_hpte_remove()
434 hptep->v = 0; in native_hpte_remove()
436 return i; in native_hpte_remove()
454 * We need to invalidate the TLB always because hpte_remove doesn't do in native_hpte_updatepp()
455 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less in native_hpte_updatepp()
456 * random entry from it. When we do that we don't invalidate the TLB in native_hpte_updatepp()
461 DBG_LOW(" -> miss\n"); in native_hpte_updatepp()
462 ret = -1; in native_hpte_updatepp()
469 ret = -1; in native_hpte_updatepp()
471 DBG_LOW(" -> hit\n"); in native_hpte_updatepp()
473 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & in native_hpte_updatepp()
484 * Ensure it is out of the tlb too if it is not a nohpte fault in native_hpte_updatepp()
496 unsigned long i; in __native_hpte_find() local
498 for (i = 0; i < HPTES_PER_GROUP; i++) { in __native_hpte_find()
508 return -1; in __native_hpte_find()
532 return -1; in native_hpte_find()
557 if (slot == -1) in native_hpte_updateboltedpp()
562 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & in native_hpte_updateboltedpp()
566 * Ensure it is out of the tlb too. Bolted entries base and in native_hpte_updateboltedpp()
588 if (slot == -1) in native_hpte_removebolted()
589 return -ENOENT; in native_hpte_removebolted()
593 VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED)); in native_hpte_removebolted()
596 hptep->v = 0; in native_hpte_removebolted()
598 /* Invalidate the TLB */ in native_hpte_removebolted()
626 hptep->v = 0; in native_hpte_invalidate()
631 * We need to invalidate the TLB always because hpte_remove doesn't do in native_hpte_invalidate()
632 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less in native_hpte_invalidate()
633 * random entry from it. When we do that we don't invalidate the TLB in native_hpte_invalidate()
648 int i; in native_hugepage_invalidate() local
657 max_hpte_count = 1U << (PMD_SHIFT - shift); in native_hugepage_invalidate()
660 for (i = 0; i < max_hpte_count; i++) { in native_hugepage_invalidate()
661 valid = hpte_valid(hpte_slot_array, i); in native_hugepage_invalidate()
664 hidx = hpte_hash_index(hpte_slot_array, i); in native_hugepage_invalidate()
667 addr = s_addr + (i * (1ul << shift)); in native_hugepage_invalidate()
680 /* Even if we miss, we need to invalidate the TLB */ in native_hugepage_invalidate()
691 hptep->v = 0; in native_hugepage_invalidate()
696 * We need to do tlb invalidate for all the address, tlbie in native_hugepage_invalidate()
697 * instruction compares entry_VA in tlb with the VA specified in native_hugepage_invalidate()
718 unsigned long hpte_v = be64_to_cpu(hpte->v); in hpte_decode()
719 unsigned long hpte_r = be64_to_cpu(hpte->r); in hpte_decode()
723 unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); in hpte_decode()
747 /* We only have 28 - 23 bits of seg_off in avpn */ in hpte_decode()
755 *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT; in hpte_decode()
758 /* We only have 40 - 23 bits of seg_off in avpn */ in hpte_decode()
765 *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT; in hpte_decode()
809 hpte_v = be64_to_cpu(hptep->v); in native_hpte_clear()
817 hptep->v = 0; in native_hpte_clear()
839 unsigned long psize = batch->psize; in native_flush_hash_range()
840 int ssize = batch->ssize; in native_flush_hash_range()
841 int i; in native_flush_hash_range() local
849 for (i = 0; i < number; i++) { in native_flush_hash_range()
850 vpn = batch->vpn[i]; in native_flush_hash_range()
851 pte = batch->pte[i]; in native_flush_hash_range()
873 hptep->v = 0; in native_flush_hash_range()
880 for (i = 0; i < number; i++) { in native_flush_hash_range()
881 vpn = batch->vpn[i]; in native_flush_hash_range()
882 pte = batch->pte[i]; in native_flush_hash_range()
897 for (i = 0; i < number; i++) { in native_flush_hash_range()
898 vpn = batch->vpn[i]; in native_flush_hash_range()
899 pte = batch->pte[i]; in native_flush_hash_range()