Lines Matching +full:tlb +full:- +full:split
6 * Synthesize TLB refill handlers at runtime.
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
34 #include <asm/cpu-type.h>
53 * TLB load/store/modify handlers.
132 * CVMSEG starts at address -32768 and extends for in scratchpad_offset()
136 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; in scratchpad_offset()
231 * TLB exception handlers.
263 unsigned int count = (end - start) / sizeof(u32); in dump_handler()
280 /* The only general purpose registers allowed in TLB handlers. */
308 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
345 return -1; in allocate_kscratch()
347 r--; /* make it zero based */ in allocate_kscratch()
413 * The R3000 TLB handler is simple.
442 panic("TLB refill handler space exceeded"); in build_r3000_tlb_refill_handler()
444 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r3000_tlb_refill_handler()
445 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
454 * The R4000 TLB handler is much more complicated. We have two
477 * The software work-around is to not allow the instruction preceding the TLBP
478 * to stall - make it an NOP or some other instruction guaranteed not to stall.
622 panic("No TLB refill handler yet (CPU type: %d)", in build_tlb_write_entry()
643 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
710 /* Set huge page tlb entry size */ in build_huge_tlb_write_entry()
744 * of the large TLB entry size we intend to use. in build_huge_update_entries()
745 * A TLB entry half the size of the configured in build_huge_update_entries()
829 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
836 if (pgd_reg != -1) { in build_get_pmde64()
871 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
873 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
878 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
879 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
885 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
886 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
974 if (pgd_reg != -1) { in build_get_pgde32()
1003 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; in build_adjust_context()
1004 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); in build_adjust_context()
1127 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1138 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_fast_tlb_refill_handler()
1141 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1147 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1159 if (pgd_reg == -1) in build_fast_tlb_refill_handler()
1166 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1182 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1188 * fall-through case = badvaddr *pgd_current in build_fast_tlb_refill_handler()
1194 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1199 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1210 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1211 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1225 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1226 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1299 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1302 * unused TLB refill exception.
1341 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1378 * free instruction slot for the wrap-around branch. In worst in build_r4000_tlb_refill_handler()
1389 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1390 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1392 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1397 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1400 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1401 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1403 tlb_handler + MIPS64_REFILL_INSNS - 3))) in build_r4000_tlb_refill_handler()
1404 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1406 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1409 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1412 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1419 u32 *split; in build_r4000_tlb_refill_handler() local
1426 split = labels[i].addr; in build_r4000_tlb_refill_handler()
1431 if (split > tlb_handler + MIPS64_REFILL_INSNS || in build_r4000_tlb_refill_handler()
1432 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1437 * Split two instructions before the end. One in build_r4000_tlb_refill_handler()
1441 split = tlb_handler + MIPS64_REFILL_INSNS - 2; in build_r4000_tlb_refill_handler()
1448 if (uasm_insn_has_bdelay(relocs, split - 1)) in build_r4000_tlb_refill_handler()
1449 split--; in build_r4000_tlb_refill_handler()
1452 uasm_copy_handler(relocs, labels, tlb_handler, split, f); in build_r4000_tlb_refill_handler()
1453 f += split - tlb_handler; in build_r4000_tlb_refill_handler()
1459 if (uasm_insn_has_bdelay(relocs, split)) in build_r4000_tlb_refill_handler()
1463 split, split + 1, f); in build_r4000_tlb_refill_handler()
1464 uasm_move_labels(labels, f, f + 1, -1); in build_r4000_tlb_refill_handler()
1466 split++; in build_r4000_tlb_refill_handler()
1471 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1472 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + in build_r4000_tlb_refill_handler()
1473 (p - split); in build_r4000_tlb_refill_handler()
1480 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r4000_tlb_refill_handler()
1504 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_ORDER; in setup_pw()
1507 pmd_w = PMD_SHIFT - PAGE_SHIFT; in setup_pw()
1509 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_ORDER; in setup_pw()
1513 pt_w = PAGE_SHIFT - 3; in setup_pw()
1547 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_loongson3_tlb_refill_handler()
1604 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1609 if (pgd_reg == -1) { in build_setup_pgd()
1624 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1653 if (pgd_reg != -1) { in build_setup_pgd()
1667 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1760 * the page table where this PTE is located, PTE will be re-loaded
1782 /* You lose the SMP race :-(*/ in build_pte_present()
1795 /* You lose the SMP race :-(*/ in build_pte_present()
1832 /* You lose the SMP race :-(*/ in build_pte_writable()
1869 /* You lose the SMP race :-(*/ in build_pte_modifiable()
1878 * R3000 style TLB load/store/modify handlers.
1945 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1950 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1952 build_make_valid(&p, &r, K0, K1, -1); in build_r3000_tlb_load_handler()
1960 panic("TLB load handler fastpath space exceeded"); in build_r3000_tlb_load_handler()
1963 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r3000_tlb_load_handler()
1964 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1975 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1980 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1982 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_store_handler()
1990 panic("TLB store handler fastpath space exceeded"); in build_r3000_tlb_store_handler()
1993 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r3000_tlb_store_handler()
1994 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
2005 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
2010 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
2012 build_make_write(&p, &r, K0, K1, -1); in build_r3000_tlb_modify_handler()
2020 panic("TLB modify handler fastpath space exceeded"); in build_r3000_tlb_modify_handler()
2023 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r3000_tlb_modify_handler()
2024 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
2033 * When a Hardware Table Walker is running it can replace TLB entries in cpu_has_tlbex_tlbp_race()
2051 * R4000 style TLB load/store/modify handlers.
2067 * For huge tlb entries, pmd doesn't contain an address but in build_r4000_tlbchange_handler_head()
2068 * instead contains the tlb pte. Check the PAGE_HUGE bit and in build_r4000_tlbchange_handler_head()
2069 * see if we need to jump to huge tlb processing. in build_r4000_tlbchange_handler_head()
2076 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2077 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2122 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2134 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2160 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2236 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2307 panic("TLB load handler fastpath space exceeded"); in build_r4000_tlb_load_handler()
2310 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r4000_tlb_load_handler()
2311 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2323 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2363 panic("TLB store handler fastpath space exceeded"); in build_r4000_tlb_store_handler()
2366 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r4000_tlb_store_handler()
2367 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2379 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2420 panic("TLB modify handler fastpath space exceeded"); in build_r4000_tlb_modify_handler()
2423 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r4000_tlb_modify_handler()
2424 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()
2484 * We are using 2-level page tables, so we only need to in config_htw_params()
2493 /* re-initialize the GDI field */ in config_htw_params()
2496 /* re-initialize the PTI field including the even/odd bit */ in config_htw_params()
2531 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT) in config_htw_params()
2540 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of in config_htw_params()
2592 /* clear all non-PFN bits */ in check_pabits()
2593 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); in check_pabits()
2598 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0); in check_pabits()
2601 fillbits -= min_t(unsigned, fillbits, 2); in check_pabits()
2612 * The refill handler is generated per-CPU, multi-node systems in build_tlb_refill_handler()
2625 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_tlb_refill_handler()
2640 panic("No R3000 TLB refill handler"); in build_tlb_refill_handler()