1 // SPDX-License-Identifier: GPL-2.0-or-later
61 #include <asm/code-patching.h>
66 #include <asm/feature-fixups.h>
99 		 * If we boot via kdump on a non-primary thread,  in setup_tlb_core_data()
106 		paca_ptrs[cpu]->tcd_ptr = &paca_ptrs[first]->tcd;  in setup_tlb_core_data()
124 static char *smt_enabled_cmdline;
126 /* Look for ibm,smt-enabled OF option */
153 			smt_option = of_get_property(dn, "ibm,smt-enabled",  in check_smt_enabled()
168 /* Look for smt-enabled= cmdline option */
169 static int __init early_smt_enabled(char *p)  in early_smt_enabled()
174 early_param("smt-enabled", early_smt_enabled);
179 static void __init fixup_boot_paca(void)  in fixup_boot_paca()
182 	get_paca()->cpu_start = 1;  in fixup_boot_paca()
184 	get_paca()->data_offset = 0;  in fixup_boot_paca()
189 static void __init configure_exceptions(void)  in configure_exceptions()
199 		/* Enable AIL if possible */  in configure_exceptions()
202 			cur_cpu_spec->cpu_user_features2 &= ~PPC_FEATURE2_SCV;  in configure_exceptions()
225 static void cpu_ready_for_interrupts(void)  in cpu_ready_for_interrupts()
228 	 * Enable AIL if supported, and we are in hypervisor mode. This  in cpu_ready_for_interrupts()
243 	 * told TM is off via the dt-ftrs but told to (partially) use  in cpu_ready_for_interrupts()
245 	 * will be off from dt-ftrs but we need to turn it on for the  in cpu_ready_for_interrupts()
256 	get_paca()->kernel_msr = MSR_KERNEL;  in cpu_ready_for_interrupts()
273  * some early parsing of the device-tree to setup out MEMBLOCK
283  * device-tree is not accessible via normal means at this point.
288 	static __initdata struct paca_struct boot_paca;  in early_setup()
290 	/* -------- printk is _NOT_ safe to use here ! ------- */  in early_setup()
315 	/* -------- printk is now safe to use ------- */  in early_setup()
322 	/* Enable early debugging if any specified (see udbg.h) */  in early_setup()
325 	udbg_printf(" -> %s(), dt_ptr: 0x%lx\n", __func__, dt_ptr);  in early_setup()
373 	 * have IR and DR set and enable AIL if it exists  in early_setup()
378 	 * We enable ftrace here, but since we only support DYNAMIC_FTRACE, it  in early_setup()
384 	udbg_printf(" <- %s()\n", __func__);  in early_setup()
391 	 * which means the real-mode access trick that btext does will  in early_setup()
408 	/* Perform any KUP setup that is per-cpu */  in early_setup_secondary()
430 static bool use_spinloop(void)  in use_spinloop()
434 		 * See comments in head_64.S -- not all platforms insert  in use_spinloop()
447 	return of_property_read_bool(of_chosen, "linux,booted-from-kexec");  in use_spinloop()
465 			- PHYSICAL_START);  in smp_release_cpus()
488 static void init_cache_info(struct ppc_cache_info *info, u32 size, u32 lsize,  in init_cache_info()
491 	info->size = size;  in init_cache_info()
492 	info->sets = sets;  in init_cache_info()
493 	info->line_size = lsize;  in init_cache_info()
494 	info->block_size = bsize;  in init_cache_info()
495 	info->log_block_size = __ilog2(bsize);  in init_cache_info()
497 		info->blocks_per_page = PAGE_SIZE / bsize;  in init_cache_info()
499 		info->blocks_per_page = 0;  in init_cache_info()
502 		info->assoc = 0xffff;  in init_cache_info()
504 		info->assoc = size / (sets * lsize);  in init_cache_info()
507 static bool __init parse_cache_info(struct device_node *np,  in parse_cache_info()
511 	static const char *ipropnames[] __initdata = {  in parse_cache_info()
512 		"i-cache-size",  in parse_cache_info()
513 		"i-cache-sets",  in parse_cache_info()
514 		"i-cache-block-size",  in parse_cache_info()
515 		"i-cache-line-size",  in parse_cache_info()
517 	static const char *dpropnames[] __initdata = {  in parse_cache_info()
518 		"d-cache-size",  in parse_cache_info()
519 		"d-cache-sets",  in parse_cache_info()
520 		"d-cache-block-size",  in parse_cache_info()
521 		"d-cache-line-size",  in parse_cache_info()
529 	sets = -1u;  in parse_cache_info()
530 	lsize = bsize = cur_cpu_spec->dcache_bsize;  in parse_cache_info()
573 	 * puts incorrect information in the device-tree. This will  in initialize_cache_info()
590 	 * d-cache and i-cache sizes... -Peter  in initialize_cache_info()
601 		 * unified and use the D-side properties.  in initialize_cache_info()
620 	cur_cpu_spec->dcache_bsize = dcache_bsize;  in initialize_cache_info()
621 	cur_cpu_spec->icache_bsize = icache_bsize;  in initialize_cache_info()
631  * re-entrant interrupts.
654 static void *__init alloc_stack(unsigned long limit, int cpu)  in alloc_stack()
695 		paca_ptrs[i]->crit_kstack = sp + THREAD_SIZE;  in exc_lvl_early_init()
699 		paca_ptrs[i]->dbg_kstack = sp + THREAD_SIZE;  in exc_lvl_early_init()
703 		paca_ptrs[i]->mc_kstack = sp + THREAD_SIZE;  in exc_lvl_early_init()
723 	 * SLB misses on them. The ABI also requires them to be 128-byte  in emergency_stack_init()
738 	 * Machine check on pseries calls rtas, but can't use the static  in emergency_stack_init()
747 		paca_ptrs[i]->emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;  in emergency_stack_init()
751 		paca_ptrs[i]->nmi_emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;  in emergency_stack_init()
754 		paca_ptrs[i]->mc_emergency_sp = alloc_stack(mce_limit, i) + THREAD_SIZE;  in emergency_stack_init()
761  * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
773 static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size,  in pcpu_alloc_bootmem()
783 		pr_info("cpu %d has no node %d or node-local memory\n",  in pcpu_alloc_bootmem()
799 static void __init pcpu_free_bootmem(void *ptr, size_t size)  in pcpu_free_bootmem()
804 static int pcpu_cpu_distance(unsigned int from, unsigned int to)  in pcpu_cpu_distance()
815 static void __init pcpu_populate_pte(unsigned long addr)  in pcpu_populate_pte()
866 	int rc = -EINVAL;  in setup_per_cpu_areas()
893 	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;  in setup_per_cpu_areas()
896 		paca_ptrs[cpu]->data_offset = __per_cpu_offset[cpu];  in setup_per_cpu_areas()
925  * disable it by default. Book3S has a soft-nmi hardlockup detector based
931 static int __init disable_hardlockup_detector(void)  in disable_hardlockup_detector()
945 static enum l1d_flush_type enabled_flush_types;
946 static void *l1d_flush_fallback_area;
947 static bool no_rfi_flush;
948 static bool no_entry_flush;
949 static bool no_uaccess_flush;
956 static int __init handle_no_rfi_flush(char *p)  in handle_no_rfi_flush()
958 	pr_info("rfi-flush: disabled on command line.");  in handle_no_rfi_flush()
964 static int __init handle_no_entry_flush(char *p)  in handle_no_entry_flush()
966 	pr_info("entry-flush: disabled on command line.");  in handle_no_entry_flush()
972 static int __init handle_no_uaccess_flush(char *p)  in handle_no_uaccess_flush()
974 	pr_info("uaccess-flush: disabled on command line.");  in handle_no_uaccess_flush()
984 static int __init handle_no_pti(char *p)  in handle_no_pti()
986 	pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");  in handle_no_pti()
992 static void do_nothing(void *unused)  in do_nothing()
1000 void rfi_flush_enable(bool enable)  in rfi_flush_enable()  argument
1002 	if (enable) {  in rfi_flush_enable()
1008 	rfi_flush = enable;  in rfi_flush_enable()
1011 void entry_flush_enable(bool enable)  in entry_flush_enable()  argument
1013 	if (enable) {  in entry_flush_enable()
1020 	entry_flush = enable;  in entry_flush_enable()
1023 void uaccess_flush_enable(bool enable)  in uaccess_flush_enable()  argument
1025 	if (enable) {  in uaccess_flush_enable()
1034 	uaccess_flush = enable;  in uaccess_flush_enable()
1037 static void __ref init_fallback_flush(void)  in init_fallback_flush()
1049 	 * If there is no d-cache-size property in the device tree, l1d_size  in init_fallback_flush()
1051 	 * 2^64-1, and then walking off the end of the fallback area and  in init_fallback_flush()
1075 		paca->rfi_flush_fallback_area = l1d_flush_fallback_area;  in init_fallback_flush()
1076 		paca->l1d_flush_size = l1d_size;  in init_fallback_flush()
1080 void setup_rfi_flush(enum l1d_flush_type types, bool enable)  in setup_rfi_flush()  argument
1083 		pr_info("rfi-flush: fallback displacement flush available\n");  in setup_rfi_flush()
1088 		pr_info("rfi-flush: ori type flush available\n");  in setup_rfi_flush()
1091 		pr_info("rfi-flush: mttrig type flush available\n");  in setup_rfi_flush()
1096 		rfi_flush_enable(enable);  in setup_rfi_flush()
1099 void setup_entry_flush(bool enable)  in setup_entry_flush()  argument
1105 		entry_flush_enable(enable);  in setup_entry_flush()
1108 void setup_uaccess_flush(bool enable)  in setup_uaccess_flush()  argument
1114 		uaccess_flush_enable(enable);  in setup_uaccess_flush()
1118 static int rfi_flush_set(void *data, u64 val)  in rfi_flush_set()
1120 	bool enable;  in rfi_flush_set()  local
1123 		enable = true;  in rfi_flush_set()
1125 		enable = false;  in rfi_flush_set()
1127 		return -EINVAL;  in rfi_flush_set()
1130 	if (enable != rfi_flush)  in rfi_flush_set()
1131 		rfi_flush_enable(enable);  in rfi_flush_set()
1136 static int rfi_flush_get(void *data, u64 *val)  in rfi_flush_get()
1144 static int entry_flush_set(void *data, u64 val)  in entry_flush_set()
1146 	bool enable;  in entry_flush_set()  local
1149 		enable = true;  in entry_flush_set()
1151 		enable = false;  in entry_flush_set()
1153 		return -EINVAL;  in entry_flush_set()
1156 	if (enable != entry_flush)  in entry_flush_set()
1157 		entry_flush_enable(enable);  in entry_flush_set()
1162 static int entry_flush_get(void *data, u64 *val)  in entry_flush_get()
1170 static int uaccess_flush_set(void *data, u64 val)  in uaccess_flush_set()
1172 	bool enable;  in uaccess_flush_set()  local
1175 		enable = true;  in uaccess_flush_set()
1177 		enable = false;  in uaccess_flush_set()
1179 		return -EINVAL;  in uaccess_flush_set()
1182 	if (enable != uaccess_flush)  in uaccess_flush_set()
1183 		uaccess_flush_enable(enable);  in uaccess_flush_set()
1188 static int uaccess_flush_get(void *data, u64 *val)  in uaccess_flush_get()
1196 static __init int rfi_flush_debugfs_init(void)  in rfi_flush_debugfs_init()