29 #include <linux/cpu.h>
71 /* State of each CPU during hotplug phases */
98  * On big-cores system, cpu_l1_cache_map for each CPU corresponds to
112  * Returns 1 if the specified cpu should be brought up during boot.
246 void smp_muxed_ipi_set_message(int cpu, int msg)  in smp_muxed_ipi_set_message()  argument
248 	struct cpu_messages *info = &per_cpu(ipi_message, cpu);  in smp_muxed_ipi_set_message()
258 void smp_muxed_ipi_message_pass(int cpu, int msg)  in smp_muxed_ipi_message_pass()  argument
260 	smp_muxed_ipi_set_message(cpu, msg);  in smp_muxed_ipi_message_pass()
266 	smp_ops->cause_ipi(cpu);  in smp_muxed_ipi_message_pass()
320 static inline void do_message_pass(int cpu, int msg)  in do_message_pass()  argument
323 		smp_ops->message_pass(cpu, msg);  in do_message_pass()
326 		smp_muxed_ipi_message_pass(cpu, msg);  in do_message_pass()
330 void smp_send_reschedule(int cpu)  in smp_send_reschedule()  argument
333 		do_message_pass(cpu, PPC_MSG_RESCHEDULE);  in smp_send_reschedule()
337 void arch_send_call_function_single_ipi(int cpu)  in arch_send_call_function_single_ipi()  argument
339 	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);  in arch_send_call_function_single_ipi()
344 	unsigned int cpu;  in arch_send_call_function_ipi_mask()  local
346 	for_each_cpu(cpu, mask)  in arch_send_call_function_ipi_mask()
347 		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);  in arch_send_call_function_ipi_mask()
438 static void do_smp_send_nmi_ipi(int cpu, bool safe)  in do_smp_send_nmi_ipi()  argument
440 	if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))  in do_smp_send_nmi_ipi()
443 	if (cpu >= 0) {  in do_smp_send_nmi_ipi()
444 		do_message_pass(cpu, PPC_MSG_NMI_IPI);  in do_smp_send_nmi_ipi()
457  * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
462 static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *),  in __smp_send_nmi_ipi()  argument
469 	BUG_ON(cpu == me);  in __smp_send_nmi_ipi()
470 	BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);  in __smp_send_nmi_ipi()
486 	if (cpu < 0) {  in __smp_send_nmi_ipi()
491 		cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);  in __smp_send_nmi_ipi()
498 	do_smp_send_nmi_ipi(cpu, safe);  in __smp_send_nmi_ipi()
527 int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)  in smp_send_nmi_ipi()  argument
529 	return __smp_send_nmi_ipi(cpu, fn, delay_us, false);  in smp_send_nmi_ipi()
532 int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)  in smp_send_safe_nmi_ipi()  argument
534 	return __smp_send_nmi_ipi(cpu, fn, delay_us, true);  in smp_send_safe_nmi_ipi()
541 	unsigned int cpu;  in tick_broadcast()  local
543 	for_each_cpu(cpu, mask)  in tick_broadcast()
544 		do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);  in tick_broadcast()
563 	int cpu;  in crash_send_ipi()  local
567 		for_each_present_cpu(cpu) {  in crash_send_ipi()
568 			if (cpu_online(cpu))  in crash_send_ipi()
579 			do_smp_send_nmi_ipi(cpu, false);  in crash_send_ipi()
642  * Relationships between CPUs are maintained in a set of per-cpu cpumasks so
644  * returns the that cpumask for the given CPU.
663  *                      property for the CPU device node @dn and stores
668  * @dn: The device node of the CPU device.
675  * the CPU-device node can be grouped together based on the property.
743  *                              that @cpu belongs to.
745  * @cpu : The logical CPU whose thread group is being searched.
746  * @tg : The thread-group structure of the CPU node which @cpu belongs
750  * of the thread_group that @cpu belongs to.
752  * Returns -1 if cpu doesn't belong to any of the groups pointed to by
755 static int get_cpu_thread_group_start(int cpu, struct thread_groups *tg)  in get_cpu_thread_group_start()  argument
757 	int hw_cpu_id = get_hard_smp_processor_id(cpu);  in get_cpu_thread_group_start()
774 static int init_cpu_l1_cache_map(int cpu)  in init_cpu_l1_cache_map()  argument
777 	struct device_node *dn = of_get_cpu_node(cpu, NULL);  in init_cpu_l1_cache_map()
781 	int first_thread = cpu_first_thread_sibling(cpu);  in init_cpu_l1_cache_map()
791 	zalloc_cpumask_var_node(&per_cpu(cpu_l1_cache_map, cpu),  in init_cpu_l1_cache_map()
793 				cpu_to_node(cpu));  in init_cpu_l1_cache_map()
795 	cpu_group_start = get_cpu_thread_group_start(cpu, &tg);  in init_cpu_l1_cache_map()
813 			cpumask_set_cpu(i, per_cpu(cpu_l1_cache_map, cpu));  in init_cpu_l1_cache_map()
823 	int cpu;  in init_big_cores()  local
825 	for_each_possible_cpu(cpu) {  in init_big_cores()
826 		int err = init_cpu_l1_cache_map(cpu);  in init_big_cores()
831 		zalloc_cpumask_var_node(&per_cpu(cpu_smallcore_map, cpu),  in init_big_cores()
833 					cpu_to_node(cpu));  in init_big_cores()
842 	unsigned int cpu;  in smp_prepare_cpus()  local
847 	 * setup_cpu may need to be called on the boot cpu. We havent  in smp_prepare_cpus()
852 	/* Fixup boot cpu */  in smp_prepare_cpus()
856 	for_each_possible_cpu(cpu) {  in smp_prepare_cpus()
857 		zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),  in smp_prepare_cpus()
858 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
859 		zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu),  in smp_prepare_cpus()
860 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
861 		zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),  in smp_prepare_cpus()
862 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
866 		if (cpu_present(cpu)) {  in smp_prepare_cpus()
867 			set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);  in smp_prepare_cpus()
868 			set_cpu_numa_mem(cpu,  in smp_prepare_cpus()
869 				local_memory_node(numa_cpu_lookup_table[cpu]));  in smp_prepare_cpus()
873 	/* Init the cpumasks so the boot CPU is related to itself */  in smp_prepare_cpus()
902 	unsigned int cpu = smp_processor_id();  in generic_cpu_disable()  local
904 	if (cpu == boot_cpuid)  in generic_cpu_disable()
907 	set_cpu_online(cpu, false);  in generic_cpu_disable()
911 	/* Update affinity of all IRQs previously aimed at this CPU */  in generic_cpu_disable()
916 	 * that one of the interrupts we just migrated away from this CPU is  in generic_cpu_disable()
917 	 * actually already pending on this CPU. If we leave it in that state  in generic_cpu_disable()
920 	 * be received (and EOI'ed), before we take this CPU offline.  in generic_cpu_disable()
929 void generic_cpu_die(unsigned int cpu)  in generic_cpu_die()  argument
935 		if (is_cpu_dead(cpu))  in generic_cpu_die()
939 	printk(KERN_ERR "CPU%d didn't die...\n", cpu);  in generic_cpu_die()
942 void generic_set_cpu_dead(unsigned int cpu)  in generic_set_cpu_dead()  argument
944 	per_cpu(cpu_state, cpu) = CPU_DEAD;  in generic_set_cpu_dead()
952 void generic_set_cpu_up(unsigned int cpu)  in generic_set_cpu_up()  argument
954 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;  in generic_set_cpu_up()
957 int generic_check_cpu_restart(unsigned int cpu)  in generic_check_cpu_restart()  argument
959 	return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;  in generic_check_cpu_restart()
962 int is_cpu_dead(unsigned int cpu)  in is_cpu_dead()  argument
964 	return per_cpu(cpu_state, cpu) == CPU_DEAD;  in is_cpu_dead()
978 static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)  in cpu_idle_thread_init()  argument
981 	paca_ptrs[cpu]->__current = idle;  in cpu_idle_thread_init()
982 	paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) +  in cpu_idle_thread_init()
985 	idle->cpu = cpu;  in cpu_idle_thread_init()
986 	secondary_current = current_set[cpu] = idle;  in cpu_idle_thread_init()
989 int __cpu_up(unsigned int cpu, struct task_struct *tidle)  in __cpu_up()  argument
997 	    cpu_thread_in_subcore(cpu))  in __cpu_up()
1001 	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))  in __cpu_up()
1004 	cpu_idle_thread_init(cpu, tidle);  in __cpu_up()
1008 	 * up the CPU  in __cpu_up()
1011 		rc = smp_ops->prepare_cpu(cpu);  in __cpu_up()
1016 	/* Make sure callin-map entry is 0 (can be leftover a CPU  in __cpu_up()
1019 	cpu_callin_map[cpu] = 0;  in __cpu_up()
1028 	DBG("smp: kicking cpu %d\n", cpu);  in __cpu_up()
1029 	rc = smp_ops->kick_cpu(cpu);  in __cpu_up()
1031 		pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);  in __cpu_up()
1036 	 * wait to see if the cpu made a callin (is actually up).  in __cpu_up()
1041 		for (c = 50000; c && !cpu_callin_map[cpu]; c--)  in __cpu_up()
1049 		for (c = 5000; c && !cpu_callin_map[cpu]; c--)  in __cpu_up()
1053 	if (!cpu_callin_map[cpu]) {  in __cpu_up()
1054 		printk(KERN_ERR "Processor %u is stuck.\n", cpu);  in __cpu_up()
1058 	DBG("Processor %u found.\n", cpu);  in __cpu_up()
1063 	/* Wait until cpu puts itself in the online & active maps */  in __cpu_up()
1064 	spin_until_cond(cpu_online(cpu));  in __cpu_up()
1070  * logical cpu.
1072 int cpu_to_core_id(int cpu)  in cpu_to_core_id()  argument
1078 	np = of_get_cpu_node(cpu, NULL);  in cpu_to_core_id()
1093 /* Helper routines for cpu to core mapping */
1094 int cpu_core_index_of_thread(int cpu)  in cpu_core_index_of_thread()  argument
1096 	return cpu >> threads_shift;  in cpu_core_index_of_thread()
1107  * i.e. during cpu online or offline.
1109 static struct device_node *cpu_to_l2cache(int cpu)  in cpu_to_l2cache()  argument
1114 	if (!cpu_present(cpu))  in cpu_to_l2cache()
1117 	np = of_get_cpu_node(cpu, NULL);  in cpu_to_l2cache()
1128 static bool update_mask_by_l2(int cpu, struct cpumask *(*mask_fn)(int))  in update_mask_by_l2()  argument
1133 	l2_cache = cpu_to_l2cache(cpu);  in update_mask_by_l2()
1139 		 * when updating the marks the current CPU has not been marked  in update_mask_by_l2()
1147 			set_cpus_related(cpu, i, mask_fn);  in update_mask_by_l2()
1157 static void remove_cpu_from_masks(int cpu)  in remove_cpu_from_masks()  argument
1162 	for_each_cpu(i, cpu_core_mask(cpu)) {  in remove_cpu_from_masks()
1163 		set_cpus_unrelated(cpu, i, cpu_core_mask);  in remove_cpu_from_masks()
1164 		set_cpus_unrelated(cpu, i, cpu_l2_cache_mask);  in remove_cpu_from_masks()
1165 		set_cpus_unrelated(cpu, i, cpu_sibling_mask);  in remove_cpu_from_masks()
1167 			set_cpus_unrelated(cpu, i, cpu_smallcore_mask);  in remove_cpu_from_masks()
1172 static inline void add_cpu_to_smallcore_masks(int cpu)  in add_cpu_to_smallcore_masks()  argument
1174 	struct cpumask *this_l1_cache_map = per_cpu(cpu_l1_cache_map, cpu);  in add_cpu_to_smallcore_masks()
1175 	int i, first_thread = cpu_first_thread_sibling(cpu);  in add_cpu_to_smallcore_masks()
1180 	cpumask_set_cpu(cpu, cpu_smallcore_mask(cpu));  in add_cpu_to_smallcore_masks()
1184 			set_cpus_related(i, cpu, cpu_smallcore_mask);  in add_cpu_to_smallcore_masks()
1188 static void add_cpu_to_masks(int cpu)  in add_cpu_to_masks()  argument
1190 	int first_thread = cpu_first_thread_sibling(cpu);  in add_cpu_to_masks()
1191 	int chipid = cpu_to_chip_id(cpu);  in add_cpu_to_masks()
1195 	 * This CPU will not be in the online mask yet so we need to manually  in add_cpu_to_masks()
1198 	cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));  in add_cpu_to_masks()
1202 			set_cpus_related(i, cpu, cpu_sibling_mask);  in add_cpu_to_masks()
1204 	add_cpu_to_smallcore_masks(cpu);  in add_cpu_to_masks()
1207 	 * and mark any CPUs that share an L2 with this CPU.  in add_cpu_to_masks()
1209 	for_each_cpu(i, cpu_sibling_mask(cpu))  in add_cpu_to_masks()
1210 		set_cpus_related(cpu, i, cpu_l2_cache_mask);  in add_cpu_to_masks()
1211 	update_mask_by_l2(cpu, cpu_l2_cache_mask);  in add_cpu_to_masks()
1215 	 * any CPUs on the same chip as this CPU.  in add_cpu_to_masks()
1217 	for_each_cpu(i, cpu_l2_cache_mask(cpu))  in add_cpu_to_masks()
1218 		set_cpus_related(cpu, i, cpu_core_mask);  in add_cpu_to_masks()
1225 			set_cpus_related(cpu, i, cpu_core_mask);  in add_cpu_to_masks()
1233 	unsigned int cpu = smp_processor_id();  in start_secondary()  local
1239 	smp_store_cpu_info(cpu);  in start_secondary()
1242 	cpu_callin_map[cpu] = 1;  in start_secondary()
1245 		smp_ops->setup_cpu(cpu);  in start_secondary()
1257 	/* Update topology CPU masks */  in start_secondary()
1258 	add_cpu_to_masks(cpu);  in start_secondary()
1266 	if (!cpumask_equal(cpu_l2_cache_mask(cpu), sibling_mask(cpu)))  in start_secondary()
1269 	set_numa_node(numa_cpu_lookup_table[cpu]);  in start_secondary()
1270 	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));  in start_secondary()
1273 	notify_cpu_starting(cpu);  in start_secondary()
1274 	set_cpu_online(cpu, true);  in start_secondary()
1330 static const struct cpumask *shared_cache_mask(int cpu)  in shared_cache_mask()  argument
1332 	return cpu_l2_cache_mask(cpu);  in shared_cache_mask()
1336 static const struct cpumask *smallcore_smt_mask(int cpu)  in smallcore_smt_mask()  argument
1338 	return cpu_smallcore_mask(cpu);  in smallcore_smt_mask()
1354 	 * We are running pinned to the boot CPU, see rest_init().  in smp_cpus_done()
1364 	 * Hence, get numa topology before dumping cpu topology  in smp_cpus_done()
1377 	 * If any CPU detects that it's sharing a cache with another CPU then  in smp_cpus_done()
1392 	int cpu = smp_processor_id();  in __cpu_disable()  local
1405 	remove_cpu_from_masks(cpu);  in __cpu_disable()
1410 void __cpu_die(unsigned int cpu)  in __cpu_die()  argument
1413 		smp_ops->cpu_die(cpu);  in __cpu_die()