229 void smp_muxed_ipi_set_message(int cpu, int msg)  in smp_muxed_ipi_set_message()  argument
231 	struct cpu_messages *info = &per_cpu(ipi_message, cpu);  in smp_muxed_ipi_set_message()
241 void smp_muxed_ipi_message_pass(int cpu, int msg)  in smp_muxed_ipi_message_pass()  argument
243 	smp_muxed_ipi_set_message(cpu, msg);  in smp_muxed_ipi_message_pass()
249 	smp_ops->cause_ipi(cpu);  in smp_muxed_ipi_message_pass()
303 static inline void do_message_pass(int cpu, int msg)  in do_message_pass()  argument
306 		smp_ops->message_pass(cpu, msg);  in do_message_pass()
309 		smp_muxed_ipi_message_pass(cpu, msg);  in do_message_pass()
313 void smp_send_reschedule(int cpu)  in smp_send_reschedule()  argument
316 		do_message_pass(cpu, PPC_MSG_RESCHEDULE);  in smp_send_reschedule()
320 void arch_send_call_function_single_ipi(int cpu)  in arch_send_call_function_single_ipi()  argument
322 	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);  in arch_send_call_function_single_ipi()
327 	unsigned int cpu;  in arch_send_call_function_ipi_mask()  local
329 	for_each_cpu(cpu, mask)  in arch_send_call_function_ipi_mask()
330 		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);  in arch_send_call_function_ipi_mask()
434 static void do_smp_send_nmi_ipi(int cpu, bool safe)  in do_smp_send_nmi_ipi()  argument
436 	if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))  in do_smp_send_nmi_ipi()
439 	if (cpu >= 0) {  in do_smp_send_nmi_ipi()
440 		do_message_pass(cpu, PPC_MSG_NMI_IPI);  in do_smp_send_nmi_ipi()
458 int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool safe)  in __smp_send_nmi_ipi()  argument
464 	BUG_ON(cpu == me);  in __smp_send_nmi_ipi()
465 	BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);  in __smp_send_nmi_ipi()
480 	if (cpu < 0) {  in __smp_send_nmi_ipi()
486 		cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);  in __smp_send_nmi_ipi()
491 	do_smp_send_nmi_ipi(cpu, safe);  in __smp_send_nmi_ipi()
534 int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)  in smp_send_nmi_ipi()  argument
536 	return __smp_send_nmi_ipi(cpu, fn, delay_us, false);  in smp_send_nmi_ipi()
539 int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)  in smp_send_safe_nmi_ipi()  argument
541 	return __smp_send_nmi_ipi(cpu, fn, delay_us, true);  in smp_send_safe_nmi_ipi()
548 	unsigned int cpu;  in tick_broadcast()  local
550 	for_each_cpu(cpu, mask)  in tick_broadcast()
551 		do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);  in tick_broadcast()
570 	int cpu;  in crash_send_ipi()  local
574 		for_each_present_cpu(cpu) {  in crash_send_ipi()
575 			if (cpu_online(cpu))  in crash_send_ipi()
586 			do_smp_send_nmi_ipi(cpu, false);  in crash_send_ipi()
679 	unsigned int cpu;  in smp_prepare_cpus()  local
693 	for_each_possible_cpu(cpu) {  in smp_prepare_cpus()
694 		zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),  in smp_prepare_cpus()
695 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
696 		zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu),  in smp_prepare_cpus()
697 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
698 		zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),  in smp_prepare_cpus()
699 					GFP_KERNEL, cpu_to_node(cpu));  in smp_prepare_cpus()
703 		if (cpu_present(cpu)) {  in smp_prepare_cpus()
704 			set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);  in smp_prepare_cpus()
705 			set_cpu_numa_mem(cpu,  in smp_prepare_cpus()
706 				local_memory_node(numa_cpu_lookup_table[cpu]));  in smp_prepare_cpus()
733 	unsigned int cpu = smp_processor_id();  in generic_cpu_disable()  local
735 	if (cpu == boot_cpuid)  in generic_cpu_disable()
738 	set_cpu_online(cpu, false);  in generic_cpu_disable()
760 void generic_cpu_die(unsigned int cpu)  in generic_cpu_die()  argument
766 		if (is_cpu_dead(cpu))  in generic_cpu_die()
770 	printk(KERN_ERR "CPU%d didn't die...\n", cpu);  in generic_cpu_die()
773 void generic_set_cpu_dead(unsigned int cpu)  in generic_set_cpu_dead()  argument
775 	per_cpu(cpu_state, cpu) = CPU_DEAD;  in generic_set_cpu_dead()
783 void generic_set_cpu_up(unsigned int cpu)  in generic_set_cpu_up()  argument
785 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;  in generic_set_cpu_up()
788 int generic_check_cpu_restart(unsigned int cpu)  in generic_check_cpu_restart()  argument
790 	return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;  in generic_check_cpu_restart()
793 int is_cpu_dead(unsigned int cpu)  in is_cpu_dead()  argument
795 	return per_cpu(cpu_state, cpu) == CPU_DEAD;  in is_cpu_dead()
809 static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)  in cpu_idle_thread_init()  argument
814 	paca_ptrs[cpu]->__current = idle;  in cpu_idle_thread_init()
815 	paca_ptrs[cpu]->kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;  in cpu_idle_thread_init()
817 	ti->cpu = cpu;  in cpu_idle_thread_init()
818 	secondary_ti = current_set[cpu] = ti;  in cpu_idle_thread_init()
821 int __cpu_up(unsigned int cpu, struct task_struct *tidle)  in __cpu_up()  argument
829 	    cpu_thread_in_subcore(cpu))  in __cpu_up()
833 	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))  in __cpu_up()
836 	cpu_idle_thread_init(cpu, tidle);  in __cpu_up()
843 		rc = smp_ops->prepare_cpu(cpu);  in __cpu_up()
851 	cpu_callin_map[cpu] = 0;  in __cpu_up()
860 	DBG("smp: kicking cpu %d\n", cpu);  in __cpu_up()
861 	rc = smp_ops->kick_cpu(cpu);  in __cpu_up()
863 		pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);  in __cpu_up()
873 		for (c = 50000; c && !cpu_callin_map[cpu]; c--)  in __cpu_up()
881 		for (c = 5000; c && !cpu_callin_map[cpu]; c--)  in __cpu_up()
885 	if (!cpu_callin_map[cpu]) {  in __cpu_up()
886 		printk(KERN_ERR "Processor %u is stuck.\n", cpu);  in __cpu_up()
890 	DBG("Processor %u found.\n", cpu);  in __cpu_up()
896 	spin_until_cond(cpu_online(cpu));  in __cpu_up()
904 int cpu_to_core_id(int cpu)  in cpu_to_core_id()  argument
910 	np = of_get_cpu_node(cpu, NULL);  in cpu_to_core_id()
926 int cpu_core_index_of_thread(int cpu)  in cpu_core_index_of_thread()  argument
928 	return cpu >> threads_shift;  in cpu_core_index_of_thread()
941 static struct device_node *cpu_to_l2cache(int cpu)  in cpu_to_l2cache()  argument
946 	if (!cpu_present(cpu))  in cpu_to_l2cache()
949 	np = of_get_cpu_node(cpu, NULL);  in cpu_to_l2cache()
960 static bool update_mask_by_l2(int cpu, struct cpumask *(*mask_fn)(int))  in update_mask_by_l2()  argument
965 	l2_cache = cpu_to_l2cache(cpu);  in update_mask_by_l2()
979 			set_cpus_related(cpu, i, mask_fn);  in update_mask_by_l2()
989 static void remove_cpu_from_masks(int cpu)  in remove_cpu_from_masks()  argument
994 	for_each_cpu(i, cpu_core_mask(cpu)) {  in remove_cpu_from_masks()
995 		set_cpus_unrelated(cpu, i, cpu_core_mask);  in remove_cpu_from_masks()
996 		set_cpus_unrelated(cpu, i, cpu_l2_cache_mask);  in remove_cpu_from_masks()
997 		set_cpus_unrelated(cpu, i, cpu_sibling_mask);  in remove_cpu_from_masks()
1002 static void add_cpu_to_masks(int cpu)  in add_cpu_to_masks()  argument
1004 	int first_thread = cpu_first_thread_sibling(cpu);  in add_cpu_to_masks()
1005 	int chipid = cpu_to_chip_id(cpu);  in add_cpu_to_masks()
1012 	cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));  in add_cpu_to_masks()
1016 			set_cpus_related(i, cpu, cpu_sibling_mask);  in add_cpu_to_masks()
1022 	for_each_cpu(i, cpu_sibling_mask(cpu))  in add_cpu_to_masks()
1023 		set_cpus_related(cpu, i, cpu_l2_cache_mask);  in add_cpu_to_masks()
1024 	update_mask_by_l2(cpu, cpu_l2_cache_mask);  in add_cpu_to_masks()
1030 	for_each_cpu(i, cpu_l2_cache_mask(cpu))  in add_cpu_to_masks()
1031 		set_cpus_related(cpu, i, cpu_core_mask);  in add_cpu_to_masks()
1038 			set_cpus_related(cpu, i, cpu_core_mask);  in add_cpu_to_masks()
1046 	unsigned int cpu = smp_processor_id();  in start_secondary()  local
1051 	smp_store_cpu_info(cpu);  in start_secondary()
1054 	cpu_callin_map[cpu] = 1;  in start_secondary()
1057 		smp_ops->setup_cpu(cpu);  in start_secondary()
1070 	add_cpu_to_masks(cpu);  in start_secondary()
1076 	if (!cpumask_equal(cpu_l2_cache_mask(cpu), cpu_sibling_mask(cpu)))  in start_secondary()
1079 	set_numa_node(numa_cpu_lookup_table[cpu]);  in start_secondary()
1080 	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));  in start_secondary()
1083 	notify_cpu_starting(cpu);  in start_secondary()
1084 	set_cpu_online(cpu, true);  in start_secondary()
1138 static const struct cpumask *shared_cache_mask(int cpu)  in shared_cache_mask()  argument
1140 	return cpu_l2_cache_mask(cpu);  in shared_cache_mask()
1186 	int cpu = smp_processor_id();  in __cpu_disable()  local
1199 	remove_cpu_from_masks(cpu);  in __cpu_disable()
1204 void __cpu_die(unsigned int cpu)  in __cpu_die()  argument
1207 		smp_ops->cpu_die(cpu);  in __cpu_die()