36 #include <linux/percpu-rwsem.h>
48  * struct cpuhp_cpu_state - Per cpu hotplug state storage
58  * @node:	Remote CPU node; for multi-instance, do a
60  * @last:	For multi-instance rollback, remember how far we got
64  * @done_up:	Signal completion to the issuer of the task for cpu-up
65  * @done_down:	Signal completion to the issuer of the task for cpu-down
97 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
99 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
119  * struct cpuhp_step - Hotplug state machine step
130 		int		(*multi)(unsigned int cpu,  member
135 		int		(*multi)(unsigned int cpu,  member
155 	return bringup ? !step->startup.single : !step->teardown.single;  in cpuhp_step_empty()
159  * cpuhp_invoke_callback - Invoke the callbacks for a given state
163  * @node:	For multi-instance, do a single entry callback for install/remove
164  * @lastp:	For multi-instance rollback, remember how far we got
180 	if (st->fail == state) {  in cpuhp_invoke_callback()
181 		st->fail = CPUHP_INVALID;  in cpuhp_invoke_callback()
182 		return -EAGAIN;  in cpuhp_invoke_callback()
190 	if (!step->multi_instance) {  in cpuhp_invoke_callback()
192 		cb = bringup ? step->startup.single : step->teardown.single;  in cpuhp_invoke_callback()
194 		trace_cpuhp_enter(cpu, st->target, state, cb);  in cpuhp_invoke_callback()
196 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
199 	cbm = bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
204 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
206 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
212 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
216 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
218 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
233 	cbm = !bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
237 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
238 		if (!cnt--)  in cpuhp_invoke_callback()
241 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
243 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
264 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in wait_for_ap_thread()
270 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in complete_ap_thread()
294  * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
330 		} else if (now - start < NSEC_PER_MSEC) {  in cpuhp_wait_for_sync_state()
346  * cpuhp_ap_report_dead - Update synchronization state to DEAD
387  * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
446 		ret = -EIO;  in cpuhp_bp_sync_alive()
478  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
565 	cpu_hotplug_disabled--;  in __cpu_hotplug_enable()
589  * Architectures that need SMT-specific errata handling during SMT hotplug
694 	enum cpuhp_state prev_state = st->state;  in cpuhp_set_state()
695 	bool bringup = st->state < target;  in cpuhp_set_state()
697 	st->rollback = false;  in cpuhp_set_state()
698 	st->last = NULL;  in cpuhp_set_state()
700 	st->target = target;  in cpuhp_set_state()
701 	st->single = false;  in cpuhp_set_state()
702 	st->bringup = bringup;  in cpuhp_set_state()
713 	bool bringup = !st->bringup;  in cpuhp_reset_state()
715 	st->target = prev_state;  in cpuhp_reset_state()
721 	if (st->rollback)  in cpuhp_reset_state()
724 	st->rollback = true;  in cpuhp_reset_state()
727 	 * If we have st->last we need to undo partial multi_instance of this  in cpuhp_reset_state()
730 	if (!st->last) {  in cpuhp_reset_state()
731 		if (st->bringup)  in cpuhp_reset_state()
732 			st->state--;  in cpuhp_reset_state()
734 			st->state++;  in cpuhp_reset_state()
737 	st->bringup = bringup;  in cpuhp_reset_state()
745 	if (!st->single && st->state == st->target)  in __cpuhp_kick_ap()
748 	st->result = 0;  in __cpuhp_kick_ap()
754 	st->should_run = true;  in __cpuhp_kick_ap()
755 	wake_up_process(st->thread);  in __cpuhp_kick_ap()
756 	wait_for_ap_thread(st, st->bringup);  in __cpuhp_kick_ap()
767 	if ((ret = st->result)) {  in cpuhp_kick_ap()
782 		return -ECANCELED;  in bringup_wait_for_ap_online()
785 	kthread_unpark(st->thread);  in bringup_wait_for_ap_online()
795 		return -ECANCELED;  in bringup_wait_for_ap_online()
803 		return -EAGAIN;  in cpuhp_kick_ap_alive()
830 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in cpuhp_bringup_ap()
833 	return cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_bringup_ap()
847 		return -EAGAIN;  in bringup_cpu()
874 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in bringup_cpu()
877 	return cpuhp_kick_ap(cpu, st, st->target);  in bringup_cpu()
888 	struct mm_struct *mm = idle->active_mm;  in finish_cpu()
895 		idle->active_mm = &init_mm;  in finish_cpu()
908  * st->state will be modified ahead of time, to match state_to_run, as if it
918 			if (st->state >= target)  in cpuhp_next_state()
921 			*state_to_run = ++st->state;  in cpuhp_next_state()
923 			if (st->state <= target)  in cpuhp_next_state()
926 			*state_to_run = st->state--;  in cpuhp_next_state()
955 				cpuhp_get_step(st->state)->name,  in __cpuhp_invoke_callback_range()
956 				st->state, err);  in __cpuhp_invoke_callback_range()
957 			ret = -1;  in __cpuhp_invoke_callback_range()
994 	return st->state <= CPUHP_BRINGUP_CPU;  in can_rollback_cpu()
1000 	enum cpuhp_state prev_state = st->state;  in cpuhp_up_callbacks()
1006 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_up_callbacks()
1007 			 st->state);  in cpuhp_up_callbacks()
1024 	return st->should_run;  in cpuhp_should_run()
1035  *  - single: runs st->cb_state
1036  *  - up:     runs ++st->state, while st->state < st->target
1037  *  - down:   runs st->state--, while st->state > st->target
1044 	bool bringup = st->bringup;  in cpuhp_thread_fun()
1047 	if (WARN_ON_ONCE(!st->should_run))  in cpuhp_thread_fun()
1051 	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures  in cpuhp_thread_fun()
1052 	 * that if we see ->should_run we also see the rest of the state.  in cpuhp_thread_fun()
1064 	if (st->single) {  in cpuhp_thread_fun()
1065 		state = st->cb_state;  in cpuhp_thread_fun()
1066 		st->should_run = false;  in cpuhp_thread_fun()
1068 		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);  in cpuhp_thread_fun()
1069 		if (!st->should_run)  in cpuhp_thread_fun()
1077 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1083 		WARN_ON_ONCE(st->result);  in cpuhp_thread_fun()
1085 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1088 	if (st->result) {  in cpuhp_thread_fun()
1094 		WARN_ON_ONCE(st->rollback);  in cpuhp_thread_fun()
1095 		st->should_run = false;  in cpuhp_thread_fun()
1102 	if (!st->should_run)  in cpuhp_thread_fun()
1127 	if (!st->thread)  in cpuhp_invoke_ap_callback()
1130 	st->rollback = false;  in cpuhp_invoke_ap_callback()
1131 	st->last = NULL;  in cpuhp_invoke_ap_callback()
1133 	st->node = node;  in cpuhp_invoke_ap_callback()
1134 	st->bringup = bringup;  in cpuhp_invoke_ap_callback()
1135 	st->cb_state = state;  in cpuhp_invoke_ap_callback()
1136 	st->single = true;  in cpuhp_invoke_ap_callback()
1143 	if ((ret = st->result) && st->last) {  in cpuhp_invoke_ap_callback()
1144 		st->rollback = true;  in cpuhp_invoke_ap_callback()
1145 		st->bringup = !bringup;  in cpuhp_invoke_ap_callback()
1154 	st->node = st->last = NULL;  in cpuhp_invoke_ap_callback()
1161 	enum cpuhp_state prev_state = st->state;  in cpuhp_kick_ap_work()
1170 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);  in cpuhp_kick_ap_work()
1171 	ret = cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_kick_ap_work()
1172 	trace_cpuhp_exit(cpu, st->state, prev_state, ret);  in cpuhp_kick_ap_work()
1192 		init_completion(&st->done_up);  in cpuhp_init_state()
1193 		init_completion(&st->done_down);  in cpuhp_init_state()
1256  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1261  * trivial, there are various non-obvious corner cases, which this function
1274 	 * their mm mask. -- Peter Zijlstra  in clear_tasks_mm_cpumask()
1276 	 * full-fledged tasklist_lock.  in clear_tasks_mm_cpumask()
1290 		arch_clear_mm_cpumask_cpu(cpu, t->mm);  in clear_tasks_mm_cpumask()
1300 	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);  in take_cpu_down()
1310 	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.  in take_cpu_down()
1312 	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));  in take_cpu_down()
1334 	kthread_park(st->thread);  in takedown_cpu()
1350 		kthread_unpark(st->thread);  in takedown_cpu()
1363 	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);  in takedown_cpu()
1390 	BUG_ON(st->state != CPUHP_AP_OFFLINE);  in cpuhp_report_idle_dead()
1392 	st->state = CPUHP_AP_IDLE_DEAD;  in cpuhp_report_idle_dead()
1404 	enum cpuhp_state prev_state = st->state;  in cpuhp_down_callbacks()
1410 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_down_callbacks()
1411 			 st->state);  in cpuhp_down_callbacks()
1415 		if (st->state < prev_state)  in cpuhp_down_callbacks()
1431 		return -EBUSY;  in _cpu_down()
1434 		return -EINVAL;  in _cpu_down()
1445 	if (st->state > CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1446 		st->target = max((int)target, CPUHP_TEARDOWN_CPU);  in _cpu_down()
1459 		if (st->state > CPUHP_TEARDOWN_CPU)  in _cpu_down()
1462 		st->target = target;  in _cpu_down()
1469 	if (ret && st->state < prev_state) {  in _cpu_down()
1470 		if (st->state == CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1499 	return _cpu_down(work->cpu, 0, work->target);  in __cpu_down_maps_locked()
1511 		return -EOPNOTSUPP;  in cpu_down_maps_locked()
1513 		return -EBUSY;  in cpu_down_maps_locked()
1517 	 * CPU to prevent a deadlock against cfs_b->period_timer.  in cpu_down_maps_locked()
1521 		return -EBUSY;  in cpu_down_maps_locked()
1536  * cpu_device_down - Bring down a cpu device
1547 	return cpu_down(dev->id, CPUHP_OFFLINE);  in cpu_device_down()
1583 			pr_err("Failed to offline CPU%d - error=%d",  in smp_shutdown_nonboot_cpus()
1609  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1618 	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);  in notify_cpu_starting()
1650 	st->state = CPUHP_AP_ONLINE_IDLE;  in cpuhp_online_idle()
1664 		ret = -EINVAL;  in _cpu_up()
1672 	if (st->state >= target)  in _cpu_up()
1675 	if (st->state == CPUHP_OFFLINE) {  in _cpu_up()
1697 	if (st->state > CPUHP_BRINGUP_CPU) {  in _cpu_up()
1726 		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",  in cpu_up()
1731 		return -EINVAL;  in cpu_up()
1741 		err = -EBUSY;  in cpu_up()
1745 		err = -EPERM;  in cpu_up()
1756  * cpu_device_up - Bring up a cpu device
1767 	return cpu_up(dev->id, CPUHP_ONLINE);  in cpu_device_up()
1783  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1800 			pr_err("Failed to bring hibernate-CPU up!\n");  in bringup_hibernate_cpu()
1824 		if (!--ncpus)  in cpuhp_bringup_mask()
1880 		ncpus -= num_online_cpus();  in cpuhp_bringup_cpus_parallel()
1888 	/* Bring the not-yet started CPUs up */  in cpuhp_bringup_cpus_parallel()
1915 	if (primary == -1) {  in freeze_secondary_cpus()
1925 	 * We take down all of the non-boot CPUs in one shot to avoid races  in freeze_secondary_cpus()
1930 	pr_info("Disabling non-boot CPUs ...\n");  in freeze_secondary_cpus()
1937 			error = -EBUSY;  in freeze_secondary_cpus()
1955 		pr_err("Non-boot CPUs are not disabled\n");  in freeze_secondary_cpus()
1986 	pr_info("Enabling non-boot CPUs ...\n");  in thaw_secondary_cpus()
2011 		return -ENOMEM;  in alloc_frozen_cpus()
2124 	 * On the tear-down path, timers_dead_cpu() must be invoked
2156 	 * All-in-one CPU bringup state which includes the kick alive.
2277 		return -EINVAL;  in cpuhp_cb_check()
2301 		return -EINVAL;  in cpuhp_reserve_state()
2305 		if (!step->name)  in cpuhp_reserve_state()
2309 	return -ENOSPC;  in cpuhp_reserve_state()
2338 	if (name && sp->name)  in cpuhp_store_callbacks()
2339 		return -EBUSY;  in cpuhp_store_callbacks()
2341 	sp->startup.single = startup;  in cpuhp_store_callbacks()
2342 	sp->teardown.single = teardown;  in cpuhp_store_callbacks()
2343 	sp->name = name;  in cpuhp_store_callbacks()
2344 	sp->multi_instance = multi_instance;  in cpuhp_store_callbacks()
2345 	INIT_HLIST_HEAD(&sp->list);  in cpuhp_store_callbacks()
2351 	return cpuhp_get_step(state)->teardown.single;  in cpuhp_get_teardown_cb()
2399 		int cpustate = st->state;  in cpuhp_rollback_install()
2421 	if (sp->multi_instance == false)  in __cpuhp_state_add_instance_cpuslocked()
2422 		return -EINVAL;  in __cpuhp_state_add_instance_cpuslocked()
2426 	if (!invoke || !sp->startup.multi)  in __cpuhp_state_add_instance_cpuslocked()
2435 		int cpustate = st->state;  in __cpuhp_state_add_instance_cpuslocked()
2442 			if (sp->teardown.multi)  in __cpuhp_state_add_instance_cpuslocked()
2449 	hlist_add_head(node, &sp->list);  in __cpuhp_state_add_instance_cpuslocked()
2468  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2497 		return -EINVAL;  in __cpuhp_setup_state_cpuslocked()
2519 		int cpustate = st->state;  in __cpuhp_setup_state_cpuslocked()
2568 	if (!sp->multi_instance)  in __cpuhp_state_remove_instance()
2569 		return -EINVAL;  in __cpuhp_state_remove_instance()
2583 		int cpustate = st->state;  in __cpuhp_state_remove_instance()
2599  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2618 	if (sp->multi_instance) {  in __cpuhp_remove_state_cpuslocked()
2619 		WARN(!hlist_empty(&sp->list),  in __cpuhp_remove_state_cpuslocked()
2635 		int cpustate = st->state;  in __cpuhp_remove_state_cpuslocked()
2659 	dev->offline = true;  in cpuhp_offline_cpu_device()
2661 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);  in cpuhp_offline_cpu_device()
2668 	dev->offline = false;  in cpuhp_online_cpu_device()
2670 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);  in cpuhp_online_cpu_device()
2736 			  struct device_attribute *attr, char *buf)  in state_show()  argument
2738 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in state_show()
2740 	return sprintf(buf, "%d\n", st->state);  in state_show()
2744 static ssize_t target_store(struct device *dev, struct device_attribute *attr,  in target_store()  argument
2747 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_store()
2757 		return -EINVAL;  in target_store()
2760 		return -EINVAL;  in target_store()
2769 	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;  in target_store()
2774 	if (st->state < target)  in target_store()
2775 		ret = cpu_up(dev->id, target);  in target_store()
2776 	else if (st->state > target)  in target_store()
2777 		ret = cpu_down(dev->id, target);  in target_store()
2778 	else if (WARN_ON(st->target != target))  in target_store()
2779 		st->target = target;  in target_store()
2786 			   struct device_attribute *attr, char *buf)  in target_show()  argument
2788 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_show()
2790 	return sprintf(buf, "%d\n", st->target);  in target_show()
2794 static ssize_t fail_store(struct device *dev, struct device_attribute *attr,  in fail_store()  argument
2797 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_store()
2806 		st->fail = fail;  in fail_store()
2811 		return -EINVAL;  in fail_store()
2817 		return -EINVAL;  in fail_store()
2825 	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)  in fail_store()
2826 		return -EINVAL;  in fail_store()
2833 	if (!sp->startup.single && !sp->teardown.single)  in fail_store()
2834 		ret = -EINVAL;  in fail_store()
2839 	st->fail = fail;  in fail_store()
2845 			 struct device_attribute *attr, char *buf)  in fail_show()  argument
2847 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_show()
2849 	return sprintf(buf, "%d\n", st->fail);  in fail_show()
2855 	&dev_attr_state.attr,
2856 	&dev_attr_target.attr,
2857 	&dev_attr_fail.attr,
2868 				 struct device_attribute *attr, char *buf)  in states_show()  argument
2877 		if (sp->name) {  in states_show()
2878 			cur = sprintf(buf, "%3d: %s\n", i, sp->name);  in states_show()
2889 	&dev_attr_states.attr,
2909 __store_smt_control(struct device *dev, struct device_attribute *attr,  in __store_smt_control()  argument
2916 		return -EPERM;  in __store_smt_control()
2919 		return -ENODEV;  in __store_smt_control()
2936 			return -EINVAL;  in __store_smt_control()
2938 		return -EINVAL;  in __store_smt_control()
2961 __store_smt_control(struct device *dev, struct device_attribute *attr,  in __store_smt_control()  argument
2964 	return -ENODEV;  in __store_smt_control()
2977 			    struct device_attribute *attr, char *buf)  in control_show()  argument
2992 	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);  in control_show()
2995 static ssize_t control_store(struct device *dev, struct device_attribute *attr,  in control_store()  argument
2998 	return __store_smt_control(dev, attr, buf, count);  in control_store()
3003 			   struct device_attribute *attr, char *buf)  in active_show()  argument
3005 	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());  in active_show()
3010 	&dev_attr_control.attr,
3011 	&dev_attr_active.attr,
3024 	int ret = -ENODEV;  in cpu_smt_sysfs_init()
3028 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);  in cpu_smt_sysfs_init()
3045 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);  in cpuhp_sysfs_init()
3056 		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);  in cpuhp_sysfs_init()
3073 /* cpu_bit_bitmap[0] is empty - so we can back into it */