1 // SPDX-License-Identifier: GPL-2.0
3  * This file contains functions which emulate a local clock-event
6  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
7  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
8  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
20 #include "tick-internal.h"
23  * Broadcast support for broken x86 hardware, where the local apic
24  * timer stops in C3 state.
88 	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||  in tick_check_broadcast_device()
89 	    (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||  in tick_check_broadcast_device()
90 	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_check_broadcast_device()
94 	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_check_broadcast_device()
97 	return !curdev || newdev->rating > curdev->rating;  in tick_check_broadcast_device()
123 	if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||  in tick_set_oneshot_wakeup_device()
124 	    (newdev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_set_oneshot_wakeup_device()
127 	if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) ||  in tick_set_oneshot_wakeup_device()
128 	    !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_set_oneshot_wakeup_device()
131 	if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu)))  in tick_set_oneshot_wakeup_device()
134 	if (curdev && newdev->rating <= curdev->rating)  in tick_set_oneshot_wakeup_device()
137 	if (!try_module_get(newdev->owner))  in tick_set_oneshot_wakeup_device()
140 	newdev->event_handler = tick_oneshot_wakeup_handler;  in tick_set_oneshot_wakeup_device()
172 	if (!try_module_get(dev->owner))  in tick_install_broadcast_device()
177 		cur->event_handler = clockevents_handle_noop;  in tick_install_broadcast_device()
182 	if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))  in tick_install_broadcast_device()
215 	int ret = -ENODEV;  in tick_broadcast_update_freq()
228 	pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");  in err_broadcast()
233 	if (!dev->broadcast)  in tick_device_setup_broadcast_func()
234 		dev->broadcast = tick_broadcast;  in tick_device_setup_broadcast_func()
235 	if (!dev->broadcast) {  in tick_device_setup_broadcast_func()
237 			     dev->name);  in tick_device_setup_broadcast_func()
238 		dev->broadcast = err_broadcast;  in tick_device_setup_broadcast_func()
258 	 * the cpu local device.  in tick_device_uses_broadcast()
261 		dev->event_handler = tick_handle_periodic;  in tick_device_uses_broadcast()
274 		if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_device_uses_broadcast()
293 			 * state affected device to stop. Let the  in tick_device_uses_broadcast()
316 			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))  in tick_device_uses_broadcast()
330 	struct clock_event_device *evt = td->evtdev;  in tick_receive_broadcast()
333 		return -ENODEV;  in tick_receive_broadcast()
335 	if (!evt->event_handler)  in tick_receive_broadcast()
336 		return -EINVAL;  in tick_receive_broadcast()
338 	evt->event_handler(evt);  in tick_receive_broadcast()
349 	bool local = false;  in tick_do_broadcast()  local
359 		 * We only run the local handler, if the broadcast  in tick_do_broadcast()
363 		 * local timer_interrupt()  in tick_do_broadcast()
370 		local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);  in tick_do_broadcast()
381 		td->evtdev->broadcast(mask);  in tick_do_broadcast()
383 	return local;  in tick_do_broadcast()
388  * - invoke the broadcast handlers
415 		ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC);  in tick_handle_periodic_broadcast()
422 	 * We run the handler of the local cpu after dropping  in tick_handle_periodic_broadcast()
427 		td->evtdev->event_handler(td->evtdev);  in tick_handle_periodic_broadcast()
431  * tick_broadcast_control - Enable/disable or force broadcast mode
435  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
444 	/* Protects also the local clockevent device. */  in tick_broadcast_control()
447 	dev = td->evtdev;  in tick_broadcast_control()
452 	if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))  in tick_broadcast_control()
470 			 * Only shutdown the cpu local device, if:  in tick_broadcast_control()
472 			 * - the broadcast device exists  in tick_broadcast_control()
473 			 * - the broadcast device is not a hrtimer based one  in tick_broadcast_control()
474 			 * - the broadcast device is in periodic mode to  in tick_broadcast_control()
477 			if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&  in tick_broadcast_control()
517 		dev->event_handler = tick_handle_periodic;  in tick_set_periodic_handler()
519 		dev->event_handler = tick_handle_periodic_broadcast;  in tick_set_periodic_handler()
636 	if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))  in tick_broadcast_set_affinity()
639 	if (cpumask_equal(bc->cpumask, cpumask))  in tick_broadcast_set_affinity()
642 	bc->cpumask = cpumask;  in tick_broadcast_set_affinity()
643 	irq_set_affinity(bc->irq, bc->cpumask);  in tick_broadcast_set_affinity()
675 		if (td->mode == TICKDEV_MODE_ONESHOT) {  in tick_check_oneshot_broadcast_this_cpu()
676 			clockevents_switch_state(td->evtdev,  in tick_check_oneshot_broadcast_this_cpu()
693 	dev->next_event = KTIME_MAX;  in tick_handle_oneshot_broadcast()
708 		if (td->evtdev->next_event <= now) {  in tick_handle_oneshot_broadcast()
712 			 * it can avoid reprogramming the cpu local  in tick_handle_oneshot_broadcast()
713 			 * timer in tick_broadcast_oneshot_control().  in tick_handle_oneshot_broadcast()
716 		} else if (td->evtdev->next_event < next_event) {  in tick_handle_oneshot_broadcast()
717 			next_event = td->evtdev->next_event;  in tick_handle_oneshot_broadcast()
747 	 * - The global event did not expire any CPU local  in tick_handle_oneshot_broadcast()
751 	 * - There are pending events on sleeping CPUs which were not  in tick_handle_oneshot_broadcast()
761 		td->evtdev->event_handler(td->evtdev);  in tick_handle_oneshot_broadcast()
767 	if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))  in broadcast_needs_cpu()
769 	if (bc->next_event == KTIME_MAX)  in broadcast_needs_cpu()
771 	return bc->bound_on == cpu ? -EBUSY : 0;  in broadcast_needs_cpu()
779 	 * local device if our own event is the first one to expire or  in broadcast_shutdown_local()
780 	 * if we own the broadcast timer.  in broadcast_shutdown_local()
782 	if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {  in broadcast_shutdown_local()
785 		if (dev->next_event < bc->next_event)  in broadcast_shutdown_local()
795 	struct clock_event_device *bc, *dev = td->evtdev;  in ___tick_broadcast_oneshot_control()
807 		 * through the EXIT path as the local timer is not  in ___tick_broadcast_oneshot_control()
820 			if (bc->features & CLOCK_EVT_FEAT_HRTIMER)  in ___tick_broadcast_oneshot_control()
821 				ret = -EBUSY;  in ___tick_broadcast_oneshot_control()
828 			/* Conditionally shut down the local timer. */  in ___tick_broadcast_oneshot_control()
832 			 * We only reprogram the broadcast timer if we  in ___tick_broadcast_oneshot_control()
834 			 * if the cpu local event is earlier than the  in ___tick_broadcast_oneshot_control()
842 				ret = -EBUSY;  in ___tick_broadcast_oneshot_control()
843 			} else if (dev->next_event < bc->next_event) {  in ___tick_broadcast_oneshot_control()
844 				tick_broadcast_set_event(bc, cpu, dev->next_event);  in ___tick_broadcast_oneshot_control()
848 				 * timer to this cpu. If yes, remove  in ___tick_broadcast_oneshot_control()
864 			 * timer marked this cpu in the broadcast  in ___tick_broadcast_oneshot_control()
868 			 * handler. No need to reprogram the timer  in ___tick_broadcast_oneshot_control()
878 			if (dev->next_event == KTIME_MAX)  in ___tick_broadcast_oneshot_control()
886 			 * if the cpu local expiry time is already  in ___tick_broadcast_oneshot_control()
887 			 * reached, we would reprogram the cpu local  in ___tick_broadcast_oneshot_control()
888 			 * timer with an already expired event.  in ___tick_broadcast_oneshot_control()
890 			 * This can lead to a ping-pong when we return  in ___tick_broadcast_oneshot_control()
892 			 * timer before the cpu local timer was able  in ___tick_broadcast_oneshot_control()
897 			 * enough out that the ping-pong starts.  in ___tick_broadcast_oneshot_control()
899 			 * If the cpu local next_event has expired  in ___tick_broadcast_oneshot_control()
900 			 * then we know that the broadcast timer  in ___tick_broadcast_oneshot_control()
905 			 * will invoke the cpu local handler.  in ___tick_broadcast_oneshot_control()
913 			if (dev->next_event <= now) {  in ___tick_broadcast_oneshot_control()
919 			 * the cpu local timer device.  in ___tick_broadcast_oneshot_control()
921 			tick_program_event(dev->next_event, 1);  in ___tick_broadcast_oneshot_control()
935 	dev = td->evtdev;  in tick_oneshot_wakeup_control()
936 	if (td->mode != TICKDEV_MODE_ONESHOT)  in tick_oneshot_wakeup_control()
937 		return -EINVAL;  in tick_oneshot_wakeup_control()
941 		return -ENODEV;  in tick_oneshot_wakeup_control()
947 		clockevents_program_event(wd, dev->next_event, 1);  in tick_oneshot_wakeup_control()
952 			return -ENODEV;  in tick_oneshot_wakeup_control()
973 	return -EBUSY;  in __tick_broadcast_oneshot_control()
995 		if (td->evtdev)  in tick_broadcast_init_next_event()
996 			td->evtdev->next_event = expires;  in tick_broadcast_init_next_event()
1017  * tick_broadcast_setup_oneshot - setup the broadcast device
1027 	if (bc->event_handler != tick_handle_oneshot_broadcast) {  in tick_broadcast_setup_oneshot()
1030 		bc->event_handler = tick_handle_oneshot_broadcast;  in tick_broadcast_setup_oneshot()
1050 			bc->next_event = KTIME_MAX;  in tick_broadcast_setup_oneshot()
1092 		clockevents_program_event(bc, bc->next_event, 1);  in hotplug_cpu__broadcast_tick_pull()
1106 	 * Clear the broadcast masks for the dead cpu, but do not stop  in tick_broadcast_oneshot_offline()
1130 	return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;  in tick_broadcast_oneshot_available()
1138 	if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))  in __tick_broadcast_oneshot_control()
1139 		return -EBUSY;  in __tick_broadcast_oneshot_control()