Lines Matching +full:local +full:- +full:timer +full:- +full:stop
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
26 #include <linux/posix-timers.h>
32 #include "tick-internal.h"
34 #include <trace/events/timer.h>
37 * Per-CPU nohz control structure
199 if (ts->inidle) in tick_sched_do_timer()
200 ts->got_idle_tick = 1; in tick_sched_do_timer()
214 if (ts->tick_stopped) { in tick_sched_handle()
217 ts->idle_jiffies++; in tick_sched_handle()
223 ts->next_tick = 0; in tick_sched_handle()
280 if (check_tick_dependency(&ts->tick_dep_mask)) in can_stop_full_tick()
283 if (check_tick_dependency(¤t->tick_dep_mask)) in can_stop_full_tick()
286 if (check_tick_dependency(¤t->signal->tick_dep_mask)) in can_stop_full_tick()
302 * re-evaluate its dependency on the tick and restart it if necessary.
316 * re-evaluate its dependency on the tick and restart it if necessary.
334 * activate_task() STORE p->tick_dep_mask in tick_nohz_kick_task()
335 * STORE p->on_rq in tick_nohz_kick_task()
337 * LOCK rq->lock LOAD p->on_rq in tick_nohz_kick_task()
340 * LOAD p->tick_dep_mask in tick_nohz_kick_task()
351 * STORE p->cpu = @cpu in tick_nohz_kick_task()
353 * LOCK rq->lock in tick_nohz_kick_task()
354 * smp_mb__after_spin_lock() STORE p->tick_dep_mask in tick_nohz_kick_task()
356 * LOAD p->tick_dep_mask LOAD p->cpu in tick_nohz_kick_task()
367 * Kick all full dynticks CPUs in order to force these to re-evaluate
408 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
418 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
421 /* Perf needs local kick that is NMI safe */ in tick_nohz_dep_set_cpu()
425 /* Remote irq work not NMI-safe */ in tick_nohz_dep_set_cpu()
438 atomic_andnot(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_clear_cpu()
443 * Set a per-task tick dependency. RCU need this. Also posix CPU timers
448 if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) in tick_nohz_dep_set_task()
455 atomic_andnot(BIT(bit), &tsk->tick_dep_mask); in tick_nohz_dep_clear_task()
460 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
467 struct signal_struct *sig = tsk->signal; in tick_nohz_dep_set_signal()
469 prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask); in tick_nohz_dep_set_signal()
473 lockdep_assert_held(&tsk->sighand->siglock); in tick_nohz_dep_set_signal()
481 atomic_andnot(BIT(bit), &sig->tick_dep_mask); in tick_nohz_dep_clear_signal()
485 * Re-evaluate the need for the tick as we switch the current task.
498 if (ts->tick_stopped) { in __tick_nohz_task_switch()
499 if (atomic_read(¤t->tick_dep_mask) || in __tick_nohz_task_switch()
500 atomic_read(¤t->signal->tick_dep_mask)) in __tick_nohz_task_switch()
505 /* Get the boot-time nohz CPU list from the kernel parameters. */
522 return -EBUSY; in tick_nohz_cpu_down()
539 pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n"); in tick_nohz_init()
569 * NOHZ - aka dynamic tick functionality
591 return ts->tick_stopped; in tick_nohz_tick_stopped()
598 return ts->tick_stopped; in tick_nohz_tick_stopped_cpu()
602 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
625 * Updates the per-CPU time idle statistics counters
632 if (ts->idle_active) { in update_ts_time_stats()
633 delta = ktime_sub(now, ts->idle_entrytime); in update_ts_time_stats()
635 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in update_ts_time_stats()
637 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in update_ts_time_stats()
638 ts->idle_entrytime = now; in update_ts_time_stats()
649 ts->idle_active = 0; in tick_nohz_stop_idle()
656 ts->idle_entrytime = ktime_get(); in tick_nohz_start_idle()
657 ts->idle_active = 1; in tick_nohz_start_idle()
662 * get_cpu_idle_time_us - get the total idle time of a CPU
673 * This function returns -1 if NOHZ is not enabled.
681 return -1; in get_cpu_idle_time_us()
686 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
688 if (ts->idle_active && !nr_iowait_cpu(cpu)) { in get_cpu_idle_time_us()
689 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_idle_time_us()
691 idle = ktime_add(ts->idle_sleeptime, delta); in get_cpu_idle_time_us()
693 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
703 * get_cpu_iowait_time_us - get the total iowait time of a CPU
714 * This function returns -1 if NOHZ is not enabled.
722 return -1; in get_cpu_iowait_time_us()
727 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
729 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { in get_cpu_iowait_time_us()
730 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_iowait_time_us()
732 iowait = ktime_add(ts->iowait_sleeptime, delta); in get_cpu_iowait_time_us()
734 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
744 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
745 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
748 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_restart()
750 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
751 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
754 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_restart()
758 * Reset to make sure next tick stop doesn't get fooled by past in tick_nohz_restart()
761 ts->next_tick = 0; in tick_nohz_restart()
781 ts->last_jiffies = basejiff; in tick_nohz_next_event()
782 ts->timer_expires_base = basemono; in tick_nohz_next_event()
787 * Aside of that check whether the local timer softirq is in tick_nohz_next_event()
789 * because there is an already expired timer, so it will request in tick_nohz_next_event()
790 * immediate expiry, which rearms the hardware timer with a in tick_nohz_next_event()
799 * Get the next pending timer. If high resolution in tick_nohz_next_event()
800 * timers are enabled this only takes the timer wheel in tick_nohz_next_event()
806 ts->next_timer = next_tmr; in tick_nohz_next_event()
813 * force prod the timer. in tick_nohz_next_event()
815 delta = next_tick - basemono; in tick_nohz_next_event()
818 * Tell the timer code that the base is not idle, i.e. undo in tick_nohz_next_event()
823 * We've not stopped the tick yet, and there's a timer in the in tick_nohz_next_event()
826 if (!ts->tick_stopped) { in tick_nohz_next_event()
827 ts->timer_expires = 0; in tick_nohz_next_event()
839 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) in tick_nohz_next_event()
843 if (delta < (KTIME_MAX - basemono)) in tick_nohz_next_event()
848 ts->timer_expires = min_t(u64, expires, next_tick); in tick_nohz_next_event()
851 return ts->timer_expires; in tick_nohz_next_event()
857 u64 basemono = ts->timer_expires_base; in tick_nohz_stop_tick()
858 u64 expires = ts->timer_expires; in tick_nohz_stop_tick()
861 /* Make sure we won't be trying to stop it twice in a row. */ in tick_nohz_stop_tick()
862 ts->timer_expires_base = 0; in tick_nohz_stop_tick()
867 * the tick timer next, which might be this CPU as well. If we in tick_nohz_stop_tick()
874 ts->do_timer_last = 1; in tick_nohz_stop_tick()
876 ts->do_timer_last = 0; in tick_nohz_stop_tick()
880 if (ts->tick_stopped && (expires == ts->next_tick)) { in tick_nohz_stop_tick()
882 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) in tick_nohz_stop_tick()
886 …printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->exp… in tick_nohz_stop_tick()
887 basemono, ts->next_tick, dev->next_event, in tick_nohz_stop_tick()
888 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); in tick_nohz_stop_tick()
898 if (!ts->tick_stopped) { in tick_nohz_stop_tick()
902 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_tick()
903 ts->tick_stopped = 1; in tick_nohz_stop_tick()
907 ts->next_tick = tick; in tick_nohz_stop_tick()
910 * If the expiration time == KTIME_MAX, then we simply stop in tick_nohz_stop_tick()
911 * the tick timer. in tick_nohz_stop_tick()
914 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_tick()
915 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_tick()
919 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_tick()
920 hrtimer_start(&ts->sched_timer, tick, in tick_nohz_stop_tick()
923 hrtimer_set_expires(&ts->sched_timer, tick); in tick_nohz_stop_tick()
930 ts->timer_expires_base = 0; in tick_nohz_retain_tick()
949 * Clear the timer idle flag, so we avoid IPIs on remote queueing and in tick_nohz_restart_sched_tick()
957 * Cancel the scheduled timer and restore the tick in tick_nohz_restart_sched_tick()
959 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
971 else if (ts->tick_stopped) in __tick_nohz_full_update_tick()
981 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_update_tick()
992 * the CPU which runs the tick timer next. If we don't drop in can_stop_idle_tick()
1003 ts->next_tick = 0; in can_stop_idle_tick()
1007 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
1018 pr_warn("NOHZ tick-stop error: Non-RCU local softirq work is pending, handler #%02x!!!\n", in can_stop_idle_tick()
1033 /* Should not happen for nohz-full */ in can_stop_idle_tick()
1048 * tick timer expiration time is known already. in __tick_nohz_idle_stop_tick()
1050 if (ts->timer_expires_base) in __tick_nohz_idle_stop_tick()
1051 expires = ts->timer_expires; in __tick_nohz_idle_stop_tick()
1057 ts->idle_calls++; in __tick_nohz_idle_stop_tick()
1060 int was_stopped = ts->tick_stopped; in __tick_nohz_idle_stop_tick()
1064 ts->idle_sleeps++; in __tick_nohz_idle_stop_tick()
1065 ts->idle_expires = expires; in __tick_nohz_idle_stop_tick()
1067 if (!was_stopped && ts->tick_stopped) { in __tick_nohz_idle_stop_tick()
1068 ts->idle_jiffies = ts->last_jiffies; in __tick_nohz_idle_stop_tick()
1077 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
1079 * When the next event is more than a tick into the future, stop the idle tick
1097 * tick_nohz_idle_enter - prepare for entering idle on the current CPU
1111 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_enter()
1113 ts->inidle = 1; in tick_nohz_idle_enter()
1120 * tick_nohz_irq_exit - update next tick event from interrupt exit
1123 * a reschedule, it may still add, modify or delete a timer, enqueue
1125 * So we need to re-calculate and reprogram the next tick event.
1131 if (ts->inidle) in tick_nohz_irq_exit()
1138 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
1144 if (ts->got_idle_tick) { in tick_nohz_idle_got_tick()
1145 ts->got_idle_tick = 0; in tick_nohz_idle_got_tick()
1152 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
1160 return __this_cpu_read(tick_cpu_device.evtdev)->next_event; in tick_nohz_get_next_hrtimer()
1164 * tick_nohz_get_sleep_length - return the expected length of the current sleep
1182 ktime_t now = ts->idle_entrytime; in tick_nohz_get_sleep_length()
1185 WARN_ON_ONCE(!ts->inidle); in tick_nohz_get_sleep_length()
1187 *delta_next = ktime_sub(dev->next_event, now); in tick_nohz_get_sleep_length()
1197 * If the next highres timer to expire is earlier than next_event, the in tick_nohz_get_sleep_length()
1201 hrtimer_next_event_without(&ts->sched_timer)); in tick_nohz_get_sleep_length()
1207 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1216 return ts->idle_calls; in tick_nohz_get_idle_calls_cpu()
1220 * tick_nohz_get_idle_calls - return the current idle calls counter value
1228 return ts->idle_calls; in tick_nohz_get_idle_calls()
1236 ts->idle_exittime = now; in tick_nohz_account_idle_time()
1245 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_time()
1257 if (ts->tick_stopped) { in tick_nohz_idle_restart_tick()
1275 * tick_nohz_idle_exit - restart the idle tick from the idle task
1289 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
1290 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_exit()
1292 ts->inidle = 0; in tick_nohz_idle_exit()
1293 idle_active = ts->idle_active; in tick_nohz_idle_exit()
1294 tick_stopped = ts->tick_stopped; in tick_nohz_idle_exit()
1317 dev->next_event = KTIME_MAX; in tick_nohz_handler()
1323 if (unlikely(ts->tick_stopped)) in tick_nohz_handler()
1326 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_handler()
1327 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_handler()
1334 ts->nohz_mode = mode; in tick_nohz_activate()
1341 * tick_nohz_switch_to_nohz - switch to nohz mode
1358 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_nohz_switch_to_nohz()
1362 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1363 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); in tick_nohz_switch_to_nohz()
1364 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_switch_to_nohz()
1373 if (!ts->idle_active && !ts->tick_stopped) in tick_nohz_irq_enter()
1376 if (ts->idle_active) in tick_nohz_irq_enter()
1378 if (ts->tick_stopped) in tick_nohz_irq_enter()
1400 * High resolution timer specific code
1404 * We rearm the timer until we get disabled by the idle code.
1407 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) in tick_sched_timer() argument
1410 container_of(timer, struct tick_sched, sched_timer); in tick_sched_timer()
1423 ts->next_tick = 0; in tick_sched_timer()
1426 if (unlikely(ts->tick_stopped)) in tick_sched_timer()
1429 hrtimer_forward(timer, now, TICK_NSEC); in tick_sched_timer()
1445 * tick_setup_sched_timer - setup the tick emulation timer
1453 * Emulate tick processing via per-CPU hrtimers: in tick_setup_sched_timer()
1455 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_setup_sched_timer()
1456 ts->sched_timer.function = tick_sched_timer; in tick_setup_sched_timer()
1458 /* Get the next period (per-CPU) */ in tick_setup_sched_timer()
1459 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1466 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1469 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_setup_sched_timer()
1470 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); in tick_setup_sched_timer()
1481 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1482 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1507 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1513 * Called cyclic from the hrtimer softirq (driven by the timer
1514 * softirq) allow_nohz signals, that we can switch into low-res nohz
1522 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1525 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()