116 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)  in tick_sched_do_timer()  argument
137 	if (ts->inidle)  in tick_sched_do_timer()
138 		ts->got_idle_tick = 1;  in tick_sched_do_timer()
141 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)  in tick_sched_handle()  argument
152 	if (ts->tick_stopped) {  in tick_sched_handle()
155 			ts->idle_jiffies++;  in tick_sched_handle()
161 		ts->next_tick = 0;  in tick_sched_handle()
201 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)  in can_stop_full_tick()  argument
211 	if (check_tick_dependency(&ts->tick_dep_mask))  in can_stop_full_tick()
306 	struct tick_sched *ts;  in tick_nohz_dep_set_cpu()  local
308 	ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_set_cpu()
310 	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_set_cpu()
327 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_clear_cpu()  local
329 	atomic_andnot(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_clear_cpu()
372 	struct tick_sched *ts;  in __tick_nohz_task_switch()  local
379 	ts = this_cpu_ptr(&tick_cpu_sched);  in __tick_nohz_task_switch()
381 	if (ts->tick_stopped) {  in __tick_nohz_task_switch()
470 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_tick_stopped()  local
472 	return ts->tick_stopped;  in tick_nohz_tick_stopped()
477 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_tick_stopped_cpu()  local
479 	return ts->tick_stopped;  in tick_nohz_tick_stopped_cpu()
509 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)  in update_ts_time_stats()  argument
513 	if (ts->idle_active) {  in update_ts_time_stats()
514 		delta = ktime_sub(now, ts->idle_entrytime);  in update_ts_time_stats()
516 			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);  in update_ts_time_stats()
518 			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in update_ts_time_stats()
519 		ts->idle_entrytime = now;  in update_ts_time_stats()
527 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
529 	update_ts_time_stats(smp_processor_id(), ts, now, NULL);  in tick_nohz_stop_idle()
530 	ts->idle_active = 0;  in tick_nohz_stop_idle()
535 static void tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
537 	ts->idle_entrytime = ktime_get();  in tick_nohz_start_idle()
538 	ts->idle_active = 1;  in tick_nohz_start_idle()
558 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
566 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_idle_time_us()
567 		idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
569 		if (ts->idle_active && !nr_iowait_cpu(cpu)) {  in get_cpu_idle_time_us()
570 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_idle_time_us()
572 			idle = ktime_add(ts->idle_sleeptime, delta);  in get_cpu_idle_time_us()
574 			idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
599 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_iowait_time_us()  local
607 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_iowait_time_us()
608 		iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
610 		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {  in get_cpu_iowait_time_us()
611 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_iowait_time_us()
613 			iowait = ktime_add(ts->iowait_sleeptime, delta);  in get_cpu_iowait_time_us()
615 			iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
623 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
625 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
626 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);  in tick_nohz_restart()
629 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_restart()
631 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_restart()
632 		hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);  in tick_nohz_restart()
634 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_restart()
640 	ts->next_tick = 0;  in tick_nohz_restart()
648 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)  in tick_nohz_next_event()  argument
659 	ts->last_jiffies = basejiff;  in tick_nohz_next_event()
660 	ts->timer_expires_base = basemono;  in tick_nohz_next_event()
684 		ts->next_timer = next_tmr;  in tick_nohz_next_event()
704 		if (!ts->tick_stopped) {  in tick_nohz_next_event()
705 			ts->timer_expires = 0;  in tick_nohz_next_event()
717 	    (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))  in tick_nohz_next_event()
726 	ts->timer_expires = min_t(u64, expires, next_tick);  in tick_nohz_next_event()
729 	return ts->timer_expires;  in tick_nohz_next_event()
732 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)  in tick_nohz_stop_tick()  argument
735 	u64 basemono = ts->timer_expires_base;  in tick_nohz_stop_tick()
736 	u64 expires = ts->timer_expires;  in tick_nohz_stop_tick()
740 	ts->timer_expires_base = 0;  in tick_nohz_stop_tick()
752 		ts->do_timer_last = 1;  in tick_nohz_stop_tick()
754 		ts->do_timer_last = 0;  in tick_nohz_stop_tick()
758 	if (ts->tick_stopped && (expires == ts->next_tick)) {  in tick_nohz_stop_tick()
760 		if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))  in tick_nohz_stop_tick()
765 			    basemono, ts->next_tick, dev->next_event,  in tick_nohz_stop_tick()
766 			    hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));  in tick_nohz_stop_tick()
776 	if (!ts->tick_stopped) {  in tick_nohz_stop_tick()
781 		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_tick()
782 		ts->tick_stopped = 1;  in tick_nohz_stop_tick()
786 	ts->next_tick = tick;  in tick_nohz_stop_tick()
793 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_tick()
794 			hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_tick()
798 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_stop_tick()
799 		hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);  in tick_nohz_stop_tick()
801 		hrtimer_set_expires(&ts->sched_timer, tick);  in tick_nohz_stop_tick()
806 static void tick_nohz_retain_tick(struct tick_sched *ts)  in tick_nohz_retain_tick()  argument
808 	ts->timer_expires_base = 0;  in tick_nohz_retain_tick()
812 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)  in tick_nohz_stop_sched_tick()  argument
814 	if (tick_nohz_next_event(ts, cpu))  in tick_nohz_stop_sched_tick()
815 		tick_nohz_stop_tick(ts, cpu);  in tick_nohz_stop_sched_tick()
817 		tick_nohz_retain_tick(ts);  in tick_nohz_stop_sched_tick()
821 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
838 	ts->tick_stopped  = 0;  in tick_nohz_restart_sched_tick()
839 	ts->idle_exittime = now;  in tick_nohz_restart_sched_tick()
841 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
844 static void tick_nohz_full_update_tick(struct tick_sched *ts)  in tick_nohz_full_update_tick()  argument
852 	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)  in tick_nohz_full_update_tick()
855 	if (can_stop_full_tick(cpu, ts))  in tick_nohz_full_update_tick()
856 		tick_nohz_stop_sched_tick(ts, cpu);  in tick_nohz_full_update_tick()
857 	else if (ts->tick_stopped)  in tick_nohz_full_update_tick()
858 		tick_nohz_restart_sched_tick(ts, ktime_get());  in tick_nohz_full_update_tick()
862 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)  in can_stop_idle_tick()  argument
878 		ts->next_tick = 0;  in can_stop_idle_tick()
882 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))  in can_stop_idle_tick()
918 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)  in __tick_nohz_idle_stop_tick()  argument
927 	if (ts->timer_expires_base)  in __tick_nohz_idle_stop_tick()
928 		expires = ts->timer_expires;  in __tick_nohz_idle_stop_tick()
929 	else if (can_stop_idle_tick(cpu, ts))  in __tick_nohz_idle_stop_tick()
930 		expires = tick_nohz_next_event(ts, cpu);  in __tick_nohz_idle_stop_tick()
934 	ts->idle_calls++;  in __tick_nohz_idle_stop_tick()
937 		int was_stopped = ts->tick_stopped;  in __tick_nohz_idle_stop_tick()
939 		tick_nohz_stop_tick(ts, cpu);  in __tick_nohz_idle_stop_tick()
941 		ts->idle_sleeps++;  in __tick_nohz_idle_stop_tick()
942 		ts->idle_expires = expires;  in __tick_nohz_idle_stop_tick()
944 		if (!was_stopped && ts->tick_stopped) {  in __tick_nohz_idle_stop_tick()
945 			ts->idle_jiffies = ts->last_jiffies;  in __tick_nohz_idle_stop_tick()
949 		tick_nohz_retain_tick(ts);  in __tick_nohz_idle_stop_tick()
980 	struct tick_sched *ts;  in tick_nohz_idle_enter()  local
986 	ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_enter()
988 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_enter()
990 	ts->inidle = 1;  in tick_nohz_idle_enter()
991 	tick_nohz_start_idle(ts);  in tick_nohz_idle_enter()
1006 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_exit()  local
1008 	if (ts->inidle)  in tick_nohz_irq_exit()
1009 		tick_nohz_start_idle(ts);  in tick_nohz_irq_exit()
1011 		tick_nohz_full_update_tick(ts);  in tick_nohz_irq_exit()
1019 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_got_tick()  local
1021 	if (ts->got_idle_tick) {  in tick_nohz_idle_got_tick()
1022 		ts->got_idle_tick = 0;  in tick_nohz_idle_got_tick()
1037 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
1043 	ktime_t now = ts->idle_entrytime;  in tick_nohz_get_sleep_length()
1046 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_get_sleep_length()
1050 	if (!can_stop_idle_tick(cpu, ts))  in tick_nohz_get_sleep_length()
1053 	next_event = tick_nohz_next_event(ts, cpu);  in tick_nohz_get_sleep_length()
1062 			   hrtimer_next_event_without(&ts->sched_timer));  in tick_nohz_get_sleep_length()
1075 	struct tick_sched *ts = tick_get_tick_sched(cpu);  in tick_nohz_get_idle_calls_cpu()  local
1077 	return ts->idle_calls;  in tick_nohz_get_idle_calls_cpu()
1087 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_idle_calls()  local
1089 	return ts->idle_calls;  in tick_nohz_get_idle_calls()
1092 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)  in tick_nohz_account_idle_ticks()  argument
1104 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_account_idle_ticks()
1113 static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)  in __tick_nohz_idle_restart_tick()  argument
1115 	tick_nohz_restart_sched_tick(ts, now);  in __tick_nohz_idle_restart_tick()
1116 	tick_nohz_account_idle_ticks(ts);  in __tick_nohz_idle_restart_tick()
1121 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_restart_tick()  local
1123 	if (ts->tick_stopped)  in tick_nohz_idle_restart_tick()
1124 		__tick_nohz_idle_restart_tick(ts, ktime_get());  in tick_nohz_idle_restart_tick()
1136 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_exit()  local
1142 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_idle_exit()
1143 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_exit()
1145 	ts->inidle = 0;  in tick_nohz_idle_exit()
1146 	idle_active = ts->idle_active;  in tick_nohz_idle_exit()
1147 	tick_stopped = ts->tick_stopped;  in tick_nohz_idle_exit()
1153 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
1156 		__tick_nohz_idle_restart_tick(ts, now);  in tick_nohz_idle_exit()
1166 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_handler()  local
1172 	tick_sched_do_timer(ts, now);  in tick_nohz_handler()
1173 	tick_sched_handle(ts, regs);  in tick_nohz_handler()
1176 	if (unlikely(ts->tick_stopped))  in tick_nohz_handler()
1179 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_handler()
1180 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_handler()
1183 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)  in tick_nohz_activate()  argument
1187 	ts->nohz_mode = mode;  in tick_nohz_activate()
1198 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_switch_to_nohz()  local
1211 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_nohz_switch_to_nohz()
1215 	hrtimer_set_expires(&ts->sched_timer, next);  in tick_nohz_switch_to_nohz()
1216 	hrtimer_forward_now(&ts->sched_timer, tick_period);  in tick_nohz_switch_to_nohz()
1217 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_switch_to_nohz()
1218 	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);  in tick_nohz_switch_to_nohz()
1223 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_enter()  local
1226 	if (!ts->idle_active && !ts->tick_stopped)  in tick_nohz_irq_enter()
1229 	if (ts->idle_active)  in tick_nohz_irq_enter()
1230 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1231 	if (ts->tick_stopped)  in tick_nohz_irq_enter()
1239 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }  in tick_nohz_activate()  argument
1262 	struct tick_sched *ts =  in tick_sched_timer()  local
1267 	tick_sched_do_timer(ts, now);  in tick_sched_timer()
1274 		tick_sched_handle(ts, regs);  in tick_sched_timer()
1276 		ts->next_tick = 0;  in tick_sched_timer()
1279 	if (unlikely(ts->tick_stopped))  in tick_sched_timer()
1302 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_setup_sched_timer()  local
1308 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_setup_sched_timer()
1309 	ts->sched_timer.function = tick_sched_timer;  in tick_setup_sched_timer()
1312 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
1319 		hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
1322 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_setup_sched_timer()
1323 	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);  in tick_setup_sched_timer()
1324 	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);  in tick_setup_sched_timer()
1331 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_cancel_sched_timer()  local
1334 	if (ts->sched_timer.base)  in tick_cancel_sched_timer()
1335 		hrtimer_cancel(&ts->sched_timer);  in tick_cancel_sched_timer()
1338 	memset(ts, 0, sizeof(*ts));  in tick_cancel_sched_timer()
1358 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_oneshot_notify()  local
1360 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
1373 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_check_oneshot_change()  local
1375 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
1378 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)  in tick_check_oneshot_change()