166 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,  in lock_hrtimer_base()  argument
172 		base = timer->base;  in lock_hrtimer_base()
175 			if (likely(base == timer->base))  in lock_hrtimer_base()
194 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)  in hrtimer_check_target()  argument
198 	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);  in hrtimer_check_target()
226 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,  in switch_hrtimer_base()  argument
248 		if (unlikely(hrtimer_callback_running(timer)))  in switch_hrtimer_base()
252 		timer->base = &migration_base;  in switch_hrtimer_base()
257 		    hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
261 			timer->base = base;  in switch_hrtimer_base()
264 		timer->base = new_base;  in switch_hrtimer_base()
267 		    hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
278 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in lock_hrtimer_base()  argument
280 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()
354 	struct hrtimer *timer = addr;  in hrtimer_fixup_init()  local
358 		hrtimer_cancel(timer);  in hrtimer_fixup_init()
359 		debug_object_init(timer, &hrtimer_debug_descr);  in hrtimer_fixup_init()
388 	struct hrtimer *timer = addr;  in hrtimer_fixup_free()  local
392 		hrtimer_cancel(timer);  in hrtimer_fixup_free()
393 		debug_object_free(timer, &hrtimer_debug_descr);  in hrtimer_fixup_free()
408 static inline void debug_hrtimer_init(struct hrtimer *timer)  in debug_hrtimer_init()  argument
410 	debug_object_init(timer, &hrtimer_debug_descr);  in debug_hrtimer_init()
413 static inline void debug_hrtimer_activate(struct hrtimer *timer,  in debug_hrtimer_activate()  argument
416 	debug_object_activate(timer, &hrtimer_debug_descr);  in debug_hrtimer_activate()
419 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)  in debug_hrtimer_deactivate()  argument
421 	debug_object_deactivate(timer, &hrtimer_debug_descr);  in debug_hrtimer_deactivate()
424 static inline void debug_hrtimer_free(struct hrtimer *timer)  in debug_hrtimer_free()  argument
426 	debug_object_free(timer, &hrtimer_debug_descr);  in debug_hrtimer_free()
429 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
432 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init_on_stack()  argument
435 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);  in hrtimer_init_on_stack()
436 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init_on_stack()
440 void destroy_hrtimer_on_stack(struct hrtimer *timer)  in destroy_hrtimer_on_stack()  argument
442 	debug_object_free(timer, &hrtimer_debug_descr);  in destroy_hrtimer_on_stack()
448 static inline void debug_hrtimer_init(struct hrtimer *timer) { }  in debug_hrtimer_init()  argument
449 static inline void debug_hrtimer_activate(struct hrtimer *timer,  in debug_hrtimer_activate()  argument
451 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }  in debug_hrtimer_deactivate()  argument
455 debug_init(struct hrtimer *timer, clockid_t clockid,  in debug_init()  argument
458 	debug_hrtimer_init(timer);  in debug_init()
459 	trace_hrtimer_init(timer, clockid, mode);  in debug_init()
462 static inline void debug_activate(struct hrtimer *timer,  in debug_activate()  argument
465 	debug_hrtimer_activate(timer, mode);  in debug_activate()
466 	trace_hrtimer_start(timer, mode);  in debug_activate()
469 static inline void debug_deactivate(struct hrtimer *timer)  in debug_deactivate()  argument
471 	debug_hrtimer_deactivate(timer);  in debug_deactivate()
472 	trace_hrtimer_cancel(timer);  in debug_deactivate()
502 		struct hrtimer *timer;  in __hrtimer_next_event_base()  local
505 		timer = container_of(next, struct hrtimer, node);  in __hrtimer_next_event_base()
506 		if (timer == exclude) {  in __hrtimer_next_event_base()
512 			timer = container_of(next, struct hrtimer, node);  in __hrtimer_next_event_base()
514 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in __hrtimer_next_event_base()
522 			if (timer->is_soft)  in __hrtimer_next_event_base()
523 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
525 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
764 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)  in hrtimer_reprogram()  argument
767 	struct hrtimer_clock_base *base = timer->base;  in hrtimer_reprogram()
768 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
770 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);  in hrtimer_reprogram()
779 	if (timer->is_soft) {  in hrtimer_reprogram()
795 		timer_cpu_base->softirq_next_timer = timer;  in hrtimer_reprogram()
824 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
885 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in unlock_hrtimer_base()  argument
887 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
907 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)  in hrtimer_forward()  argument
912 	delta = ktime_sub(now, hrtimer_get_expires(timer));  in hrtimer_forward()
917 	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))  in hrtimer_forward()
927 		hrtimer_add_expires_ns(timer, incr * orun);  in hrtimer_forward()
928 		if (hrtimer_get_expires_tv64(timer) > now)  in hrtimer_forward()
936 	hrtimer_add_expires(timer, interval);  in hrtimer_forward()
950 static int enqueue_hrtimer(struct hrtimer *timer,  in enqueue_hrtimer()  argument
954 	debug_activate(timer, mode);  in enqueue_hrtimer()
958 	timer->state = HRTIMER_STATE_ENQUEUED;  in enqueue_hrtimer()
960 	return timerqueue_add(&base->active, &timer->node);  in enqueue_hrtimer()
973 static void __remove_hrtimer(struct hrtimer *timer,  in __remove_hrtimer()  argument
978 	u8 state = timer->state;  in __remove_hrtimer()
980 	timer->state = newstate;  in __remove_hrtimer()
984 	if (!timerqueue_del(&base->active, &timer->node))  in __remove_hrtimer()
995 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1003 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)  in remove_hrtimer()  argument
1005 	if (hrtimer_is_queued(timer)) {  in remove_hrtimer()
1006 		u8 state = timer->state;  in remove_hrtimer()
1017 		debug_deactivate(timer);  in remove_hrtimer()
1023 		__remove_hrtimer(timer, base, state, reprogram);  in remove_hrtimer()
1029 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,  in hrtimer_update_lowres()  argument
1038 	timer->is_rel = mode & HRTIMER_MODE_REL;  in hrtimer_update_lowres()
1039 	if (timer->is_rel)  in hrtimer_update_lowres()
1070 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in __hrtimer_start_range_ns()  argument
1077 	remove_hrtimer(timer, base, true);  in __hrtimer_start_range_ns()
1082 	tim = hrtimer_update_lowres(timer, tim, mode);  in __hrtimer_start_range_ns()
1084 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);  in __hrtimer_start_range_ns()
1087 	new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);  in __hrtimer_start_range_ns()
1089 	return enqueue_hrtimer(timer, new_base, mode);  in __hrtimer_start_range_ns()
1101 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in hrtimer_start_range_ns()  argument
1111 	WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);  in hrtimer_start_range_ns()
1113 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
1115 	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))  in hrtimer_start_range_ns()
1116 		hrtimer_reprogram(timer, true);  in hrtimer_start_range_ns()
1118 	unlock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
1132 int hrtimer_try_to_cancel(struct hrtimer *timer)  in hrtimer_try_to_cancel()  argument
1144 	if (!hrtimer_active(timer))  in hrtimer_try_to_cancel()
1147 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1149 	if (!hrtimer_callback_running(timer))  in hrtimer_try_to_cancel()
1150 		ret = remove_hrtimer(timer, base, false);  in hrtimer_try_to_cancel()
1152 	unlock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1167 int hrtimer_cancel(struct hrtimer *timer)  in hrtimer_cancel()  argument
1170 		int ret = hrtimer_try_to_cancel(timer);  in hrtimer_cancel()
1184 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)  in __hrtimer_get_remaining()  argument
1189 	lock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1191 		rem = hrtimer_expires_remaining_adjusted(timer);  in __hrtimer_get_remaining()
1193 		rem = hrtimer_expires_remaining(timer);  in __hrtimer_get_remaining()
1194 	unlock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1268 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in __hrtimer_init()  argument
1275 	memset(timer, 0, sizeof(struct hrtimer));  in __hrtimer_init()
1288 	timer->is_soft = softtimer;  in __hrtimer_init()
1289 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1290 	timerqueue_init(&timer->node);  in __hrtimer_init()
1305 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init()  argument
1308 	debug_init(timer, clock_id, mode);  in hrtimer_init()
1309 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init()
1320 bool hrtimer_active(const struct hrtimer *timer)  in hrtimer_active()  argument
1326 		base = READ_ONCE(timer->base);  in hrtimer_active()
1329 		if (timer->state != HRTIMER_STATE_INACTIVE ||  in hrtimer_active()
1330 		    base->running == timer)  in hrtimer_active()
1334 		 base != READ_ONCE(timer->base));  in hrtimer_active()
1360 			  struct hrtimer *timer, ktime_t *now,  in __run_hrtimer()  argument
1368 	debug_deactivate(timer);  in __run_hrtimer()
1369 	base->running = timer;  in __run_hrtimer()
1380 	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);  in __run_hrtimer()
1381 	fn = timer->function;  in __run_hrtimer()
1389 		timer->is_rel = false;  in __run_hrtimer()
1397 	trace_hrtimer_expire_entry(timer, now);  in __run_hrtimer()
1398 	restart = fn(timer);  in __run_hrtimer()
1399 	trace_hrtimer_expire_exit(timer);  in __run_hrtimer()
1412 	    !(timer->state & HRTIMER_STATE_ENQUEUED))  in __run_hrtimer()
1413 		enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);  in __run_hrtimer()
1424 	WARN_ON_ONCE(base->running != timer);  in __run_hrtimer()
1441 			struct hrtimer *timer;  in __hrtimer_run_queues()  local
1443 			timer = container_of(node, struct hrtimer, node);  in __hrtimer_run_queues()
1457 			if (basenow < hrtimer_get_softexpires_tv64(timer))  in __hrtimer_run_queues()
1460 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1638 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)  in hrtimer_wakeup()  argument
1641 		container_of(timer, struct hrtimer_sleeper, timer);  in hrtimer_wakeup()
1653 	sl->timer.function = hrtimer_wakeup;  in hrtimer_init_sleeper()
1685 		hrtimer_start_expires(&t->timer, mode);  in do_nanosleep()
1690 		hrtimer_cancel(&t->timer);  in do_nanosleep()
1702 		ktime_t rem = hrtimer_expires_remaining(&t->timer);  in do_nanosleep()
1719 	hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,  in hrtimer_nanosleep_restart()
1721 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);  in hrtimer_nanosleep_restart()
1724 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep_restart()
1740 	hrtimer_init_on_stack(&t.timer, clockid, mode);  in hrtimer_nanosleep()
1741 	hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);  in hrtimer_nanosleep()
1754 	restart->nanosleep.clockid = t.timer.base->clockid;  in hrtimer_nanosleep()
1755 	restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);  in hrtimer_nanosleep()
1757 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep()
1829 	struct hrtimer *timer;  in migrate_hrtimer_list()  local
1833 		timer = container_of(node, struct hrtimer, node);  in migrate_hrtimer_list()
1834 		BUG_ON(hrtimer_callback_running(timer));  in migrate_hrtimer_list()
1835 		debug_deactivate(timer);  in migrate_hrtimer_list()
1842 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);  in migrate_hrtimer_list()
1843 		timer->base = new_base;  in migrate_hrtimer_list()
1852 		enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);  in migrate_hrtimer_list()
1939 	hrtimer_init_on_stack(&t.timer, clock_id, mode);  in schedule_hrtimeout_range_clock()
1940 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);  in schedule_hrtimeout_range_clock()
1944 	hrtimer_start_expires(&t.timer, mode);  in schedule_hrtimeout_range_clock()
1949 	hrtimer_cancel(&t.timer);  in schedule_hrtimeout_range_clock()
1950 	destroy_hrtimer_on_stack(&t.timer);  in schedule_hrtimeout_range_clock()