138 	.clock_base = { { .cpu_base = &migration_cpu_base, }, },
169 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
173 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
194 	return expires < new_base->cpu_base->expires_next;  in hrtimer_check_target()
248 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
249 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
253 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
254 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
282 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
487 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
497 	return &cpu_base->clock_base[idx];  in __next_base()
500 #define for_each_active_base(base, cpu_base, active)	\  argument
501 	while ((base = __next_base((cpu_base), &(active))))
503 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
511 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
534 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
536 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
567 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
573 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
574 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
575 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
576 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
579 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
583 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
584 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
585 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
611 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in __hrtimer_hres_active()  argument
614 		cpu_base->hres_active : 0;  in __hrtimer_hres_active()
628 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
635 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_force_reprogram()
637 	if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {  in hrtimer_force_reprogram()
643 		if (cpu_base->softirq_activated)  in hrtimer_force_reprogram()
644 			expires_next = __hrtimer_get_next_event(cpu_base,  in hrtimer_force_reprogram()
647 			cpu_base->softirq_expires_next = expires_next;  in hrtimer_force_reprogram()
650 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
653 	cpu_base->expires_next = expires_next;  in hrtimer_force_reprogram()
672 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_force_reprogram()
675 	tick_program_event(cpu_base->expires_next, 1);  in hrtimer_force_reprogram()
777 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
798 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
818 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
828 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
831 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
835 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
836 	cpu_base->expires_next = expires;  in hrtimer_reprogram()
847 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_reprogram()
898 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
967 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
988 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
996 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1006 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1007 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1029 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1057 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1064 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1078 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1198 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1201 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1202 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1203 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1204 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1205 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1246 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1247 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1248 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1249 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1313 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1317 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1319 	if (!__hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1320 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1322 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1336 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1340 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1342 	if (__hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1345 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1346 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1347 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1350 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1351 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1355 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1377 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_init()  local
1391 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_init()
1405 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1474 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1482 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1512 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1516 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1544 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1548 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1550 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1576 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1578 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1585 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1589 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1590 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1592 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1593 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1595 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1596 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1598 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1599 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1610 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1615 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1616 	cpu_base->nr_events++;  in hrtimer_interrupt()
1619 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1620 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1622 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1630 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1632 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1633 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1634 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1638 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1641 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_interrupt()
1646 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1647 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1648 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1652 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1669 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1670 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1671 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1680 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1681 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1682 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1685 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1686 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1723 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1727 	if (__hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1742 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1743 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1745 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1746 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1747 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1751 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1752 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
1987 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
1991 		cpu_base->clock_base[i].cpu_base = cpu_base;  in hrtimers_prepare_cpu()
1992 		timerqueue_init_head(&cpu_base->clock_base[i].active);  in hrtimers_prepare_cpu()
1995 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
1996 	cpu_base->active_bases = 0;  in hrtimers_prepare_cpu()
1997 	cpu_base->hres_active = 0;  in hrtimers_prepare_cpu()
1998 	cpu_base->hang_detected = 0;  in hrtimers_prepare_cpu()
1999 	cpu_base->next_timer = NULL;  in hrtimers_prepare_cpu()
2000 	cpu_base->softirq_next_timer = NULL;  in hrtimers_prepare_cpu()
2001 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2002 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2003 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()