139 		.cpu_base = &migration_cpu_base,
173 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
177 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
198 	return expires < new_base->cpu_base->expires_next;  in hrtimer_check_target()
252 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
253 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
257 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
258 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
286 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
486 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
496 	return &cpu_base->clock_base[idx];  in __next_base()
499 #define for_each_active_base(base, cpu_base, active)	\  argument
500 	while ((base = __next_base((cpu_base), &(active))))
502 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
510 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
533 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
535 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
566 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
572 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
573 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
574 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
575 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
578 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
582 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
583 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
584 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
610 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in __hrtimer_hres_active()  argument
613 		cpu_base->hres_active : 0;  in __hrtimer_hres_active()
627 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
634 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_force_reprogram()
636 	if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {  in hrtimer_force_reprogram()
642 		if (cpu_base->softirq_activated)  in hrtimer_force_reprogram()
643 			expires_next = __hrtimer_get_next_event(cpu_base,  in hrtimer_force_reprogram()
646 			cpu_base->softirq_expires_next = expires_next;  in hrtimer_force_reprogram()
649 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
652 	cpu_base->expires_next = expires_next;  in hrtimer_force_reprogram()
671 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_force_reprogram()
674 	tick_program_event(cpu_base->expires_next, 1);  in hrtimer_force_reprogram()
776 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
797 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
817 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
827 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
830 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
834 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
835 	cpu_base->expires_next = expires;  in hrtimer_reprogram()
846 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_reprogram()
897 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
966 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
988 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
997 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1007 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1008 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1031 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1059 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1066 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1080 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1200 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1203 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1204 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1205 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1206 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1207 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1248 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1249 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1250 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1251 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1315 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1319 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1321 	if (!__hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1322 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1324 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1338 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1342 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1344 	if (__hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1347 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1348 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1349 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1352 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1353 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1357 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1379 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_init()  local
1393 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_init()
1407 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1476 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1479 			  unsigned long flags) __must_hold(&cpu_base->lock)  in __run_hrtimer()
1485 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1515 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1523 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1551 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1555 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1557 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1583 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1585 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1592 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1596 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1597 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1599 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1600 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1602 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1603 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1605 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1606 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1617 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1622 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1623 	cpu_base->nr_events++;  in hrtimer_interrupt()
1626 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1627 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1629 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1637 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1639 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1640 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1641 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1645 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1648 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_interrupt()
1653 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1654 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1655 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1659 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1676 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1677 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1678 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1687 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1688 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1689 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1692 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1693 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1730 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1734 	if (__hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1749 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1750 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1752 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1753 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1754 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1758 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1759 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
1996 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
2000 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];  in hrtimers_prepare_cpu()
2002 		clock_b->cpu_base = cpu_base;  in hrtimers_prepare_cpu()
2003 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);  in hrtimers_prepare_cpu()
2007 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
2008 	cpu_base->active_bases = 0;  in hrtimers_prepare_cpu()
2009 	cpu_base->hres_active = 0;  in hrtimers_prepare_cpu()
2010 	cpu_base->hang_detected = 0;  in hrtimers_prepare_cpu()
2011 	cpu_base->next_timer = NULL;  in hrtimers_prepare_cpu()
2012 	cpu_base->softirq_next_timer = NULL;  in hrtimers_prepare_cpu()
2013 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2014 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2015 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()