139 		.cpu_base = &migration_cpu_base,
174 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
178 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
199 	return expires < new_base->cpu_base->expires_next;  in hrtimer_check_target()
253 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
254 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
258 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
259 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
284 	__acquires(&timer->base->cpu_base->lock)  in lock_hrtimer_base()
288 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
488 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
498 	return &cpu_base->clock_base[idx];  in __next_base()
501 #define for_each_active_base(base, cpu_base, active)	\  argument
502 	while ((base = __next_base((cpu_base), &(active))))
504 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
512 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
535 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
537 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
571 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
577 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
578 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
579 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
580 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
583 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
587 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
588 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
589 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
596 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)  in hrtimer_update_next_event()  argument
605 	if (!cpu_base->softirq_activated) {  in hrtimer_update_next_event()
606 		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_next_event()
611 		cpu_base->softirq_expires_next = soft;  in hrtimer_update_next_event()
614 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);  in hrtimer_update_next_event()
620 		cpu_base->next_timer = cpu_base->softirq_next_timer;  in hrtimer_update_next_event()
646 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in __hrtimer_hres_active()  argument
649 		cpu_base->hres_active : 0;  in __hrtimer_hres_active()
657 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_reprogram()  argument
661 	cpu_base->expires_next = expires_next;  in __hrtimer_reprogram()
680 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in __hrtimer_reprogram()
692 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
696 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_force_reprogram()
698 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
701 	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);  in hrtimer_force_reprogram()
812 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
833 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
853 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
856 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
863 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
866 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
868 	__hrtimer_reprogram(cpu_base, timer, expires);  in hrtimer_reprogram()
871 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,  in update_needs_ipi()  argument
888 	seq = cpu_base->clock_was_set_seq;  in update_needs_ipi()
889 	hrtimer_update_base(cpu_base);  in update_needs_ipi()
895 	if (seq == cpu_base->clock_was_set_seq)  in update_needs_ipi()
903 	if (cpu_base->in_hrtirq)  in update_needs_ipi()
912 	active &= cpu_base->active_bases;  in update_needs_ipi()
914 	for_each_active_base(base, cpu_base, active) {  in update_needs_ipi()
919 		if (expires < cpu_base->expires_next)  in update_needs_ipi()
925 		if (cpu_base->softirq_activated)  in update_needs_ipi()
927 		if (expires < cpu_base->softirq_expires_next)  in update_needs_ipi()
949 	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);  in clock_was_set()  local
953 	if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)  in clock_was_set()
966 		cpu_base = &per_cpu(hrtimer_bases, cpu);  in clock_was_set()
967 		raw_spin_lock_irqsave(&cpu_base->lock, flags);  in clock_was_set()
969 		if (update_needs_ipi(cpu_base, bases))  in clock_was_set()
972 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in clock_was_set()
1018 	__releases(&timer->base->cpu_base->lock)  in unlock_hrtimer_base()
1020 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
1089 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
1111 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
1120 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1130 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1131 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1155 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1191 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1198 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1212 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1230 	force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in __hrtimer_start_range_ns()
1231 	force_local &= base->cpu_base->next_timer == timer;  in __hrtimer_start_range_ns()
1270 	hrtimer_force_reprogram(new_base->cpu_base, 1);  in __hrtimer_start_range_ns()
1369 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1372 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1373 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1374 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1375 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1376 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1417 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1418 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1419 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1420 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1484 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1488 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1490 	if (!__hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1491 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1493 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1507 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1511 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1513 	if (__hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1516 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1517 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1518 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1521 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1522 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1526 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1548 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_init()  local
1562 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_init()
1576 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1645 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1648 			  unsigned long flags) __must_hold(&cpu_base->lock)  in __run_hrtimer()
1654 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1684 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1692 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1720 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1724 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1726 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1752 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1754 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1761 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1765 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1766 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1768 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1769 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1771 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1772 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1774 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1775 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1786 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1791 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1792 	cpu_base->nr_events++;  in hrtimer_interrupt()
1795 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1796 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1798 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1806 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1808 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1809 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1810 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1814 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1817 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_interrupt()
1822 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1823 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1824 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1828 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1845 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1846 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1847 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1856 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1857 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1858 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1861 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1862 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1899 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1903 	if (__hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1918 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1919 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1921 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1922 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1923 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1927 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1928 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
2167 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
2171 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];  in hrtimers_prepare_cpu()
2173 		clock_b->cpu_base = cpu_base;  in hrtimers_prepare_cpu()
2174 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);  in hrtimers_prepare_cpu()
2178 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
2179 	cpu_base->active_bases = 0;  in hrtimers_prepare_cpu()
2180 	cpu_base->hres_active = 0;  in hrtimers_prepare_cpu()
2181 	cpu_base->hang_detected = 0;  in hrtimers_prepare_cpu()
2182 	cpu_base->next_timer = NULL;  in hrtimers_prepare_cpu()
2183 	cpu_base->softirq_next_timer = NULL;  in hrtimers_prepare_cpu()
2184 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2185 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2186 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()